class AE_layer(nn.Module):
def __init__(self,layer_type,n_vis,n_hid,activation=nn.ReLU(),w_init=None):
super(AE_layer,self).__init__()
encoder_layers = []
#---------------------线性连接的方式-----------
layer = nn.Linear(n_vis,n_hid)
# -------------获取layer的两个参数：权重、偏置-----------
if w_init is not None:
# print(len(w_init))
layer.weight.data = torch.t(w_init[0])
layer.bias.data = w_init[1]
# else:
# layer.weight.data = torch.randn(n_hid,n_vis)*3e-1
# layer.bias.data = torch.zeros(n_hid)
# nn.init.xavier_normal_(layer.weight)
# -------------追加 权重、偏置-----------
encoder_layers.append(layer)
# -------------追加 激活函数-----------
if (layer_type == "First") or (layer_type == "Latent"):
encoder_layers.append(activation)
self.encoder_layers = nn.Sequential(*encoder_layers)
decoder_layers = []
layer = nn.Linear(n_hid,n_vis)
if w_init is not None:
layer.weight.data = torch.t(w_init[2])
layer.bias.data = w_init[3]
# else:
# layer.weight.data = torch.randn(n_vis,n_hid)*3e-1
# layer.bias.data = torch.zeros(n_vis)
# nn.init.xavier_normal_(layer.weight)
decoder_layers.append(layer)
if (layer_type == "Last") or (layer_type == "Latent"):
decoder_layers.append(activation)
self.decoder_layers = nn.Sequential(*decoder_layers)
def get_latent(self, inputs):
latent = self.encoder_layers(inputs)
return latent
def get_recon(self,inputs):
recon = self.decoder_layers(inputs)
return recon
def forward(self,inputs):
latent = self.encoder_layers(inputs)
outputs = self.decoder_layers(latent)
return outputs
class VAE(nn.Module):
def __init__(self,layer_sizes,inputs,activation=nn.ReLU(),w_init=None):
super(VAE,self).__init__()
n_layer = len(layer_sizes)
BATCH_SIZE = 100
dataloader = DataLoader(TensorDataset(torch.from_numpy(inputs)),batch_size=BATCH_SIZE,shuffle=True)
output_data = torch.from_numpy(inputs).cuda()
model_set = []
# -------------调用 AE_layer 函数，一层一层的（for循环5次）构建网络模型-------
for idx in range(n_layer-1):
if idx == 0:
layer_type = "First"
lr = 0.001
elif idx == n_layer-2:
layer_type = "Last"
lr = 0.0001
else:
layer_type = "Latent"
lr = 0.001
print("| Generate the {}th layer".format(idx+1))
if w_init is not None:
layer_model = AE_layer(layer_type,layer_sizes[idx],layer_sizes[idx+1],activation,w_init[4*idx:4*idx+4]).cuda()
else:
layer_model = AE_layer(layer_type,layer_sizes[idx],layer_sizes[idx+1],activation)
optimizer = optim.SGD(layer_model.parameters(),lr=lr,momentum=0.9)
lr_scheduler = StepLR(optimizer, step_size=70, gamma=0.1)
layer_model = layerwise_train(layer_model,dataloader,optimizer,lr_scheduler)
model_set.append(layer_model)
# 经过目前构造的encoder处理，output_data是当前隐层的数据。torch.Size([70000, 500])->torch.Size([70000, 500])->torch.Size([70000, 2000])->torch.Size([70000, 10])
output_data = layer_model.get_latent(output_data)
dataloader = DataLoader(TensorDataset(output_data),batch_size=BATCH_SIZE,shuffle=True)
self.model_set = model_set
# 变分层。是2000->10的地方
self.var_layer = AE_layer("Last",layer_sizes[-2],layer_sizes[-1])
#----获取网络模型的所有参数，追加到一块。我不太确定，到底有哪些参数
def get_para(self):
n_layer = len(self.model_set)
para = []
for idx in range(n_layer):
para.extend(self.model_set[idx].parameters())
para.extend(self.var_layer.parameters())
return para
def get_latent(self,inputs):
n_layer = len(self.model_set)
data = inputs
for idx in range(n_layer-1):
data = self.model_set[idx].get_latent(data)
x_mean = self.model_set[-1].get_latent(data)
x_logvar = self.var_layer.get_latent(data)
return x_mean, x_logvar
def get_recon(self,inputs):
n_layer = len(self.model_set)
data = inputs
for idx in reversed(range(n_layer)):
data = self.model_set[idx].get_recon(data)
recon = torch.sigmoid(data)
return recon
def forward(self):
pass
def gen_x(mean,std,J):
x_samples = []
v_size = mean.size()
for idx in range(J):
x_samples.append(mean + torch.mul(std,torch.randn(v_size).cuda()))
return x_samples
class Classifer(nn.Module):
def __init__(self,layer_sizes,activation=nn.ReLU()):
super(Classifer,self).__init__()
n_layer = len(layer_sizes) #2
layers = []
for idx in range(n_layer-1): #0
layers.append(nn.Linear(layer_sizes[idx],layer_sizes[idx+1]))
if idx < n_layer-2: #此句永远不成立
layers.append(activation)
else:
layers.append(nn.Softmax(dim=1))
self.model = nn.Sequential(*layers)
def forward(self,inputs):
output = self.model(inputs)
return output
class GMM_Model(nn.Module):
def __init__(self,N,K,mean=None,var=None,prior=None):
super(GMM_Model,self).__init__()
if mean is not None:
self.mean = nn.Parameter(torch.from_numpy(mean).view(1,N,K))
self.std = nn.Parameter(torch.sqrt(torch.from_numpy(var)).view(1,N,K))
else:
self.mean = nn.Parameter(torch.randn(1,N,K)) #1*10*10
self.std = nn.Parameter(torch.ones(1,N,K)) #1*10*10
self.N = N #10
self.K = K #10
def get_para(self):
return self.mean, self.std
def log_prob(self,data_mean,data_logvar,cond_prob,weight): #data_mean.shape：2100*10
term1 = torch.sum(-torch.log((self.std**2)*2*math.pi),dim=1)*0.5 #1*10,为啥不是横向压缩？？？？？？？？
#print('data_mean',data_mean[0:11,:])
#print('data_mean.view',data_mean.view(-1,self.N,1)[0,:,:])
#print('data_mean.view(-1,self.N,1).shape=', data_mean.view(-1, self.N, 1).shape) # 2100*10*1
#print('data_mean.view(-1,self.N,1)-self.mean=',(data_mean.view(-1,self.N,1)-self.mean).shape) # 2100*10*10
term2 = torch.sum(-torch.div(torch.pow(data_mean.view(-1,self.N,1)-self.mean,2)+torch.exp(data_logvar).view(-1,self.N,1),self.std**2),dim=1)*0.5 #2100*10
#print('term2.shape', term2.shape)
prob = term2 + term1
#print('prob.shape',prob.shape) # 2100*10
log_p1 = torch.sum(torch.mul(prob,cond_prob),dim=-1)
#print('log_p1.shape',log_p1.shape) # 2100
log_p = torch.sum(torch.mul(log_p1,weight))
#print('log_p',log_p) # 一个数
return log_p
def compute_prob(self,data):
prob = torch.exp(torch.sum(-torch.log((self.std**2)*2*math.pi)-torch.div(torch.pow(data.view(-1,self.N,1)-self.mean,2),self.std**2),dim=1)*0.5)
pc = torch.div(prob,(torch.sum(prob,dim=-1)).view(-1,1)+1e-10)
return pc
def compute_entropy(self,inputs,weight):
entropy1 = torch.sum(-torch.mul(inputs,torch.log(inputs+1e-10)),dim=-1)
entropy = torch.sum(torch.mul(entropy1,weight))
return entropy
def reg(self):
return torch.sum(torch.pow(torch.sum(torch.pow(self.mean,2),dim=1)-self.K*torch.ones(1,self.K).cuda(),2))
def forward(self):
pass
def pretrain(vae, optimizer,lr_scheduler,dataloader,epoch_num,use_gpu=torch.cuda.is_available()):
if use_gpu:
vae = vae.cuda()
J = 1
for epoch in range(epoch_num):
lr_scheduler.step()
Total_loss = []
Recon_loss = []
vae.train()
for batch_idx, batch in enumerate(dataloader):
if use_gpu:
inputs = Variable(batch[0].cuda())
else:
inputs = Variable(batch[0])
x_mean, x_logvar = vae.get_latent(inputs)
ELBO_rec = 0
x_re = vae.get_recon(x_mean)
ELBO_rec = ELBO_rec + torch.sum(torch.mul(inputs,torch.log(x_re+1e-10))+torch.mul(1-inputs,torch.log(1-x_re+1e-10)))
ELBO_reg = 0
loss = -ELBO_rec - ELBO_reg
optimizer.zero_grad()
loss.backward()
optimizer.step()
Total_loss.append(loss.item())
Recon_loss.append(ELBO_rec.item())
print('|Epoch:{} Total loss={:3f} Recon_loss={:3f}'.format(epoch,np.mean(Total_loss),-np.mean(Recon_loss)))
return vae
def pretrain_classifer1(vae,classifer,optimizer,lr_scheduler,dataloader,epoch_num,use_gpu=torch.cuda.is_available()):
if use_gpu:
vae = vae.cuda()
classifer= classifer.cuda()
loss_f = nn.CrossEntropyLoss()
for epoch in range(epoch_num): #30
lr_scheduler.step()
Total_loss = []
#----eval和train的区别
#1、训练NN时，用train，测试NN是，用eval
#2、train()：启用 BatchNormalization 和 Dropout
#3、eval（）：框架会自动把BN和DropOut固定住，不会取平均，而是用训练好的值
#4、因此在model(test)之前，需要加上model.eval()，否则的话，有输入数据，即使不训练，它也会改变权值。这是model中含有batch normalization层所带来的的性质。
#5、我有两个model1，model2，model2的输入是model1的输出，我只是训练model2 ，则model2.train（），请问我model1需要设置么？
#答：如果使用两个模型进行联合训练，为了收敛更容易控制，先预训练好模型model_A，并且model_A内还有若干BN层，后续需要将model_A作为一个推理模型和model_B联合训练，此时希望model_A中的BN的统计特性量不会乱变化，因此就需要将model_A.eval()设置到测试模型，否则在trainning模式下，就算是不去更新模型的参数，其BN都会变化，这将导致和预期不同的结果
vae.eval()
classifer.train()
label = []
label_pred = []
entropy_p = 0
for batch_idx, batch in enumerate(dataloader): #70000/100=700
if use_gpu:
x_mean, x_logvar = vae.get_latent(batch[0].cuda()) #image-train:70000
x_samples = gen_x(x_mean,torch.exp(0.5*x_logvar),1)
inputs = Variable(x_mean)
#print('inputs.shape=',inputs.shape) # 100*10(batch size * 10维度)
label_train = Variable(batch[2].cuda()) # kmeans lab
#print('label_train.shape',label_train.shape) # 100
label.append(batch[1]) # true lab
else:
x_mean, x_logvar = vae.get_latent(batch[0])
inputs = Variable(x_mean)
label_train = Variable(batch[2])
label.append(batch[1])
cond_prob = classifer(inputs) # vae产生的 均值 进 分类器
#print('cond_prob=',cond_prob)
#print('cond_prob.shape=', cond_prob.shape)#100*10
loss = loss_f(cond_prob,label_train.long()) #分类器的结果和kmeans的结果做loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
Total_loss.append(loss.item())
classifer.eval()
pred_l = torch.max(classifer(inputs.data),dim=-1)
label_pred.append(pred_l[-1])
entropy_p = entropy_p + torch.sum(torch.pow(cond_prob,2)).item()
label = torch.cat(label,dim=0) # true lab
label_pred = torch.cat(label_pred,dim=0) #训练的lab
NMI = metrics.normalized_mutual_info_score(to_numpy(label), to_numpy(label_pred),average_method='max')
acc,_ = cluster_acc(to_numpy(label), to_numpy(label_pred))
print('|Epoch:{} Total loss={:3f} ACC={:5f} NMI={:5f} Entropy={:2f}'.format(epoch,np.mean(Total_loss),acc,NMI,entropy_p))
return classifer
def cluster_acc(Y_pred, Y):
assert Y_pred.size == Y.size
D = max(Y_pred.max(), Y.max())+1
w = np.zeros((D,D), dtype=np.int64)
for i in range(Y_pred.size):
w[Y_pred[i], Y[i]] += 1
row_ind, col_ind = linear_sum_assignment(w.max() - w)
return w[row_ind,col_ind].sum()/Y_pred.size, w
def compute_weight(inputs,sigma,similarity_type='Gauss',use_gpu=torch.cuda.is_available()):
dist = torch.sum(torch.pow(inputs-inputs[:,:,0].unsqueeze(2),2),dim=1) #shape:100*21
#print('dist',dist.shape)
#print('dist',dist)
dist_temp,_ = torch.sort(dist) #对dist从小到大排序，0，xx，xx，...
#print('dist_temp', dist_temp)
sigma = dist_temp[11] #选择了第11个？？？？？？？？？？？？
#print('sigma', sigma)
dist = dist/sigma #第一列为nan，不应该是0吗，因为0/sigma.
#print('dist', dist)
if similarity_type == 'Gauss':
Gauss_simi = torch.exp(-dist)
Gauss_simi[:,0] = torch.sum(Gauss_simi[:,1:],dim=1)
simi = torch.div(Gauss_simi,torch.sum(Gauss_simi,dim=1,keepdim=True))
elif similarity_type == 'Student-t':
t_simi = torch(1,1+dist)
t_simi[:,0] = torch.sum(t_simi[:,1:],dim=1)
simi = torch.div(t_simi,torch.sum(t_simi,dim=1,keepdim=True))
elif similarity_type == 'inv-Guass':
Gauss_simi = torch.exp(dist)
Gauss_simi[:,0] = torch.sum(Gauss_simi[:,1:],dim=1)
simi = torch.div(Gauss_simi,torch.sum(Gauss_simi,dim=1,keepdim=True))
simi[:,0] = simi[:,0]
elif similarity_type == 'No_inform':
N = inputs.size(-1) #21
simi = torch.ones(1,N)/(N-1)
simi[0,0] = 1
#print('simi',simi)
simi = torch.mul(torch.ones(inputs.size(0),1),simi) # shape:100*21
#print('simi.shape',simi.shape)
#print('simi', simi)
simi = torch.div(simi,torch.sum(simi,dim=1,keepdim=True))
elif similarity_type == 'KL':
N = inputs.size(-1)
simi = torch.ones(1,N)/(N-1)
simi[0,0] = 1/N
simi = torch.mul(torch.ones(inputs.size(0),1),simi)
simi = torch.div(simi,torch.sum(simi,dim=1,keepdim=True))
return simi.cuda()
#可视化
fig = plt.figure(figsize=(7, 6))
plt.plot(AgDecx, AGDEC, linestyle='-',label='AG-DEC')
plt.plot(AgDecx, GDEC, linestyle='-',label='G-DEC')
plt.plot(AgDecx, GrDNFCS, linestyle='-',label='GrDNFCS')
plt.plot(AgDecx, IDEC, linestyle='-',label='IDEC')
plt.plot(AgDecx, DFKM, linestyle='-',label='DFKM')
#plt.plot(AgDecx, DEC, linestyle='-',label='DEC')
plt.plot(AgDecx, DCN, linestyle='-',label='DCN')
plt.xlim((0,300))
plt.xlabel("Number of iterations")#x轴上的名字
plt.ylabel("loss")#x轴上的名字
plt.legend(loc = 'upper right')
plt.show()
扫一扫
04-16
1847
1847
07-11
410
410
07-10
2796
2796
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
