彩票假说:随机初始化权重,得到训练后权重,若训练效果不好则再次随机初始化,训练效果好剪枝原始随机初始化。
网络剪枝:剪枝后性能下降(期望不要下降太多),要慢慢剪枝一次不要剪太多。
网络剪枝问题:权重剪枝:难以实现难以加速,网络结构变得不规整。
神经元剪枝:容易实现容易加速、网络结构规整
知识蒸馏
教师网络:大规模网络模型,集成网络,极深网络模型
学生网络:简单网络模型
代码
模型
class Student(nn.Module):
def __init__(self,img_c=3,ndf=32,num_class=10):
super(Student,self).__init__()
self.conv=nn.Sequential(
nn.Conv2d(in_channels=img_c,out_channels=ndf,kernel_size=3,stride=1,padding=1),#输出宽高不变
nn.BatchNorm2d(ndf),
nn.ReLU(inplace=True),
nn.Conv2d(in_channels=ndf,out_channels=ndf,kernel_size=3,stride=1,padding=1),
nn.BatchNorm2d(ndf),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2,stride=2),#图像大小缩小了一倍
nn.Conv2d(ndf,2*ndf,kernel_size=3,stride=1,padding=1),
nn.BatchNorm2d(2*ndf),
nn.ReLU(inplace=True),
nn.Conv2d(2*ndf,2*ndf,kernel_size=3,stride=1,padding=1),
nn.BatchNorm2d(2*ndf),
nn.ReLU(inplace=True)
)
self.fc=nn.Linear(2*ndf,num_class)
self.avg_pool=nn.AdaptiveAvgPool2d((1,1))#平均池化层将图像大小任意缩放,任意宽高变成1*1
def forward(self,x):
out=self.conv(x)
out=self.avg_pool(out)
out=out.view(out.size(0),-1)#压缩到一维中
out=self.fc(out)
return out
模型训练
class Distill_Trainer(object):
def __init__(self,lr=0.005,batch_size=64,num_epoch=1,train_data=None,test_data=None):
self.lr=lr
self.batch_size=batch_size
self.num_epoch=num_epoch
self.train_data=train_data
self.test_data=test_data
self.data_loader=Data.DataLoader(dataset=train_data,batch_size=batch_size,shuffle=True)
self.test_loader=Data.DataLoader(dataset=test_data,batch_size=batch_size,shuffle=True)
def train(self,model,save_path,save_name):
optim=torch.optim.Adam(model.parameters(),lr=self.lr)
best_acc=[]
acc_list=[]
for epoch in range(self.num_epoch):
model.train()
curr_loss=0
for i,(bx,by) in enumerate(self.data_loader):
pre=model(bx)
loss=F.cross_entropy(input=pre,target=by)
optim.zero_grad()
loss.backward()
optim.step()
curr_loss+=loss.item()
curr_acc=self.test(model)
acc_list.append(curr_acc)
return acc_list
计算准确率
def test(self,model):
acc=0
model.eval()
for i,(bx,by) in enumerate(self.test_loader):
with torch.no_grad():
pre=model(bx)
_,pred=torch.max(pre,1)
acc+=torch.sum(pred==by.data)
acc=acc.double()/self.test_loader.dataset.__len__()
return acc.item()
蒸馏训练
def train_distill(self,student_model,teacher_model,save_path,save_name);
optim=torch.optim.Adam(student_model.parameters(),lr=self.lr)
best_acc=0
acc_list=[]
kl_loss=torch.nn.KLDivLoss()
for epoch in range(self.num_epoch):
pre=student_model(bx)
for i,(bx,by) in enumerate(self.data_loader):
with torch.no_grad():
out_teacher=teacher_model(bx)
loss_dis=kl_loss(input=F.log_soft(pre/5.0,dim=1),target=F.softmax(out_teacher/5,dim=1))
loss=loss_dis+F.cross_entropy(input=pre,target=by)
optim.zero_grad()
loss.backward()
optim.step()
curr_acc=self.test(student_model)
acc_list.append(curr_acc)
return acc_list
执行
if __name__=='__main__':
lr=0.0005
batch_size=64
num_epoch=1
transforms_train=transforms.Compose([
transforms.RandomCrop(32,padding=4),
transforms.RandomHorizontalFlip(),
transforms.RandomRotation(10),
transforms.ToTensor(),
transforms.Normalize(0.5,0.5,0.5),(0.5,0.5,0.5),])
train_data=torchvision.datasets.CIFAR10(
root='./data/cifar_data',
train=True,
transform=transforms_train,
download=True)
transforms_test=transforms.Compose([
transforms.Totensor(),
transforms.Normalize((0.5,0.5,0.5),(0.5,0.5,0.5))])
test_data=torchvision.datasets.CIFAR10(
root='./data/cifar_data',
train=False,
transform=transforms_test,
download=True)
teach_model=models.resnet18(pretrained=True)
num_fc=teach_model.fc.in_features
teach_model.fc=torch.nn.Linear(num_fc,10)
student_model=Student(num_class=10)
norm_model=Student(num_class=10)
trainer=Distill_Trainer(train_data=train_data,test_data=test_data)
acc_list_normal=trainer.train(model=norm_model)
acc_List_teacher=trainer.train(mode=teacher_Model)
acc_list_student = trainer.train_distill(student_model=studen_model,teacher_model=teach_model)