Pytorch 预训练模型加载参数，VOC2007数据处理及数据集分类文件夹整理

常见的代码里的存储网络参数，和load 网络参数 

    def save(self, name):
        torch.save(self.model.state_dict(),
                   os.path.join(self.output, "%s.pth" % name))

    def load(self, path):
        print("Load model from %s" % path)
        state_dict = torch.load("%s.pth" % path)
        for key in list(state_dict.keys()):
            if '.module' in key:
                state_dict[key.replace('.module', '')] = state_dict.pop(key)
        self.model.load_state_dict(state_dict, strict=False)

简单的权重初始化操作

    def init_bert_weights(self, module):
        """ Initialize the weights.
        """
        if isinstance(module, (nn.Linear, nn.Embedding)):
            # Slightly different from the TF version which uses truncated_normal for initialization
            # cf https://github.com/pytorch/pytorch/pull/5617
            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
        elif isinstance(module, BertLayerNorm):
            module.bias.data.zero_()
            module.weight.data.fill_(1.0)
        if isinstance(module, nn.Linear) and module.bias is not None:
            module.bias.data.zero_()

model = Unet(3, 1)#,)
from torchsummaryX import summary
#summary(model, torch.zeros(5, 3, 512, 512))
for name in model.state_dict():
   print(" %s  :  %s" % (name, model.state_dict()[name].shape))
   #print(model.state_dict()[name])#这个是weight bias 对应的数值

import torch
from attention_unet_multi import AttU_Net
import torchvision.models as models #预训练模型都在这里面

vgg16 = models.vgg16(pretrained=True) #加载网络结构和预训练模型
pretrained_dict = vgg16.state_dict()  #返回内置预训练vgg模块的字典

#for name in pretrained_dict:
#   print(" %s  :  %s" % (name, pretrained_dict[name].shape))
#print(pretrained_dict)
model = AttU_Net(3, 1)
model_dict = model.state_dict()  #返回我们自己model的字典
#for name in model_dict:
#   print(" %s  :  %s" % (name, model_dict[name].shape))
#pretrained_dict = {k: v for k, v in pretrained_dict.items()}
#model_dict = {k: v for k, v in model_dict.items()}
#print(model_dict) 由于我的网络的层名称与预训练模型的层名不一致，所以可以手动构造字典，进行部分参数加载
pretr = { 'Conv1.conv.0.weight'  :  vgg16.state_dict()['features.0.weight'],
 'Conv1.conv.0.bias'  :  vgg16.state_dict()['features.0.bias'],

 'Conv1.conv.3.weight'  : vgg16.state_dict()['features.2.weight'],
 'Conv1.conv.3.bias'  :   vgg16.state_dict()['features.2.bias'],

 'Conv2.conv.0.weight' :  vgg16.state_dict()['features.5.weight'],
 'Conv2.conv.0.bias'  :   vgg16.state_dict()['features.5.bias'],

 'Conv2.conv.3.weight'  :   vgg16.state_dict()['features.7.weight'],
 'Conv2.conv.3.bias'  :   vgg16.state_dict()['features.7.bias'],


 'Conv3.conv.0.weight'  :  vgg16.state_dict()['features.10.weight'],
 'Conv3.conv.0.bias'  :  vgg16.state_dict()['features.10.bias'],


 'Conv3.conv.3.weight'  : vgg16.state_dict()['features.12.weight'],
 'Conv3.conv.3.bias'  :  vgg16.state_dict()['features.12.bias'],


 'Conv4.conv.0.weight'  : vgg16.state_dict()['features.17.weight'],
 'Conv4.conv.0.bias'  : vgg16.state_dict()['features.17.bias'],

 'Conv4.conv.3.weight'  :vgg16.state_dict()['features.19.weight'],
 'Conv4.conv.3.bias'  : vgg16.state_dict()['features.19.bias'],

}
model_dict.update(pretr)
model.load_state_dict(model_dict)

将图片尺寸满足要求大小的进行筛选，主要用于图像分割中图片大小不一的问题，为了裁剪尺寸准备

import os
import shutil
import torchvision.transforms as transforms
from PIL import Image
dir_path = './data/VOC2007/SegmentationClass'
for root, dirs, files in os.walk(dir_path):
    #list1 = ['./data/VOC2007/JPEGImages/'+n[:-4]+'.jpg' for n in files]
    list2 = [dir_path+'/'+n for n in files]
#print(list2)
file_dir = './data/trans/'
for x, d in enumerate(list2):
    #print(d)
    img = Image.open(d).convert('RGB')
    w, h = img.size
    if w < 300 or h < 300: #过滤长宽分别小于300的图片
        continue
    else:
        dd = str(file_dir+d[33:])#将长宽满足的图片拷贝到另一个文件夹下
        shutil.copy(d, dd)

将图片重新编码为了dataloader类

import os
import shutil
import torchvision.transforms as transforms
from PIL import Image
dir_path = './data/trans'
for root, dirs, files in os.walk(dir_path):
    #list1 = ['./data/VOC2007/JPEGImages/'+n[:-4]+'.jpg' for n in files]
    list2 = [n[:-4] for n in files]
print(list2)
for index, x in enumerate(list2):
    #拼接出要存放的文件夹的路径
    file_dir = './data/voc2'+'/'+str(index).zfill(3)+'.jpg'
    #将指定的文件file复制到file_dir的文件夹里面
    shutil.copy('./data/VOC2007/JPEGImages/'+x+'.jpg', file_dir)

快速生成序列文件夹，为了图像分类

import os
base = r'F:\FFOutput\test\noc' #新建文件夹的路径
i = 1
while i < 80: #新建1-80： 79个文件夹
    file_name = base+str(i) #以noc加数字为文件夹名称
    os.mkdir(file_name)
    i=i+1
————————————————
原文链接：https://blog.csdn.net/qq_38143062/article/details/85721402

图像的复制粘贴

import shutil
 
src='D:/zzz/bg.jpg'
dst='E:/tmp/bg.jpg'
shutil.copyfile(src,dst)
print('ok')

结果截图

linux 文件个数

 find . -type f -print | wc -l 

pytorch实现用Resnet提取特征并保存为txt文件的方法_-互联网文档类资源-CSDN下载 

我将上面分享的代码进行复制到这边来，感谢上面链接的分享

import os.path
import torch
import torch.nn as nn
from torchvision import models,transforms
from torch.autograd import Variable
import torchvision

import numpy as np
from PIL import Image
features_dir = './features'
img_path = "/home/byd/demo/COCO_val2014_000000218224.jpg"

transform1 = transforms.Compose([
    transforms.Scale(256),
    transforms.CenterCrop(224),
    transforms.ToTensor()
])
img = Image.open(img_path)
img1 = transform1(img)

model = torchvision.models.resnet101(pretrained=True)
model_extractor = nn.Sequential(*list(model.children())[:-1])
for param in model.parameters():
    param.requires_grad=False
x = Variable(torch.unsqueeze(img1,dim=0).float(), requires_grad=False)
y = model_extractor(x)
xx = y.data.numpy()
print(xx.shape,xx.device)
print(type(xx))

 

编写不易如果觉得不错，麻烦关注一下~