自定义数据集pytorch深度学习建立
本文包括原图像到网络层创建及loss等输出
模块的导入
里面导入了所以的将会使用的模块，torchvision已经安装，但是这个查询版本的子程序找不到。
from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import torchvision
from torchvision import datasets, models, transforms
import matplotlib.pyplot as plt
import time #用来看运行的时间
import os #用来读取文件
from torch.utils.data import DataLoader
from torch.utils.data import TensorDataset
from PIL import Image #跟cv2差不多
import torch.nn.functional as F #设置网络层的模块
from torchvision.models import AlexNet #这个可以导入很多流行的网络结构
print("PyTorch Version: ",torch.__version__) #看看安装了没
#print("Torchvision Version: ",torchvision.__version__) #不知道我安装的这个版本怎么用不了
dev = torch.device( "cuda") if torch.cuda.is_available() else torch.device("cpu") #设置使用cpu还是gpu
数据封装
这一部分是原图像的封装，需要一个txt文件，里面存放有图像路径及他的标签。类似：
1/PASSIVE1556.bmp 1
1/PASSIVE1993.bmp 1
1/PASSIVE694.bmp 1
前面是图像存放路径，以一个空格隔开，后面是标签。
class MyDataset(torch.utils.data.Dataset): #创建自己的类：MyDataset,这个类是继承的torch.utils.data.Dataset
def __init__(self,root, datatxt, transform=None, target_transform=None): #初始化一些需要传入的参数
super(MyDataset,self).__init__()
fh = open(root + datatxt, 'r') #按照传入的路径和txt文本参数，打开这个文本，并读取内容
imgs = [] #创建一个名为img的空列表，一会儿用来装东西
for line in fh: #按行循环txt文本中的内容
line = line.rstrip() # 删除 本行string 字符串末尾的指定字符，这个方法的详细介绍自己查询python
words = line.split() #通过指定分隔符对字符串进行切片，默认为所有的空字符，包括空格、换行、制表符等
imgs.append((words[0],int(words[1]))) #把txt里的内容读入imgs列表保存，具体是words【？】要看txt内容而定
# 很显然，根据我刚才截图所示txt的内容，words[0]是图片信息，words[1]是lable
self.imgs = imgs #默认必须的
self.transform = transform #后续可以选择多种，用来数据强化的。
self.target_transform = target_transform
def __getitem__(self, index): #这个方法是必须要有的，用于按照索引读取每个元素的具体内容
fn, label = self.imgs[index] #fn是图片path #fn和label分别获得imgs[index]也即是刚才每行中word[0]和word[1]的信息
img = Image.open(root+fn).convert('L').resize((image_size,image_size)) #按照path读入图片from PIL import Image # 按照路径读取图片
#并且统一了图像尺寸
if self.transform is not None:
img = self.transform(img) #是否进行transform
return img,label #return很关键，return回哪些内容，那么我们在训练时循环读取每个batch时，就能获得哪些内容
def __len__(self): #这个函数也必须要写，它返回的是数据集的长度，也就是多少张图片，要和loader的长度作区分
return len(self.imgs)
#根据自己定义的那个勒MyDataset来创建数据集！注意是数据集！而不是loader迭代器
def datacreate(root):
train_data=MyDataset(root,'train.txt', transform=transforms.ToTensor()) #读取一大波图像
val_data=MyDataset(root,'val.txt', transform=transforms.ToTensor())
train_first=0 #这个可以后续优化，得去找找函数。看起来很难受
val_first=0
for batch_index, batch in train_data: #封装数据了
if train_first==0: #用来第一个赋值的，十分别扭
x_train=batch_index.view(1,len(batch_index.reshape(-1)))
train_first=1
y_train= torch.from_numpy(np.array(batch)).view(1)
print('train_set start to deal')
continue
train_first+=1
if train_first%300==0:
print('300 have done')
x_train = torch.cat((x_train,batch_index.view(1,len(batch_index.reshape(-1)))), 0) #tensor的合并，使得每一tensor变成一个同维度的
y_train = torch.cat((y_train,torch.from_numpy(np.array(batch)).view(1)), 0)
for batch_index, batch in val_data: #同理
if val_first==0:
x_val=batch_index.view(1,len(batch_index.reshape(-1)))
val_first=1
y_val= torch.from_numpy(np.array(batch)).view(1)
print('val_set start to deal')
continue
val_first+=1
if val_first%300==0:
print('300 have done')
x_val = torch.cat((x_val,batch_index.view(1,len(batch_index.reshape(-1)))), 0)
y_val = torch.cat((y_val,torch.from_numpy(np.array(batch)).view(1)), 0)
print('finished')
#train_loader = DataLoader(dataset=train_data , batch_size=len(train_data), shuffle=True)
#val_loader = DataLoader(dataset=val_data,batch_size=len(val_data))
#for batch_index, batch in train_loader:
# train_x=batch_index
# train_x=train_x.reshape(len(train_x[:]),-1)
# train_y = batch
#for batch_index, batch in val_loader:
# val_x=batch_index
# val_x=val_x.reshape(len(val_x[:]),-1)
# val_y = batch
return x_train,y_train,x_val,y_val
实际数据封装
导入数据了。对gpu内存占得还挺大
root='./glass/'
image_size=256
x_train, y_train,x_val,y_val=datacreate(root)
建立网络结构
其中包括loss、acc、forward等建立。
def loss_batch_train(model, loss_func, xb, yb, opt=None):
loss = loss_func(model(xb), yb)
#print(model(xb),yb,loss)
if opt is not None: #如果是空，就是没有反向传递
loss.backward()
opt.step()
opt.zero_grad()
return loss.item(), len(xb)
def loss_batch_val(model, loss_func, xb, yb, opt=None):
loss = loss_func(model(xb), yb) #这个和下一个应该改可以合成一个
outputs = model(xb)
_, prediction = torch.max(outputs.data, 1) #计算ACC
test_acc = torch.sum(prediction == yb.data)
#print(loss.item(), len(xb), test_acc.item())
#print('acc is ',test_acc,' prediction ',prediction, len(xb))
#print('test_acc, ',test_acc.item(),' loss ',loss.item())
#print(model(xb),yb,loss)
if opt is not None:
loss.backward()
opt.step()
opt.zero_grad()
return loss.item(), len(xb), test_acc.item()/len(xb)
def fit(epochs, model, loss_func, opt, train_dl, valid_dl):
for epoch in range(epochs):
model.train() #开启反向传递
#for xb, yb in train_dl:
#loss_batch(model, loss_func, xb, yb, opt)
losses, nums = zip(*[loss_batch_train(model, loss_func, xb, yb, opt) for xb, yb in train_dl])
# k1= [loss_batch(model, loss_func, xb, yb,opt) for xb, yb in train_dl]
# k2=losses
# k3=nums
#print(loss_batch(model, loss_func, xb, yb,opt)for xb, yb in train_dl)
train_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)
model.eval() #关闭反向传递
with torch.no_grad(): #和上面得差不多重复了 没啥用
losses, nums, prediction = zip(*[loss_batch_val(model, loss_func, xb, yb) for xb, yb in valid_dl])
val_loss = np.sum(np.multiply(losses, nums)) / np.sum(nums)
#print(prediction,nums)
acc = np.sum(np.multiply(prediction, nums)) / np.sum(nums)
scheduler.step(val_loss) #自适应学习率改变
val_loss_list.append(val_loss) #用来画曲线
train_loss_list.append(train_loss)
acc_list.append(acc)
if epoch%10==0:
print(epoch, ' val_loss is ',val_loss,' train_loss is ',train_loss,' acc is ',acc)
#return k1,k2,k3
def get_data(train_ds, valid_ds, bs): #数据得读取
return (
DataLoader(train_ds, batch_size=bs, shuffle=True),
DataLoader(valid_ds, batch_size=bs * 2), #没必要打乱数据
)
class WrappedDataLoader:
def __init__(self, dl, func):
self.dl = dl
self.func = func
def __len__(self):
return len(self.dl)
def __iter__(self):
batches = iter(self.dl)
for b in batches:
yield (self.func(*b))
def preprocess(x, y): #数据转成gpu格式
return x.view(-1, 1, image_size, image_size).to(dev), y.to(dev)
class Mynet(nn.Module): #我的网络结构
def __init__(self):
super().__init__()
self.conv1=nn.Conv2d(1,20,kernel_size=5,stride=1)
self.conv2=nn.Conv2d(20,50,kernel_size=5,stride=1)
self.fc1=nn.Linear(int(50*((((image_size-4)/2-4)/2))**2),500) #这个好麻烦，要是有一个自动的就好了
#self.fc1=nn.Linear(186050,500)
self.fc2=nn.Linear(500,2)
def forward(self,x):
#print(x.shape,x.size(0))
#print(x.shape)
x = F.max_pool2d(self.conv1(x),2)
#print(x.shape)
x = F.max_pool2d(self.conv2(x),2)
#x=self.conv2(x)
#print(x.shape)
x = x.view(x.size(0),-1)
x = self.fc1(x)
#print(x.shape)
x = self.fc2(x)
#print(x.shape)
#print(x.view(-1, x.size(1)).shape)
#print(x.view(x.size(0), -1).shape)
return (x.view(-1, x.size(1)))
开始训练
bs=64 #batchsize，越大越好，这个是我设备极限了
lr=0.0001 #初始学习率
epochs=10000 #总的迭代次数
train_loss_list=[] #用来画图的
val_loss_list=[]
acc_list=[]
train_ds = TensorDataset(x_train, y_train) #下面全是数据封装，我觉得可以弄成一个
val_ds = TensorDataset(x_val, y_val)
train_dl, valid_dl = get_data(train_ds, val_ds, bs)
train_dl = WrappedDataLoader(train_dl, preprocess)
valid_dl = WrappedDataLoader(valid_dl, preprocess)
loss_func = F.cross_entropy #loss计算方式
#model, opt = get_model()
model=Mynet() #我的网络结构
model.to(dev) #用到GPU上
#model=simpleNet(128,300,30,2)
opt = optim.SGD(model.parameters(), lr=lr, momentum=0.9) #默认的反向传递方式
scheduler=optim.lr_scheduler.ReduceLROnPlateau(opt, mode='min', factor=0.1, patience=50, verbose=True, threshold=0.0001, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08) #学习率优化
fit(epochs, model, loss_func, opt, train_dl, valid_dl) #开始跑了
以上代码大部分是从pytorch官网修改过来，比很多其他人写的精炼很多，可移植性很高。
扫一扫
6万+
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
