机器学习自用函数整理

查看模型参数

for parameters in net.parameters():
    print(parameters)

for name,parameters in net.named_parameters():
    print(name,':',parameters.size())

加载模型

#方法一：
torch.save(model.state_dict(), model_name) #保存模型，但只是把训练产生的模型参数存入一个字典

#方法二：
torch.save(model, model_name) #保存模型，提供要保存的模型和模型的名字就可以将其整个保存

#载入模型（方法二）
mymodel_path = "G:\\src\\models\\test"
model = torch.load(mymodel_path)

#载入模型（方法一）
mymodel_path = "G:\\src\\models\\test"
m_state_dict = torch.load(mymodel_path)
model = BiLSTM_CRF(                           #仅更新模型的参数
    vocab_size=len(word_to_id),
    tag_to_ix=tag_to_id,
    embedding_dim=parameters["word_dim"],
    hidden_dim=parameters["word_lstm_dim"],
    use_gpu=use_gpu,
    char_to_ix=char_to_id,
    pre_word_embeds=word_embeds,
    use_crf=parameters["crf"],
    char_mode=parameters["char_mode"],
)
model.load_state_dict(m_state_dict)

转化问题

#list 转 numpy
ndarray = np.array(list)


#numpy 转 list
list = ndarray.tolist()

#list 转 torch.Tensor
tensor=torch.Tensor(list)

#torch.Tensor 转 list
先转numpy，后转list
list = tensor.numpy().tolist()


#torch.Tensor 转 numpy
ndarray = tensor.numpy()
*gpu上的tensor不能直接转为numpy
ndarray = tensor.cpu().numpy()


#numpy 转 torch.Tensor
tensor = torch.from_numpy(ndarray) 

使用自己的数据集

利用dataset和DataLOader函数

1. 无标签数据

import torch
import torch.nn as nn
import os
import numpy as np
import torchvision
from torch.utils.data import DataLoader
from torchvision.utils import save_image
from torch.utils import data
from torchvision import datasets
import torchvision.transforms as transforms
from PIL import Image

#图片路径/文本路径（仅包含图片名称）
img_path="./face/rem_fakes"
txt_path="./face/1.txt"
#重写Dataset类
class imgDataset(data.Dataset):
    def __init__(self, img_path, txt_path ,transform=None ):
        with open(txt_path,'r') as f:
            lines = f.readlines()
            self.img_list = [os.path.join(img_path, i.split()[0]) for i in lines]
            self.transform = transform
    def __getitem__(self,index): #遍历0~len(self.img_list)个图片
        img_path = self.img_list[index]
        img = Image.open(img_path).convert('RGB')#转成Image格式
        if self.transform is not None:
            img = self.transform(img)
        return img
    def __len__(self):
        return len(self.img_list)
#转化图片类型        
mytransform = transforms.Compose([
    transforms.Resize((28,28)),
    transforms.ToTensor()
])
#加载数据
dataloader = DataLoader(
    imgDataset(img_path, txt_path ,transform = mytransform),
    batch_size = 32,
    shuffle = True
)
#查看数据
for i,img in enumerate(dataloader):
    save_image(img,"./face/rem_fakes/a1.png")
    break

2. 有标签数据

import torch
import torch.nn as nn
import os
import numpy as np
import torchvision
from torch.utils.data import DataLoader
from torchvision.utils import save_image
from torch.utils import data
from torchvision import datasets
import torchvision.transforms as transforms
from PIL import Image

#图片路径/文本路径（包含图片名称和label）
img_path="./test_dataset/train"
txt_path="./test_dataset/train.txt"
#重写Dataset类
class imgDataset(data.Dataset):
    def __init__(self, img_path, txt_path ,transform=None ):
        with open(txt_path,'r') as f:
            lines = f.readlines()
            self.img_list = [os.path.join(img_path, i.split()[0]) for i in lines]
            self.label_list = [i.split()[1] for i in lines]
            self.transform = transform
    def __getitem__(self,index): #遍历0~len(self.img_list)个图片
        img_path = self.img_list[index]
        label = self.label_list[index]
        img = Image.open(img_path).convert('RGB')#转成Image格式
        if self.transform is not None:
            img = self.transform(img)
        return img,label
    def __len__(self):
        return len(self.img_list)
#转化图片类型        
mytransform = transforms.Compose([
    transforms.Resize((28,28)),
    transforms.ToTensor()
])
#加载数据
dataloader = DataLoader(
    imgDataset(img_path, txt_path ,transform = mytransform),
    batch_size = 2,
    shuffle = True
)
#查看数据
for i,(img,label) in enumerate(dataloader):
    print(label)
    save_image(img,"./test_dataset/a1.png")
    break

模型设置参数

import argparse


parser = argparse.ArgumentParser()
parser.add_argument("--n_epochs", type=int, default=200, help="number of epochs of training")
parser.add_argument("--batch_size", type=int, default=64, help="size of the batches")
parser.add_argument("--lr", type=float, default=0.0002, help="adam: learning rate")
parser.add_argument("--b1", type=float, default=0.5, help="adam: decay of first order momentum of gradient")
parser.add_argument("--b2", type=float, default=0.999, help="adam: decay of first order momentum of gradient")
parser.add_argument("--n_cpu", type=int, default=8, help="number of cpu threads to use during batch generation")
parser.add_argument("--latent_dim", type=int, default=100, help="dimensionality of the latent space")
parser.add_argument("--img_size", type=int, default=28, help="size of each image dimension")
parser.add_argument("--channels", type=int, default=1, help="number of image channels")
parser.add_argument("--sample_interval", type=int, default=400, help="interval betwen image samples")
opt = parser.parse_args([])
print(opt)

使用参数时直接opt.***    （opt.img_size）

数据集划分

整条的数据集划分

import random
def randSplit(dataSet,rate):
    l = list(dataSet.index)
    random.shuffle(l)
    dataSet.index = l
    n = dataSet.shape[0]
    m = int(n*rate)
    train = dataSet.loc[range(m),:]
    test = dataSet.loc[range(m,n),:]
    dataSet.index = range(dataSet.shape[0])
    test.index = range(test.shape[0])
    return train, test

特征值和目标值分离的数据集划分

#划分数据集和测试集 ，划分数据集随机划分数据
from sklearn.model_selection import train_test_split
Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.25)

数据按batch_size分组

注意 这里DataLoader里的数据类型一定是tensor

import torch as nn
import numpy as np
import torch.utils.data as Data
import pandas as pd

train_set = pd.read_csv('ratings.csv',encoding='utf-8')
train_set = np.array(train_set)#将DataFrame转成array
mytensor = nn.tensor(train_set)#将array转换tensor

loader = Data.DataLoader(dataset = train_set,batch_size = 10,shuffle = False)#batch_size分组
#第一种遍历
for i,data in enumerate(loader):#遍历
    print(data)
# 第二种遍历
for b in iter(dl_train):
    print(b[0], b[1])
    break

DataLoader参数:

读取数据

Pandas读取文件（read_csv与read_table 的区别）

数据清洗

data.info()#数据信息
data.isnull().sum()#查看缺失值
data.drop(['Name','Cabin','Ticket'],inplace=True,axis=1)#删除缺失值多，目测对建模没有用的列
data = data.dropna()#把Nan去除
data["Age"] = data["Age"].fillna(data["Age"].mean())#Age列的NaN填充Age平均值
data= data.fillna(method='ffill')#在列方向上以前一个值作为值赋给NaN
data["Sex"] = (data["Sex"]=='male').astype('int')#将二分类变量转换为数值型变量
#将三分类变量转换为数值型变量
labels = data["Embarked"].unique().tolist()
data["Embarked"] = data["Embarked"].apply(lambda x: labels.index(x))

数据截取

pandas中iloc和loc的区别和用法：

1. loc函数：通过行索引 “Index” 中的具体值来取行数据（如取"Index"为"A"的行）

data.loc['b','B']#获取第二行、第二列对应的值

data.loc['b':'c','B':'C']#获取从第二行开始。第二列开始的右下角区域的值

2. iloc函数：通过行号来取行数据（如取第二行的数据）

data.iloc[1,1]#获取第二行、第二列的值
 
data.iloc[1:3,1:3]#获取从第二行，第二列开始对应的右下角的区域

随机种子

numpy版本

同一堆种子每次运行得到的随机数都一样

np.random.seed(1)#第一堆种子
a= np.random.randn(5)
np.random.seed(1)#第一堆种子
b= np.random.randn(5)
print(a)
print(b)

torch版本

torch.manual_seed(10)
print(torch.rand(6))
torch.manual_seed(10)
print(torch.rand(6))
torch.manual_seed(10)
print(torch.rand(6))

稀疏矩阵

indptr = np.array([0, 2, 3, 6])
indices = np.array([0, 2, 2, 0, 1, 2])
data = np.array([1, 2, 3, 4, 5, 6])
csr_matrix((data, indices, indptr), shape=(3, 3)).toarray()

array([[1, 0, 2],
       [0, 0, 3],
       [4, 5, 6]])

快速读取文件中图片名字

快速读取文件中图片名字

训练进度API

from tqdm import tqdm

dataloader = torch.utils.data.DataLoader...
...
for epoch in range(opt.n_epochs):
    loop = tqdm(enumerate(dataloader), total =len(dataloader))
    for i, (imgs, _) in loop:
        ...
       loop.set_description(f'Epoch [{epoch}/{opt.n_epochs}]')
       loop.set_postfix(d_loss= d_loss.item(),g_loss= g_loss.item())