分类鱼 VGG网络 python学习 深度学习 paddlepaddle学习

用于学习VGG网络，代码转载说明：分类鱼 - 飞桨AI Studio - 人工智能学习实训社区 (baidu.com)

下面是用自己准备的数据集进行训练&验证的。

更新：验证模型部分的代码修改了一下，实现了多张图片验证，输出鱼类型为中文

数据准备：

 

首先是data文件夹里，data13981里放着预测的图片，data14492里放数据集的压缩包，压缩包里一种鱼一个文件夹，文件夹名为鱼的名字。

然后是和data同一目录的，代码运行后会解压数据集，生成文件夹fish_image，如果生成的是fish_image23，把名字里的23删掉就可以了。

其他数据不需要修改。

1.训练

①导入各种模块

import os
import zipfile
import random
import json
import paddle
import sys
import numpy as np
from PIL import Image
import paddle.fluid as fluid

import matplotlib.pyplot as plt
from paddle.fluid.dygraph import Linear,Conv2D,Pool2D

②解压数据集

def unzip_data(src_path,target_path):
#     '''
#     解压原始数据集，将src_path路径下的zip包解压至data目录下
#     '''
     if(not os.path.isdir(target_path + "FishDetect")):     
         z = zipfile.ZipFile(src_path, 'r')
         z.extractall(path=target_path)
         z.close()

unzip_data('data/data14492/Trout.zip','/home/aistudio/fish_image')

③标签

dit_class = {
'翻车鱼':1,
'蝶鱼':2,
'鳕鱼':3,
'比目鱼':4,
'飞鱼':5,
'金头鲷':6,
'带鱼':7,
'金枪鱼':8,
'大黄鱼':9,
'魔鬼鱼':10,
'红鲻':11,
'红鲷':12,
'黑鲈':13,
'小虾':14,
'条纹红鲻':15,
'鳟鱼':16,
}

④数据操作

#每次生成数据列表前，首先清空train.txt和eval.txt

def get_data_list(path,eval_list_path,train_list_path,dit_class):
    train_list = []
    eval_list = []
    class_num = 0
    all_img_num =0
    dirs_data = os.listdir(path)
    for dir_data in dirs_data:
        class_num += 1
        dirs_class =  path+'//'+ dir_data
        dirs_name = os.listdir(dirs_class)
        for dir_name in dirs_name:
            img_path = dirs_class + '//' +dir_name
            lab = dit_class[dir_data]
            if all_img_num % 9 == 0:
                eval_list.append(img_path + '\t' + str(lab) + '\n')
            else:
                train_list.append(img_path + '\t' + str(lab) + '\n')
            all_img_num += 1
    # print(class_num)
    # print(all_img_num)

    
    random.shuffle(eval_list)
    random.shuffle(train_list)             #打乱数据集
    with open(eval_list_path,'a') as f:
        for eval_img in eval_list:
            f.write(eval_img)

    with open(train_list_path,'a') as f:
        for train_img in train_list:
            f.write(train_img)

def data_reader(file_list):
    '''
    自定义reader
    '''
    def reader():
        with open(file_list, 'r') as f:
            lines = f.readlines()
            for line in lines:
                img_path, lab = line.strip().split('\t')
                img = Image.open(img_path) 
                if img.mode != 'RGB': 
                    img = img.convert('RGB') 
                img = img.resize((64,64), Image.BILINEAR)
                img = np.array(img).astype('float32') 
                img = img.transpose((2, 0, 1))  # HWC to CHW 
                img = img/255.0                # 像素值归一化 
                yield img, int(lab) 
    return reader

train_list_path = "train_list.txt"
eval_list_path = "eval_list.txt"
#每次生成数据列表前，首先清空train.txt和eval.txt
with open(train_list_path, 'w') as f: 
    f.seek(0)
    f.truncate() 
with open("eval_list.txt", 'w') as f: 
    f.seek(0)
    f.truncate() 
get_data_list('fish_image',eval_list_path,train_list_path,dit_class)
train_reader = paddle.batch(data_reader(train_list_path),batch_size=64,drop_last=True)
eval_reader = paddle.batch(data_reader(eval_list_path),batch_size=64,drop_last=True)

Batch = 0
Batchs = []
all_train_loss = []
all_train_accs = []
#可视化损失值和准确率
def draw_train_loss(Batchs,train_loss):
    plt.title('loss', fontsize = 24)
    plt.xlabel('batch',  fontsize = 24)
    plt.ylabel('loss',  fontsize = 24)
    plt.plot(Batchs,train_loss,color='green',label='loss')
    plt.legend()
    plt.grid()
    plt.show()

def draw_train_accs(Batchs,train_accs):
    plt.title('accs', fontsize = 24)
    plt.xlabel('batch', fontsize = 14)
    plt.ylabel('acc', fontsize = 14)
    plt.plot(Batchs,train_accs,color = 'red',label='accs')
    plt.legend()
    plt.grid()
    plt.show()

⑤模型建立

#模型建立
class VggNet(fluid.dygraph.Layer):
    def __init__(self):
        super(VggNet,self).__init__()

        self.hidden1_1 = Conv2D(3,64,3,1,1,act='relu')   #padding = 1,  64 -> 68
        self.hidden1_2 = Conv2D(64,64,3,1,1,act='relu')
        self.hidden1_3 = Pool2D(pool_size=2,pool_type='max',pool_stride=2)    # 32

        self.hidden2_1 = Conv2D(64,128,3,1,1,act='relu')
        self.hidden2_2 = Conv2D(128,128,3,1,1,act='relu')
        self.hidden2_3 = Pool2D(pool_size=2,pool_type='max',pool_stride=2)    # 16

        self.hidden3_1 = Conv2D(128,256,3,1,1,act='relu')
        self.hidden3_2 = Conv2D(256,256,3,1,1,act='relu')
        self.hidden3_3 = Conv2D(256,256,3,1,1,act='relu')
        self.hidden3_4 = Pool2D(pool_size=2,pool_type='max',pool_stride=2)    # 8 

        # self.hidden4_1 = Conv2D(256,512,3,1,1,act='relu')
        # self.hidden4_2 = Conv2D(512,512,3,1,1,act='relu')
        # self.hidden4_3 = Conv2D(512,512,3,1,1,act='relu')
        # self.hidden4_4 = Pool2D(pool_size=2,pool_type='max',pool_stride=2)

        # self.hidden5_1 = Conv2D(512,512,3,1,1,act='relu')
        # self.hidden5_2 = Conv2D(512,512,3,1,1,act='relu')
        # self.hidden5_3 = Conv2D(512,512,3,1,1,act='relu')
        # self.hidden5_4 = Pool2D(pool_size=2,pool_type='max',pool_stride=2)

        
        self.hidden6 = Linear(8*8*256,2048,act='relu')
        self.hidden7 = Linear(2048,2048,act='relu')
        self.hidden8 = Linear(2048,24,act='softmax')

    def forward(self,inputs):
        x = self.hidden1_1(inputs)
        x = self.hidden1_2(x)
        x = self.hidden1_3(x)

        x = self.hidden2_1(x)
        x = self.hidden2_2(x)
        x = self.hidden2_3(x)

        x = self.hidden3_1(x)
        x = self.hidden3_2(x)
        x = self.hidden3_3(x)
        x = self.hidden3_4(x)

        # x = self.hidden4_1(x)
        # x = self.hidden4_2(x)
        # x = self.hidden4_3(x)
        # x = self.hidden4_4(x)

        # x = self.hidden5_1(x)
        # x = self.hidden5_2(x)
        # x = self.hidden5_3(x)
        # x = self.hidden5_4(x)

        y = fluid.layers.reshape(x,shape=[-1,8*8*256])
        y = self.hidden6(y)
        y = self.hidden7(y)
        y = self.hidden8(y)

        return y

⑥模型训练

#模型训练
with fluid.dygraph.guard():
# with fluid.dygraph.guard(place=fluid.CUDAPlace(0)):
    vgg = VggNet()
    opt = fluid.optimizer.AdamOptimizer(learning_rate=0.001,parameter_list=vgg.parameters())
    epochs = 10
    vgg.train()  #训练模式

    for pass_num in range(epochs):
        for batch_id,data in enumerate(train_reader()):
            images = np.array([x[0] for x in data], np.float32)
            labels = np.array([x[1] for x in data]).astype('int64')
            labels = labels[:,np.newaxis]

            image = fluid.dygraph.to_variable(images)
            label = fluid.dygraph.to_variable(labels)
            
            # print(image.shape)
            # print(label.shape)

            predict = vgg(image)

            loss = fluid.layers.cross_entropy(predict,label)
            avg_loss = fluid.layers.mean(loss)
            acc = fluid.layers.accuracy(predict,label)

            if batch_id != 0 and batch_id %50 == 0 :
                Batch += 50
                Batchs.append(Batch)
                all_train_accs.append(acc.numpy()[0])
                all_train_loss.append(avg_loss.numpy()[0])
                print('epochs:{},batch_id:{},train_loss:{},train_acc{}'.format(pass_num,batch_id,avg_loss.numpy(),acc.numpy()))

            avg_loss.backward()
            opt.minimize(avg_loss)
            vgg.clear_gradients()
    fluid.save_dygraph(vgg.state_dict(),'vgg')


#draw_train_accs(Batchs,all_train_accs)
#draw_train_loss(Batchs,all_train_loss)

⑦模型校验

#模型校验
with fluid.dygraph.guard():
    accs = []
    model = VggNet()
    model_dict,_ = fluid.load_dygraph('vgg')
    model.load_dict(model_dict)
    model.eval()

    for batch_id ,data in enumerate(eval_reader()):
        images = np.array([x[0] for x in data]).astype('float32')
        labels = np.array([x[1] for x in data]).astype('int64')
        labels = labels[:,np.newaxis]

        image = fluid.dygraph.to_variable(images)
        label = fluid.dygraph.to_variable(labels)

        predict = model(image)
        acc = fluid.layers.accuracy(predict,label)
        accs.append(acc.numpy()[0])
        avg_acc = np.mean(accs)
    print(avg_acc)

2.模型验证

为了能够快速预测，我把训练模型下载下来了，然后单独写一份验证的代码进行验证，就不用每次都训练那么久再验证。

数据准备：

将代码的python文件和pdparams文件放在同一目录下，建立文件夹test_image，里面放一张需要预测的图片，预测图片没有命名和格式要求。

①训练模型

将上面的训练模型下载下来。

② 前期准备

先导入包，不需要和上面训练模型那里一样导入的那么多。

import paddle
import numpy as np
from PIL import Image
import paddle.fluid as fluid
from paddle.fluid.dygraph import Linear,Conv2D,Pool2D
import matplotlib.pyplot as plt

然后把标签也导入进去，实际上就是复制上面的③。

再将模型的建立也导入，即复制上面的⑤。

③模型验证

#模型验证
def load_image(img_path):
    img = Image.open(img_path)
    if img.mode != 'RGB':
        img = img.convert('RGB')
    img = img.resize((64,64),Image.BILINEAR)
    img = np.array(img).astype('float32')
    img = img.transpose((2,0,1))
    img = img/255.0
    return img

with fluid.dygraph.guard():
    model = VggNet()
    model_dict, _ = fluid.load_dygraph('vgg')
    model.load_dict( model_dict)
    model.eval()

    i=0
    file_num=len(os.listdir('test_image'))
    print('共需要预测的图片数量为：',file_num)
    for file in os.listdir('test_image'):
        file_name=os.listdir('test_image')[i]
        infer_path = 'test_image/'+file_name
        img = Image.open(infer_path)
        plt.imshow(img)
        plt.show
    
        #对该图片就行预测
        infer_img = load_image(infer_path)
        # infer_img = np.array(infer_img)
        infer_img = infer_img[np.newaxis,:]
        infer_img = fluid.dygraph.to_variable(infer_img)
        result = model(infer_img)
        label = np.argmax(result.numpy())
    
        # new_dict = {v : k for k, v dit_class.items()}
        for k,v in dit_class.items():
            if v== label:
                lab = k
    
        print(file_name,'预测类型为：',lab)
        i=i+1