利用pytorch的载入训练npy类型数据代码

 utils

import os
import sys
import json
import pickle
import random
import imageio
import numpy as np
import torch
from tqdm import tqdm


def train_one_epoch(model, optimizer, data_loader, device, epoch):
    model.train()
    loss_function = torch.nn.CrossEntropyLoss()
    # from loss_function import FocalLoss1
    # loss_function = FocalLoss1().cuda()
    accu_loss = torch.zeros(1).to(device)  # 累计损失
    accu_num = torch.zeros(1).to(device)   # 累计预测正确的样本数
    optimizer.zero_grad()
    k = 0
    sample_num = 0
    data_loader = tqdm(data_loader)
    for step, data in enumerate(data_loader):
        k = k+1
        images, labels = data
        sample_num += images.shape[0]
        # print(images[0].shape)
        # c = np.array(images[0])
        # c = c.transpose((1, 2, 0))
        # imageio.imwrite('data' + '%d.jpg' % (k + 1), c)


        pred = model(images.to(device))
        # print(images.shape)
        pred_classes = torch.max(pred, dim=1)[1]
        accu_num += torch.eq(pred_classes, labels.to(device)).sum()

        loss = loss_function(pred, labels.to(device))
        loss.backward()
        accu_loss += loss.detach()

        data_loader.desc = "[train epoch {}] loss: {:.3f}, acc: {:.3f}".format(epoch,
                                                                               accu_loss.item() / (step + 1),
                                                                               accu_num.item() / sample_num)

        if not torch.isfinite(loss):
            print('WARNING: non-finite loss, ending training ', loss)
            sys.exit(1)

        optimizer.step()
        optimizer.zero_grad()

    return accu_loss.item() / (step + 1), accu_num.item() / sample_num

@torch.no_grad()
def evaluate(model, data_loader, device, epoch):
    # from loss_function import FocalLoss1
    # loss_function = FocalLoss1().cuda()
    loss_function = torch.nn.CrossEntropyLoss()
    model.eval()

    accu_num = torch.zeros(1).to(device)   # 累计预测正确的样本数
    accu_loss = torch.zeros(1).to(device)  # 累计损失

    sample_num = 0
    data_loader = tqdm(data_loader)
    for step, data in enumerate(data_loader):
        images, labels = data
        sample_num += images.shape[0]

        pred = model(images.to(device))
        pred_classes = torch.max(pred, dim=1)[1]
        accu_num += torch.eq(pred_classes, labels.to(device)).sum()

        loss = loss_function(pred, labels.to(device))
        accu_loss += loss

        data_loader.desc = "[valid epoch {}] loss: {:.3f}, acc: {:.3f}".format(epoch,
                                                                               accu_loss.item() / (step + 1),
                                                                               accu_num.item() / sample_num)

    return accu_loss.item() / (step + 1), accu_num.item() / sample_num

TRAIN

import os
import math
import argparse
from torch.utils.data import DataLoader
import torch
import torch.optim as optim
from torch.utils.tensorboard import SummaryWriter
from torchvision import transforms
import torch.optim.lr_scheduler as lr_scheduler
import matplotlib.pyplot as plt
from densenet_model import densenet121, load_state_dict,densenet201
# from model import densenet121
# from my_dataset import MyDataSet
import monai
# from utils import read_split_data, train_one_epoch, evaluate
from utils import train_one_epoch, evaluate


def main(args):
    device = torch.device(args.device if torch.cuda.is_available() else "cpu")

    print(args)
    # print('Start Tensorboard with "tensorboard --logdir=runs", view at http://localhost:6006/')
    tb_writer = SummaryWriter()
    if os.path.exists("./weights") is False:
        os.makedirs("./weights")
    #############################################################################################################################################################
    # train_images_path, train_images_label, val_images_path, val_images_label = read_split_data(args.data_path)
    batch_size = args.batch_size
    # nw = min([os.cpu_count(), batch_size if batch_size > 1 else 0, 8])  # number of workers
    # print('Using {} dataloader workers every process'.format(nw))

    #############################################################################################################################################################
    data_transform = {
        "train": transforms.Compose([
            # transforms.CenterCrop(224),
            # transforms.RandomGrayscale(0.1),
            transforms.RandomHorizontalFlip(),
            transforms.RandomVerticalFlip(),
            transforms.RandomRotation(degrees=(0, 180)),
            # transforms.ToTensor(),
            transforms.Normalize(mean=[0.5], std=[0.5]),
        ]),
        "val": transforms.Compose([
            # transforms.CenterCrop(224),
            # transforms.RandomGrayscale(0.1),
            # transforms.RandomHorizontalFlip(),
            # transforms.RandomVerticalFlip(),
            # transforms.RandomRotation(degrees=(0, 180)),
            # transforms.ToTensor(),
            transforms.Normalize(mean=[0.5], std=[0.5]),
    ])}

    #############################################################################################################################################################


    from dataset import MyDataSet,valDataset

    val_data_path = args.val_data_path
    data_path = args.data_path

    train_dataset =MyDataSet(
        data=data_path,
        label=data_path,
        transform=data_transform["train"])
    #
    val_dataset = valDataset(
        data=val_data_path,
        label=val_data_path,
        transform=data_transform["val"])


    #############################################################################################################################################################

    #############################################################################################################################################################
    train_loader = DataLoader(train_dataset,
                                               batch_size=batch_size,
                                               shuffle=True,
                                               # num_workers=8,
                                               pin_memory=True,
                                               drop_last = True,
                                               # collate_fn=train_dataset.collate_fn
                                               )


    val_loader = DataLoader(val_dataset,
                                             batch_size=1,
                                             shuffle=False,
                                             # num_workers=8,
                                             pin_memory=True,
                                             # drop_last=True,
                                             # ,collate_fn=train_dataset.collate_fn
                                             )
    #############################################################################################################################################################

    #############################################################################################################################################################



    # 如果存在预训练权重则载入
    model = densenet121(num_classes=args.num_classes).to(device)
    # model = densenet201(num_classes=args.num_classes,drop_rate = 0.7).to(device)
    # from vit_model import vit_large_patch32_224_in21k as create_model
    # model = create_model(num_classes=2, has_logits=False).to(device)

    # model = monai.networks.nets.resnet34(
    #     spatial_dims=2,
    #     # in_channels=3,
    #     # block = 'SEBottleneck',
    #     # layers = (1, 1, 1, 1),
    #     num_classes = 2,
    #     # groups=32,
    #     # reduction=16,
    #     # pretrained = True,
    #     # progress = True,
    #     # stride = 1,
    #     # dropout_dim = 3,
    #     # dropout_prob=None
    # )

    model.cuda()
    # *****************************************载预权重*************************************************************************
    if args.weights != "":
        if os.path.exists(args.weights):
            # load_state_dict(model, args.weights)
            model.load_state_dict(torch.load(args.weights, map_location=device))
        else:
            raise FileNotFoundError("not found weights file: {}".format(args.weights))


        # print(model.load_state_dict(args.weights, strict=False))
        # checkpoint = torch.load(args.weights)
        # model.load_state_dict(checkpoint['model'], strict=False)
        # model.load_state_dict(checkpoint['state_dict'], strict=False)
        # model.load_state_dict(checkpoint, strict=False)

        model.to(device)

    if args.freeze_layers:
        for name, para in model.named_parameters():
            # 除最后的全连接层外，其他权重全部冻结
            if "classifier" not in name:
                para.requires_grad_(False)

    # ********************************拼接训练****************************# ************************************************************

    #############################################################################################################################################################
    pg = [p for p in model.parameters() if p.requires_grad]
    # optimizer = optim.SGD(pg, lr=args.lr, momentum=0.9, weight_decay=1E-4, nesterov=True)
    optimizer = torch.optim.Adam(pg, lr=0.0001, weight_decay=0.000)  # L2正则
    # Scheduler https://arxiv.org/pdf/1812.01187.pdf
    lf = lambda x: ((1 + math.cos(x * math.pi / args.epochs)) / 2) * (1 - args.lrf) + args.lrf  # cosine
    scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lf)
    # scheduler = CosineAnnealingLR(optimizer, T_max=32)
    #############################################################################################################################################################
    x_axis = []
    t_loss = []
    v_loss = []
    t_acc = []
    v_acc = []
    j = 0
    for epoch in range(1, args.epochs):
        # train
        j += 1
        train_loss, train_acc = train_one_epoch(model=model,
                                                optimizer=optimizer,
                                                data_loader=train_loader,
                                                device=device,
                                                epoch=epoch)

        scheduler.step()


        val_loss, val_acc = evaluate(model=model,
                                     data_loader=val_loader,
                                     device=device,
                                     epoch=epoch)


        #############################################################################################################################################################
        # ************************************************************# ************************************************************
        x_axis.append(j)
        t_loss.append(train_loss)
        v_loss.append(val_loss)
        t_acc.append(train_acc)
        v_acc.append(val_acc)
        try:
            train_loss_lines.remove(train_loss_lines[0])  # 移除上一步曲线
            val_loss_lines.remove(val_loss_lines[0])


        except Exception:
            pass
        plt.figure(1)
        train_loss_lines = plt.plot(x_axis, t_loss, 'r', lw=0.5)  # lw为曲线宽度
        val_loss_lines = plt.plot(x_axis, v_loss, 'b', lw=0.5)
        plt.title("loss")
        plt.xlabel("steps")
        plt.ylabel("loss")
        plt.legend(["train_loss", "val_loss"])
        # plt.pause(0.1)  # 图片停留0.1s
        if epoch % 10 == 0:
            plt.savefig('loss' + '%d.jpg' % j)
            plt.figure(2)
            train_loss_lines = plt.plot(x_axis, t_acc, 'r', lw=0.5)  # lw为曲线宽度
            val_loss_lines = plt.plot(x_axis, v_acc, 'b', lw=0.5)
            plt.title("acc")
            plt.xlabel("steps")
            plt.ylabel("acc")
            plt.legend(["train_acc", "val_acc"])
            plt.savefig('acc' + '%d.jpg' % epoch)
            plt.cla()
        # ************************************************************# ************************************************************
        tags = ["train_loss", "train_acc", "val_loss", "val_acc", "learning_rate"]
        tb_writer.add_scalar(tags[0], train_loss, epoch)
        tb_writer.add_scalar(tags[1], train_acc, epoch)
        tb_writer.add_scalar(tags[2], val_loss, epoch)
        tb_writer.add_scalar(tags[3], val_acc, epoch)
        tb_writer.add_scalar(tags[4], optimizer.param_groups[0]["lr"], epoch)
        #############################################################################################################################################################
        if (epoch) % 200 == 0:
            torch.save(model.state_dict(), "./weights/model-{}.pth".format(epoch))


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--num_classes', type=int, default=2)
    parser.add_argument('--epochs', type=int, default=1000)
    parser.add_argument('--batch-size', type=int, default=64)
    parser.add_argument('--lr', type=float, default=0.001)
    parser.add_argument('--lrf', type=float, default=0.1)

    # 数据集所在根目录
    # http://download.tensorflow.org/example_images/flower_photos.tgz
    parser.add_argument('--data-path', type=str,
                        default=r"C:\Users\Admin\SIAT\model\caihaihua2d_NEW\zhong")
    parser.add_argument('--val-data-path', type=str,
                        default=r"C:\Users\Admin\SIAT\model\caihaihua2d_NEW\zhong")

    # densenet121 官方权重下载地址
    # https://download.pytorch.org/models/densenet121-a639ec97.pth
    parser.add_argument('--weights', type=str, default=r'',#C:\Users\Admin\SIAT\model\caihaihua2d_NEW\weights\model-800.pth
                        help='initial weights path')
    parser.add_argument('--freeze_layers', type=bool, default=False)
    parser.add_argument('--device', default='cuda:0', help='device id (i.e. 0 or 0,1 or cpu)')


    opt = parser.parse_args()

    main(opt)
#训练集验证集数据分布不一致，或者训练集过小，未包含验证集中所有情况

DATASET

from PIL import Image
import torch
from torch.utils.data import Dataset
import csv
import torch
import imageio
import tensorflow as tf
import os
from torchvision import transforms
from medpy.io import load
# from albumentations import *
import numpy as np
from PIL import Image

from torch import nn
import torch.nn.functional as F
import random
import torchio as tio
from torchio import AFFINE, DATA
import torchio
from torchio import ScalarImage, LabelMap, Subject, SubjectsDataset, Queue
from torchio.data import UniformSampler
from torchio.transforms import (
    RandomFlip,
    RandomAffine,
    RandomElasticDeformation,
    RandomNoise,
    RandomMotion,
    RandomBiasField,
    RescaleIntensity,
    Resample,
    ToCanonical,
    ZNormalization,
    CropOrPad,
RandomSpike,
RandomBlur,
RandomSwap,
    HistogramStandardization,
    OneOf,
    Clamp,
    Compose,
    RandomGhosting,
)
from pathlib import Path
from glob import glob
from os.path import dirname, join, basename, isfile
import sys


class MyDataSet(Dataset):
    """自定义数据集"""
    # def __init__(self, root,transform):
    def __init__(self, data,label, transform=True):
        self.data = data
        self.label = label
        self.images_path = torch.tensor(np.load(os.path.join(data,"train.npy")))
        self.images_class = torch.tensor(np.load(os.path.join(label,"label.npy")))
        self.transform = transform

    def __len__(self):
        return self.images_path.shape[0]  # 返回数据的总个数


    def __getitem__(self, index):
        img = self.images_path[index, :, :]  # 读取每一个npy的数据
        label = self.images_class[index]  # 读取每一个npy的数据
        img = np.expand_dims(img, axis=0)
        img = torch.Tensor(img)
        img = torch.cat([img, img, img], dim=0)
        ###############################################################################################################3

        ###############################################################################################################3
        label = label.type(torch.long)

        if self.transform is not None:
            img = self.transform(img)
        return img, label  # 返回数据还有标签

    # def collate_fn(index):
    #     # 官方实现的default_collate可以参考
    #     print(index)
    #     # https://github.com/pytorch/pytorch/blob/67b7e751e6b5931a9f45274653f4f653a4e6cdf6/torch/utils/data/_utils/collate.py
    #     images, labels = tuple(zip(*index))
    #     #
    #     images = torch.stack(images, dim=0)
    #     labels = torch.as_tensor(labels)
    #     return images, labels


class valDataset(Dataset):
    def __init__(self, data,label, transform=True):
        self.data = data
        self.label = label
        self.images_path = torch.tensor(np.load(os.path.join(data,"val.npy")))
        self.images_class = torch.tensor(np.load(os.path.join(label,"val_label.npy")))
        self.transform = transform

    def __len__(self):
        return self.images_path.shape[0] #返回数据的总个数

    def __getitem__(self, index):
        img = self.images_path[index, :, :]  # 读取每一个npy的数据
        label = self.images_class[index]  # 读取每一个npy的数据
        img = np.expand_dims(img, axis=0)
        img = torch.Tensor(img)
        img = torch.cat([img, img, img], dim=0)
        ###############################################################################################################3

        ###############################################################################################################3
        ###############################################################################################################3
        label = label.type(torch.long)

        if self.transform is not None:
            img = self.transform(img)
        return img, label  # 返回数据还有标签

    #
    # def collate_fn(batch):
    #     # 官方实现的default_collate可以参考
    #     # https://github.com/pytorch/pytorch/blob/67b7e751e6b5931a9f45274653f4f653a4e6cdf6/torch/utils/data/_utils/collate.py
    #     images, labels = tuple(zip(*batch))
    #
    #     images = torch.stack(images, dim=0)
    #     labels = torch.as_tensor(labels)
    #     return images, labels