【Pytorch】csv 2 torch tensor

sdbhewfoqi

已于 2023-06-11 12:46:47 修改

阅读量301

点赞数 1

分类专栏： Tensorflow & Pytorch 文章标签： python 机器学习

于 2023-03-20 23:51:48 首次发布

本文链接：https://blog.csdn.net/weixin_31866177/article/details/129679323

版权

Tensorflow & Pytorch 专栏收录该内容

41 篇文章 7 订阅

订阅专栏

关于如何改写自己的dataset和dataloader，涉及的dataset/loader的原理，见

Pytorch（三）：Dataset和Dataloader的理解

如何从excel,csv文件中读取数据，转成pytorch tensor用于机器学习模型训练。

就是从csv中写成dataset/loader要求的格式。

"""
如何从excel,csv文件中读取数据，转成pytorch tensor用于机器学习模型训练
# https://www.bilibili.com/video/BV1nR4y1N7Vj/?spm_id_from=333.880.my_history.page.click&vd_source=dc27cbd311ff3ca278726381630036f0
【学习总结】
1、class的__init__()是可以自定义入参的
2、
"""
# pip install openpyxl
import torch
from torch.utils.data import Dataset
from torch.utils.data import DataLoader
import numpy as np
import pandas as pd

"""
csv中内容
id  feat1   feat2   label
1   0.796   0.7831   9
2   0.2183  0.2791   5
"""

class ExcelDataset(Dataset):
    # 继承自Dataset父类
    def __init__(self, filepath="data.xlsx", sheet_name=0):
        # excel中第一个表单 sheet_name=0
        print(f"reading {filepath}, sheet={sheet_name}")

        df = pd.read_excel(
            filepath, header=0, index_col=0,
            names=['feat1', 'feat2', 'label'],
            sheet_name=sheet_name,
            dtype={"feat1":np.float32, "feat1":np.float32, "feat1":np.int32}
        ) # dtype在读取的时候确定数据类型
        # 如果同时读取两个表单，那么df是个字典，需要才能读取数据
        # df=df['corpus1']
        print(f"the shape of dataframe is {df.shape}")

        feat = df.iloc[:,:2].values # 这样可以从dataframe转为np.array
        label = df.iloc[:,2].values

        self.x = torch.from_numpy(feat) # 转为torch.tensor
        self.y = torch.from_numpy(label)

    def __len__(self):
        return len(self.y)

    def __getitem__(self, index):
        return self.x[index], self.y[index]  # 传一个元组出去

class CsvDataset(Dataset):
    def __init__(self, filepath="data.csv"):
        # there is no sheet name definition in csv format file
        print(f"reading {filepath}")

        df = pd.read_csv(
            filepath, header=0, index_col=0,
            encoding='utf-8',
            names=['feat1', 'feat2', 'label'],
            dtype={"feat1": np.float32, "feat1": np.float32, "feat1": np.int32},
            skip_blank_lines=True,
        )
        # header=0 表头不要，标题不要
        # index_col=0 第一列，id自增列不要
        print(f"the shape if dataframe is {df.shape}")
        feat = df.iloc[:,:2].values # 这样可以从dataframe转为np.array
        label = df.iloc[:,2].values

        self.x = torch.from_numpy(feat) # 转为torch.tensor
        self.y = torch.from_numpy(label)

    def __len__(self):
        return len(self.y)

    def __getitem__(self, index):
        return self.x[index], self.y[index]  # 传一个元组出去


# 不用pandas，直接将csv当做文本文件进行读取
class Csv2Dataset(Dataset):
    def __init__(self, filepath="data.csv"):
        print(f"reading {filepath}")

        with open(filepath, encoding='utf-8') as f:
            lines = f.readlines()
        feat = []
        label = []
        for line in lines[1:]:
            values = line.strip().split(',')
            row_feat = [float(v) for v in values[1:3]]
            row_label = int(values[3])
            feat.append(row_feat)
            label.append(row_label)

        feat = np.array(feat, dtype=np.float32) #### list转np.array，才能放到torch里
        label = np.array(label, dtype=np.float32)

        self.x = torch.from_numpy(feat) # 转为torch.tensor
        self.y = torch.from_numpy(label)

    def __len__(self):
        return len(self.y)

    def __getitem__(self, index):
        return self.x[index], self.y[index]  # 传一个元组出去


if __name__ == '__main__':
    print("test for exceldataset")
    # 1.实例化一个excel_dataset对象
    excel_dataset = ExcelDataset(sheet_name="corpus1") # 自己写的torch dataset，一个类
    # excel_dataset = ExcelDataset(sheet_name="corpus2")
    # excel_dataset = ExcelDataset(sheet_name=None) # 同时读取两个sheet
    # DataLoader是一个生成器，把对象传入DataLoader中
    excel_dataloader = DataLoader(excel_dataset, batch_size=8, shuffle=True)
    # 使用for循环去取它
    for idx, (batch_x, batch_y) in enumerate(excel_dataset):
        print(f"batch_id:{idx}, {batch_x.shape}, {batch_y.shape}")
        # 9, torch. size([8,2]), torch.size([8])
        print(batch_x, batch_y)
        """ 以下是伪代码
        output = model(batch_x)
        loss = criterion(output, batch_y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        """

    print("test for csvdataset")
    csv_dataset = CsvDataset()
    csv_dataloader = DataLoader(csv_dataset, batch_size=8, shuffle=True)
    for idx, (batch_x, batch_y) in enumerate(csv_dataloader):
        print(f"batch_id:{idx}, {batch_x.shape}, {batch_y.shape}")
        # 9, torch. size([8,2]), torch.size([8])
        print(batch_x, batch_y)

    print("test for csv2dataset")
    csv2_dataset = Csv2Dataset()
    csv2_dataloader = DataLoader(csv2_dataset, batch_size=8, shuffle=True)
    for idx, (batch_x, batch_y) in enumerate(csv2_dataloader):
        print(f"batch_id:{idx}, {batch_x.shape}, {batch_y.shape}")
        # 9, torch. size([8,2]), torch.size([8])
        print(batch_x, batch_y)