【代码】

最新推荐文章于 2023-11-30 16:14:17 发布

Miss_zhuo_

最新推荐文章于 2023-11-30 16:14:17 发布

阅读量256

点赞数

分类专栏：项目

本文链接：https://blog.csdn.net/Mr_zhuo_/article/details/102502678

版权

项目专栏收录该内容

6 篇文章 0 订阅

订阅专栏

import os
import time
import torch
import argparse

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from torch import nn
from tqdm import tqdm
from torch.utils.data import DataLoader
from torch.optim import Adam, RMSprop

from data.data_utils import CustomDataset, split_train_test, IrisDataset, SeriesDataset
from algo.net import LSTMPolicyNet, FCPolicyNet, GRUPolicyNet

parser = argparse.ArgumentParser()

parser.add_argument("-p", "--path", default="./data/data/", help="The path where data stored")
parser.add_argument("-d", "--debug", action="store_true", help="Enter debug mode.")
parser.add_argument("-r", "--restore", action="store_true", help="whether to restore model or not")
parser.add_argument("-m", "--model", choices=["lstm", "fc", "gru", "seq2seq"], default="lstm", help="Choose policy network")
args = parser.parse_args()

DATA_PATH = args.path
DEBUG = args.debug
RESTORE = args.restore
MODEL = args.model

ON_GPU = torch.cuda.is_available()
DEVICE = torch.device("cuda:0" if ON_GPU else "cpu")
MODEL_PATH = "./model/"
SAVE_PATH = os.path.join(MODEL_PATH, MODEL + "_" + time.strftime("%m%d_%H%M%S", time.localtime()) + ".pkl")

BATCH_SIZE = 64 # 样本批处理参数
NUM_EPOCHS = 25 # 召回率
LEARNING_RATE = 1e-3 #

SEQ_LEN = 4


def choose_model(model_name, input_size):
    if model_name == "lstm":
        model = LSTMPolicyNet(input_size=input_size, output_size=1, hidden_size=32, seq_len=SEQ_LEN)
    elif model_name == "fc":
        model = FCPolicyNet()
    elif model_name == "gru":
        model = GRUPolicyNet(input_size=input_size, output_size=1, hidden_size=32, seq_len=SEQ_LEN)
    elif model_name == "seq2seq":
        model = None

    return model


def load_latest_model():
    models = list(filter(lambda x: MODEL in x, os.listdir(MODEL_PATH)))
    model_path = os.path.join(MODEL_PATH, models[-1])
    print("load model from {}".format(model_path))
    model = torch.load(model_path)

    return model


def train():
    # dataset = CustomDataset(DATA_PATH, "2018")
    dataset = SeriesDataset(DATA_PATH, "2018", SEQ_LEN)
    print(len(dataset))

    # train_set, val_set = split_train_test(dataset)
    train_set = dataset
    # val_set = SeriesDataset(DATA_PATH, "2018", SEQ_LEN)
    val_set = SeriesDataset(DATA_PATH, "2017", SEQ_LEN)

    train_loader = DataLoader(train_set, batch_size=BATCH_SIZE, shuffle=False)
    val_loader = DataLoader(val_set, batch_size=BATCH_SIZE, shuffle=False)


    if RESTORE:
        model = load_latest_model()
    else:
        model = choose_model(MODEL, dataset.feature_size)
    
    model.to(DEVICE)

    criterion = nn.MSELoss()
    # criterion = nn.CrossEntropyLoss()

    optimizer = Adam(model.parameters(), lr=LEARNING_RATE)

    min_mse = float("inf")
    for epoch in range(NUM_EPOCHS):
        model.train()
        
        loss_sum = 0.0
        for feature, target in tqdm(train_loader, desc="Epoch {}".format(epoch)):
            feature = feature.to(DEVICE).float()
            # print(feature.size())
            # feature = feature.view(feature.size(0), -1)

            # print(feature.size())
            feature = torch.transpose(feature, 0, 1)
            target = target.to(DEVICE).float()

            output = model(feature)

            if DEBUG:
                print(output.detach().cpu().numpy())
                print(target.cpu().numpy())
                break

            loss = criterion(output, target)
            
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            loss_sum += output.size(0) * loss.item()
        
        training_loss = loss_sum / len(train_set)
        validation_loss = eval(model, criterion, val_loader, len(val_set))

        if validation_loss < min_mse:
            torch.save(model, SAVE_PATH)
            min_mse = validation_loss

        print("Training loss: {:.4f}, validation loss: {:.4f}.".format(training_loss, validation_loss))


def eval(model, criterion, data_loader, n_samples):
    model.eval()

    loss_sum = 0.0
    for feature, target in data_loader:
        feature = feature.to(DEVICE).float()
        # feature = feature.view(feature.size(0), -1)
        feature = torch.transpose(feature, 0, 1)
        target = target.to(DEVICE).float()

        output = model(feature)
        loss = criterion(output, target)

        loss_sum += output.size(0) * loss.item()


    return loss_sum / n_samples


def plot_predictions():
    model = load_latest_model()
    model.to(DEVICE)
    model.eval()

    dataset = SeriesDataset(DATA_PATH, "2017", SEQ_LEN)

    data_loader = DataLoader(dataset, batch_size=64)

    t = []
    o = []

    for feature, target in tqdm(data_loader):
        feature = feature.to(DEVICE).float()
        feature = torch.transpose(feature, 0, 1)
        target = target.to(DEVICE).float()
        
        output = model(feature)

        t.append(target.cpu().numpy())
        o.append(output.squeeze().detach().cpu().numpy())

    target = np.hstack(t)
    output = np.hstack(o)

    target = target * dataset.std + dataset.mean
    output = output * dataset.std + dataset.mean
    
    x = np.linspace(1, len(target), num=len(target))

    # print(target.shape, output.shape)

    plt.plot(x, target, label="target", linewidth=0.8)
    plt.plot(x, output, label="output", linewidth=0.8)
    plt.legend()

    plt.show()


if __name__ == "__main__":
    # train()
    plot_predictions()

import os
import torch

import numpy as np
import pandas as pd

from torch.utils.data import DataLoader, Dataset, random_split
from sklearn.datasets import load_iris
from sklearn.preprocessing import OneHotEncoder, LabelEncoder

class CustomDataset(Dataset):

    def __init__(self, path, year, seq_len=24):
        super().__init__()
        feature, target = load_data(path, year)
        self.feature = feature
        self.target = target
        self.seq_len = seq_len

    def __getitem__(self, index):
        pre_feature = self.feature[index + self.seq_len : index + self.seq_len * 2, :]
        pre_target = np.expand_dims(self.target[index : index + self.seq_len], axis=1)

        # print(pre_feature.shape, pre_target.shape)
        feature = np.concatenate([self.feature[index : index + self.seq_len, :], pre_target], axis=1)
        target = self.target[index + self.seq_len]
        
        print(type(feature), type(target))
        return feature, target

    def __len__(self):
        return self.feature.shape[0] - self.seq_len


class SeriesDataset(Dataset):

    def __init__(self, path, year, seq_len=20):
        super().__init__()
        self.data = load_data(path, year)
        self.feature = None
        self.target = None
        self.seq_len = seq_len

        self.construct_feature_and_target()
        self.mean, self.std = self.normlize()
        self.feature_size = self.feature.shape[1]

    def __getitem__(self, index):
        feature = self.feature[index : index + self.seq_len, :]
        target = self.target[index + self.seq_len]

        return feature, target

    def __len__(self):
        return self.feature.shape[0] - self.seq_len
        
    def construct_feature_and_target(self):
        output = self.data[0]
        waterhead = self.data[1]
        program = self.data[2]
        datetime_index = pd.to_datetime(output["time"])

        # output = np.expand_dims(output["data"].values, axis=1)
        # waterhead = np.expand_dims(waterhead["data"].values, axis=1)
        # program = np.expand_dims(program["data"].values, axis=1)
        # time_feature = generate_time_stamp(datetime_index)
        # print(type(output["data"])) # <class 'pandas.core.series.Series'>
        print(type(output["data"]))  # <class 'pandas.core.series.Series'>
        output = np.expand_dims(output["data"].values[:-96], axis=1) #-96指到倒数第96行
        waterhead = np.expand_dims(waterhead["data"].values[96:], axis=1)
        program = np.expand_dims(program["data"].values[96:], axis=1)
        time_feature = generate_time_stamp(datetime_index)[96:, :]

        print(output.shape)
        print(waterhead.shape)
        print(program.shape)
        print(time_feature.shape)

        # print([x.shape for x in [output, waterhead, program, time_feature]])
        
        self.target = np.squeeze(program)
        self.feature = np.concatenate([output, waterhead, program, time_feature], axis=1) #axis=0是在行上增加，axis=1是在列上增加

    def normlize(self):
        feature_mean = np.mean(self.feature[:, :3], axis=0) # 均值    Q：为嘛是3
        feature_std = np.std(self.feature[:, :3], axis=0)   # 标准差

        target_mean = feature_mean[2]    # Q：target和feature有啥不一样？
        target_std = feature_std[2]

        time_max = np.max(self.feature[:, 3:], axis=0)
        time_min = np.min(self.feature[:, 3:], axis=0)        

        self.feature[:, :3] = (self.feature[:, :3] - feature_mean) / (feature_std + 1e-7)
        self.feature[:, 3:] = (self.feature[:, 3:] - time_min) / (time_max - time_min)
        self.target = (self.target - target_mean) / (target_std + 1e-7)

        return target_mean, target_std


class Seq2seqDataset(Dataset):
    pass


class IrisDataset(Dataset):

    def __init__(self):
        super().__init__()
        iris = load_iris(True)
        
        self.feature = iris[0]
        self.target = iris[1]
        print(self.feature.shape)
        print(self.target.shape)

    def __getitem__(self, index):
        feature = self.feature[index, :]
        target = self.target[index]

        return feature, target

    def __len__(self):
        return self.feature.shape[0]



def load_data(path, year):
    paths = os.listdir(path)
    paths = list(filter(lambda x: ".csv" in x, paths))

    data_files = []
    
    for p in paths:
        if year in p:
            data_files.append(p)
    
    output = None
    waterhead = None
    program = None

    for f in data_files:
        if "output" in f:
            output = os.path.join(path, f)
        elif "waterhead" in f:
            waterhead = os.path.join(path, f)
        else:
            program = os.path.join(path, f)

    # feature_series = concat_feature(output, waterhead, program)
    # targets = pd.read_csv(program, names=["time", "data"])["data"].values[96:]

    output = pd.read_csv(output, names=["time", "data"])
    waterhead = pd.read_csv(waterhead, names=["time", "data"])
    program = pd.read_csv(program, names=["time", "data"])

    return output, waterhead, program


def generate_time_stamp(dt_index, onehot_encode=False):
    months = dt_index.dt.month.values.astype(np.int).reshape(-1, 1)
    days = dt_index.dt.day.values.astype(np.int).reshape(-1, 1)
    hours = dt_index.dt.hour.values.astype(np.int).reshape(-1, 1)
    minutes = dt_index.dt.minute.values.astype(np.int).reshape(-1, 1)

    if onehot_encode:
        encoder = OneHotEncoder(categories="auto")

        months = encoder.fit_transform(months).toarray()
        days = encoder.fit_transform(days).toarray()
        hours = encoder.fit_transform(hours).toarray()
        minutes = encoder.fit_transform(minutes).toarray()

    time_feature = np.concatenate([months, days, hours, minutes], axis=1).astype(np.float)

    return time_feature


def convert_to_ndarray(file, require_time_stamp=False):
    df = pd.read_csv(file, names=["time", "data"])
    
    if require_time_stamp:
        df["time"] = pd.to_datetime(df["time"])
        temp_df = pd.DataFrame(columns=["month", "day", "hour", "minute", "data"])

        temp_df["month"] = df["time"].dt.month
        temp_df["day"] = df["time"].dt.day
        temp_df["hour"] = df["time"].dt.hour
        temp_df["minute"] = df["time"].dt.minute
        temp_df["data"] = df["data"]

        temp_arr = temp_df.values

        temp_list = []
        for i in range(1, temp_arr.shape[0] - 15, 15):
            temp_list.append(np.expand_dims(temp_arr[i:i+15, :], axis=1))

        temp_arr = np.concatenate(temp_list, axis=1)
    else:
        temp_arr = pd.Series(data=df["data"]).values

        temp_list = []
        for i in range(1, temp_arr.shape[0] - 15, 15):
            temp_list.append(np.expand_dims(np.expand_dims(temp_arr[i:i+15], axis=1), axis=2))

        temp_arr = np.concatenate(temp_list, axis=1)

    return temp_arr

def concat_feature(output, waterhead, program):
    output_df = pd.read_csv(output, names=["time", "data"])
    waterhead_df = pd.read_csv(waterhead, names=["time", "data"])
    # program_df = pd.read_csv(program, names=["time", "data"])

    output = pd.Series(data=output_df["data"])
    waterhead = pd.Series(data=waterhead_df["data"])
    output.index = pd.to_datetime(output_df["time"])
    waterhead.index = pd.to_datetime(waterhead_df["time"])
    # program = pd.Series(data=program_df["data"])
    # program.index = pd.to_datetime(program_df["time"])

    year_days = pd.date_range(start=waterhead.index[0] + pd.Timedelta(days=1), end=waterhead.index[-1], freq="D")

    features = []
    for day in year_days:
        pre_day = (day - pd.Timedelta(days=1)).strftime("%Y-%m-%d")
        cur_day = day.strftime("%Y-%m-%d")
        feature = np.tile(np.concatenate([output[pre_day], waterhead[cur_day]], axis=0), reps=(96, 1))
        features.append(feature)
        # break

    features = np.vstack(features)    

    return features


def split_train_test(dataset, ratio=0.8):
    length = len(dataset)
    train_len = int(length * ratio)
    test_len = length - train_len
    split_list = [train_len, test_len]

    train_set, test_set = random_split(dataset, split_list)

    return train_set, test_set


if __name__ == "__main__":

    dataset = SeriesDataset("./data/", "2017")
    # dataset = CustomDataset("./data/", "2018")
    feature, _ = dataset.__getitem__(1)
    _, target = dataset.__getitem__(0)
    # feature = dataset.__getitem__(len(dataset) - 1)
    print(feature, target)
    # print(len(dataset))
    # print(feature.dtype)
    # print(target)