hongyi lee 作业1

用以记录hongyi lee 机器学习课程第一次作业
调参记录：
1.features的选择，需要结合22年sample的feature选择，结合到21例中
2.激活函数选择，sigmoid效果要优于relu，优化器选择，尝试SGD
目前效果最好的是，SGD+Tanh，lr=0.0015
最低的loss为1.2
注意：主体参考Heng-Jui Chang @ NTUEE.
Copying or reusing this code is required to specify the original author.
E.g.
Source: Heng-Jui Chang @ NTUEE (https://github.com/ga642381/ML2021-Spring/blob/main/HW01/HW01.ipynb)
仅完成了simple baseline以及部分medium baseline，效果远远不够，仅供参考。

#导入包
import torch
import torch.nn as nn
from torch.utils.data import DataLoader,Dataset
import numpy as np
import os
import csv
import matplotlib.pyplot as plt
from  matplotlib.pyplot import figure

#
train_path="D:\data\covid.train.csv"
test_path="D:\data\covid.test.csv"
myseed = 42069  # set a random seed for reproducibility
#固定随机数种子，目的是保持算法的可复现性。
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
np.random.seed(myseed)
torch.manual_seed(myseed)
device= torch.device("cuda" if torch.cuda.is_available() else "cpu")

#设置gpu的运行
def get_device():
    ''' Get device (if GPU is available, use GPU) '''
    return 'cuda' if torch.cuda.is_available() else 'cpu'
def plot_learning_curve(loss_record, title=''):
    ''' Plot learning curve of your DNN (train & dev loss) '''
    total_steps = len(loss_record['train'])
    x_1 = range(total_steps)
    x_2 = x_1[::len(loss_record['train']) // len(loss_record['dev'])]
    figure(figsize=(6, 4))
    plt.plot(x_1, loss_record['train'], c='tab:red', label='train')
    plt.plot(x_2, loss_record['dev'], c='tab:cyan', label='dev')
    plt.ylim(0.0, 5.)
    plt.xlabel('Training steps')
    plt.ylabel('MSE loss')
    plt.title('Learning curve of {}'.format(title))
    plt.legend()
    plt.show()
def plot_pred(dv_set, model, device, lim=35., preds=None, targets=None):
    ''' Plot prediction of your DNN '''
    if preds is None or targets is None:
        model.eval()
        preds, targets = [], []
        for x, y in dv_set:
            x, y = x.to(device), y.to(device)
            with torch.no_grad():
                pred = model(x)
                preds.append(pred.detach().cpu())
                targets.append(y.detach().cpu())
        preds = torch.cat(preds, dim=0).numpy()
        targets = torch.cat(targets, dim=0).numpy()

    figure(figsize=(5, 5))
    plt.scatter(targets, preds, c='r', alpha=0.5)
    plt.plot([-0.2, lim], [-0.2, lim], c='b')
    plt.xlim(-0.2, lim)
    plt.ylim(-0.2, lim)
    plt.xlabel('ground truth value')
    plt.ylabel('predicted value')
    plt.title('Ground Truth v.s. Prediction')
    plt.show()


class COVID19Dataset(Dataset):
    ''' Dataset for loading and preprocessing the COVID19 dataset '''

    def __init__(self,
                 path,
                 mode='train',
                 target_only=False):
        self.mode = mode

        # Read data into numpy arrays
        with open(path, 'r') as fp:
            data = list(csv.reader(fp))
            data = np.array(data[1:])[:, 1:].astype(float)

        if not target_only:
            feats = list(range(93))
        else:
            # TODO: Using 40 states & 2 tested_positive features (indices = 57 & 75)
            pass

        if mode == 'test':
            # Testing data
            # data: 893 x 93 (40 states + day 1 (18) + day 2 (18) + day 3 (17))
            data = data[:, feats]
            self.data = torch.FloatTensor(data)
        else:
            # Training data (train/dev sets)
            # data: 2700 x 94 (40 states + day 1 (18) + day 2 (18) + day 3 (18))
            target = data[:, -1]
            data = data[:, feats]

            # Splitting training data into train & dev sets
            if mode == 'train':
                indices = [i for i in range(len(data)) if i % 10 != 0]
            elif mode == 'dev':
                indices = [i for i in range(len(data)) if i % 10 == 0]

            # Convert data into PyTorch tensors
            self.data = torch.FloatTensor(data[indices])
            self.target = torch.FloatTensor(target[indices])

            # Normalize features (you may remove this part to see what will happen)
        self.data[:, 40:] = \
            (self.data[:, 40:] - self.data[:, 40:].mean(dim=0, keepdim=True)) \
            / self.data[:, 40:].std(dim=0, keepdim=True)

        self.dim = self.data.shape[1]

        print('Finished reading the {} set of COVID19 Dataset ({} samples found, each dim = {})'
              .format(mode, len(self.data), self.dim))

    def __getitem__(self, index):
        # Returns one sample at a time
        if self.mode in ['train', 'dev']:
            # For training
            return self.data[index], self.target[index]
        else:
            # For testing (no target)
            return self.data[index]

    def __len__(self):
        # Returns the size of the dataset
        return len(self.data)
def prep_dataloader(path, mode, batch_size, n_jobs=0, target_only=False):
    ''' Generates a dataset, then is put into a dataloader. '''
    dataset = COVID19Dataset(path, mode=mode, target_only=target_only)  # Construct dataset
    dataloader = DataLoader(
        dataset, batch_size,
        shuffle=(mode == 'train'), drop_last=False,
        num_workers=n_jobs, pin_memory=True)                            # Construct dataloader
    return dataloader
class NeuralNet(nn.Module):
    ''' A simple fully-connected deep neural network '''
    def __init__(self, input_dim):
        super(NeuralNet, self).__init__()

        # Define your neural network here
        # TODO: How to modify this model to achieve better performance?
        self.net = nn.Sequential(
            nn.Linear(input_dim, 128),
            nn.ReLU(),
            nn.Linear(128, 1)
        )

        # Mean squared error loss
        self.criterion = nn.MSELoss(reduction='mean')

    def forward(self, x):
        ''' Given input of size (batch_size x input_dim), compute output of the network '''
        return self.net(x).squeeze(1)

    def cal_loss(self, pred, target):
        ''' Calculate loss '''
        # TODO: you may implement L2 regularization here
        return self.criterion(pred, target)
def train(tr_set, dv_set, model, config,device):
    ''' DNN training '''

    n_epochs = config['n_epochs']  # Maximum number of epochs

    # Setup optimizer
    optimizer = getattr(torch.optim, config['optimizer'])(
        model.parameters(), **config['optim_hparas'])

    min_mse = 100.
    loss_record = {'train': [], 'dev': []}      # for recording training loss
    early_stop_cnt = 0
    epoch = 0
    while epoch < n_epochs:
        model.train()                           # set model to training mode
        for x, y in tr_set:                     # iterate through the dataloader
            optimizer.zero_grad()               # set gradient to zero
            x, y = x.to(device), y.to(device)   # move data to device (cpu/cuda)
            pred = model(x)                     # forward pass (compute output)
            mse_loss = model.cal_loss(pred, y)  # compute loss
            mse_loss.backward()                 # compute gradient (backpropagation)
            optimizer.step()                    # update model with optimizer
            loss_record['train'].append(mse_loss.detach().cpu().item())
            # After each epoch, test your model on the validation (development) set.
            dev_mse = dev(dv_set, model, device)
            if dev_mse < min_mse:
                # Save model if your model improved
                min_mse = dev_mse
                print('Saving model (epoch = {:4d}, loss = {:.4f})'
                      .format(epoch + 1, min_mse))
                torch.save(model.state_dict(), config['save_path'])  # Save model to specified path
                early_stop_cnt = 0
            else:
                early_stop_cnt += 1

            epoch += 1
            loss_record['dev'].append(dev_mse)
            if early_stop_cnt > config['early_stop']:
                # Stop training if your model stops improving for "config['early_stop']" epochs.
                break

        print('Finished training after {} epochs'.format(epoch))
        return min_mse, loss_record
def dev(dv_set, model, device):
    model.eval()                                # set model to evalutation mode
    total_loss = 0
    for x, y in dv_set:                         # iterate through the dataloader
        x, y = x.to(device), y.to(device)       # move data to device (cpu/cuda)
        with torch.no_grad():                   # disable gradient calculation
            pred = model(x)                     # forward pass (compute output)
            mse_loss = model.cal_loss(pred, y)  # compute loss
        total_loss += mse_loss.detach().cpu().item() * len(x)  # accumulate loss
    total_loss = total_loss / len(dv_set.dataset)              # compute averaged loss

    return total_loss

def test(tt_set, model, device):
        model.eval()  # set model to evalutation mode
        preds = []
        for x in tt_set:  # iterate through the dataloader
            x = x.to(device)  # move data to device (cpu/cuda)
            with torch.no_grad():  # disable gradient calculation
                pred = model(x)  # forward pass (compute output)
                preds.append(pred.detach().cpu())  # collect prediction
        preds = torch.cat(preds, dim=0).numpy()  # concatenate all predictions and convert to a numpy array
        return preds
device = get_device()                 # get the current available device ('cpu' or 'cuda')
os.makedirs('models', exist_ok=True)  # The trained model will be saved to ./models/
target_only = False                   # TODO: Using 40 states & 2 tested_positive features

# TODO: How to tune these hyper-parameters to improve your model's performance?
config = {
    'n_epochs': 3000,                # maximum number of epochs
    'batch_size': 3,               # mini-batch size for dataloader
    'optimizer': 'SGD',              # optimization algorithm (optimizer in torch.optim)
    'optim_hparas': {                # hyper-parameters for the optimizer (depends on which optimizer you are using)
        'lr': 0.0005,                 # learning rate of SGD
        'momentum': 0.90           # momentum for SGD
    },
    'early_stop': 200,               # early stopping epochs (the number epochs since your model's last improvement)
    'save_path': 'models/model.pth'  # your model will be saved here
}
tr_set = prep_dataloader(train_path, 'train', config['batch_size'], target_only=target_only)
dv_set = prep_dataloader(train_path, 'dev', config['batch_size'], target_only=target_only)
tt_set = prep_dataloader(test_path, 'test', config['batch_size'], target_only=target_only)
model = NeuralNet(tr_set.dataset.dim)  # Construct model and move to device
model_loss, model_loss_record = train(tr_set, dv_set, model, config,device)
plot_learning_curve(model_loss_record, title='deep model')
del model
model = NeuralNet(tr_set.dataset.dim).to(device)
ckpt = torch.load(config['save_path'], map_location='cpu')  # Load your best model
model.load_state_dict(ckpt)
plot_pred(dv_set, model, device)  # Show prediction on the validation set
def save_pred(preds, file):
    ''' Save predictions to specified file '''
    print('Saving results to {}'.format(file))
    with open(file, 'w') as fp:
        writer = csv.writer(fp)
        writer.writerow(['id', 'tested_positive'])
        for i, p in enumerate(preds):
            writer.writerow([i, p])

preds = test(tt_set, model, device)  # predict COVID-19 cases with your model
save_pred(preds, 'pred.csv')         # save prediction file to pred.csv

结果如下

E:\abc\d\envs\pytorch\python.exe "D:/pycharm-project、/hongyi lee.py"
Finished reading the train set of COVID19 Dataset (2430 samples found, each dim = 93)
Finished reading the dev set of COVID19 Dataset (270 samples found, each dim = 93)
Finished reading the test set of COVID19 Dataset (893 samples found, each dim = 93)
Saving model (epoch =   13, loss = 70.6493)
Saving model (epoch =   14, loss = 49.4650)
Saving model (epoch =   15, loss = 46.5434)
Saving model (epoch =   26, loss = 34.4032)
Saving model (epoch =   50, loss = 28.3461)
Saving model (epoch =   51, loss = 22.2500)
Saving model (epoch =   85, loss = 21.6892)
Saving model (epoch =   86, loss = 21.3725)
Saving model (epoch =   99, loss = 20.6347)
Saving model (epoch =  100, loss = 19.8852)
Saving model (epoch =  101, loss = 19.3809)
Saving model (epoch =  102, loss = 19.0243)
Saving model (epoch =  103, loss = 18.8308)
Saving model (epoch =  104, loss = 18.6424)
Saving model (epoch =  105, loss = 18.4617)
Saving model (epoch =  106, loss = 18.3020)
Saving model (epoch =  107, loss = 17.7380)
Saving model (epoch =  108, loss = 17.2889)
Saving model (epoch =  109, loss = 17.0242)
Saving model (epoch =  110, loss = 16.6098)
Saving model (epoch =  111, loss = 16.4818)
Saving model (epoch =  112, loss = 16.4712)
Saving model (epoch =  123, loss = 14.6666)
Saving model (epoch =  124, loss = 13.3702)
Saving model (epoch =  125, loss = 12.9795)
Saving model (epoch =  133, loss = 12.3687)
Saving model (epoch =  134, loss = 10.9519)
Saving model (epoch =  135, loss = 10.4229)
Saving model (epoch =  143, loss = 9.7748)
Saving model (epoch =  152, loss = 9.1807)
Saving model (epoch =  153, loss = 8.4716)
Saving model (epoch =  178, loss = 8.3995)
Saving model (epoch =  179, loss = 7.4104)
Saving model (epoch =  189, loss = 7.0068)
Saving model (epoch =  201, loss = 6.6279)
Saving model (epoch =  202, loss = 6.3601)
Saving model (epoch =  234, loss = 6.2692)
Saving model (epoch =  235, loss = 5.4796)
Saving model (epoch =  256, loss = 5.1620)
Saving model (epoch =  257, loss = 4.9907)
Saving model (epoch =  258, loss = 4.9527)
Saving model (epoch =  259, loss = 4.8295)
Saving model (epoch =  260, loss = 4.8031)
Saving model (epoch =  267, loss = 4.7941)
Saving model (epoch =  268, loss = 4.6891)
Saving model (epoch =  269, loss = 4.5623)
Saving model (epoch =  270, loss = 4.4726)
Saving model (epoch =  283, loss = 4.4002)
Saving model (epoch =  284, loss = 4.2521)
Saving model (epoch =  285, loss = 4.1973)
Saving model (epoch =  326, loss = 3.9615)
Saving model (epoch =  327, loss = 3.7901)
Saving model (epoch =  347, loss = 3.7351)
Saving model (epoch =  348, loss = 3.6763)
Saving model (epoch =  349, loss = 3.6575)
Saving model (epoch =  350, loss = 3.6378)
Saving model (epoch =  353, loss = 3.6033)
Saving model (epoch =  354, loss = 3.5658)
Saving model (epoch =  355, loss = 3.4985)
Saving model (epoch =  356, loss = 3.4540)
Saving model (epoch =  363, loss = 3.3690)
Saving model (epoch =  364, loss = 3.3445)
Saving model (epoch =  387, loss = 3.3172)
Saving model (epoch =  416, loss = 3.1696)
Saving model (epoch =  417, loss = 3.1600)
Saving model (epoch =  513, loss = 3.0570)
Saving model (epoch =  530, loss = 3.0378)
Saving model (epoch =  536, loss = 2.9933)
Saving model (epoch =  537, loss = 2.8749)
Saving model (epoch =  538, loss = 2.8659)
Saving model (epoch =  607, loss = 2.8252)
Saving model (epoch =  608, loss = 2.8145)
Saving model (epoch =  609, loss = 2.8123)
Saving model (epoch =  613, loss = 2.7998)
Saving model (epoch =  614, loss = 2.7489)
Saving model (epoch =  615, loss = 2.6932)
Saving model (epoch =  616, loss = 2.6855)
Saving model (epoch =  655, loss = 2.6407)
Saving model (epoch =  656, loss = 2.3548)
Saving model (epoch =  657, loss = 2.3236)
Saving model (epoch =  683, loss = 2.2893)
Saving model (epoch =  684, loss = 2.1694)
Saving model (epoch =  685, loss = 2.1339)
Saving model (epoch =  775, loss = 2.0524)
Finished training after 810 epochs
Saving results to pred.csv

进程已结束，退出代码 0

后续优化会继续记录。