学习笔记：动手学深度学习 19 Kaggle 预测房价

Python 3.8.8 (default, Apr 13 2021, 15:08:03) [MSC v.1916 64 bit (AMD64)]
Type 'copyright', 'credits' or 'license' for more information
IPython 7.22.0 -- An enhanced Interactive Python. Type '?' for help.
PyDev console: using IPython 7.22.0
Python 3.8.8 (default, Apr 13 2021, 15:08:03) [MSC v.1916 64 bit (AMD64)] on win32
import hashlib
import os
import tarfile
import zipfile
import requests
DATA_HUB = dict()
DATA_URL = 'http://d2l-data.s3-accelerate.amazonaws.com/'
"""下面的download函数用来下载数据集"""
Out[4]: '下面的download函数用来下载数据集'
def download(name, cache_dir=os.path.join('..', 'data')):  #@save
    """下载一个DATA_HUB中的文件，返回本地文件名。"""
    assert name in DATA_HUB, f"{name} 不存在于 {DATA_HUB}."
    url, sha1_hash = DATA_HUB[name]
    os.makedirs(cache_dir, exist_ok=True)
    fname = os.path.join(cache_dir, url.split('/')[-1])
    if os.path.exists(fname):
        sha1 = hashlib.sha1()
        with open(fname, 'rb') as f:
            while True:
                data = f.read(1048576)
                if not data:
                    break
                sha1.update(data)
        if sha1.hexdigest() == sha1_hash:
            return fname  # Hit cache
    print(f'正在从{url}下载{fname}...')
    r = requests.get(url, stream=True, verify=True)
    with open(fname, 'wb') as f:
        f.write(r.content)
    return fname
def download_extract(name, folder=None):  #@save
    """下载并解压zip/tar文件。"""
    fname = download(name)
    base_dir = os.path.dirname(fname)
    data_dir, ext = os.path.splitext(fname)
    if ext == '.zip':
        fp = zipfile.ZipFile(fname, 'r')
    elif ext in ('.tar', '.gz'):
        fp = tarfile.open(fname, 'r')
    else:
        assert False, '只有zip/tar文件可以被解压缩。'
    fp.extractall(base_dir)
    return os.path.join(base_dir, folder) if folder else data_dir
def download_all():  #@save
    """下载DATA_HUB中的所有文件。"""
    for name in DATA_HUB:
        download(name)
        
import matplotlib.pyplot as plt
Backend Qt5Agg is interactive backend. Turning interactive mode on.
import numpy as np
import pandas as pd
import torch
from torch import nn
from d2l import torch as d2l
"""下载并缓存Kaggle房屋数据集"""
Out[9]: '下载并缓存Kaggle房屋数据集'
DATA_HUB['kaggle_house_train'] = (  #@save
    DATA_URL + 'kaggle_house_pred_train.csv',
    '585e9cc93e70b39160e7921475f9bcd7d31219ce')
DATA_HUB['kaggle_house_test'] = (  #@save
    DATA_URL + 'kaggle_house_pred_test.csv',
    'fa19780a7b011d9b009e8bff8e99922a8ee2eb90')
train_data = pd.read_csv(download('kaggle_house_train'))
test_data = pd.read_csv(download('kaggle_house_test'))
正在从http://d2l-data.s3-accelerate.amazonaws.com/kaggle_house_pred_train.csv下载..\data\kaggle_house_pred_train.csv...
正在从http://d2l-data.s3-accelerate.amazonaws.com/kaggle_house_pred_test.csv下载..\data\kaggle_house_pred_test.csv...
print(train_data.shape)
print(test_data.shape)
(1460, 81)
(1459, 80)
"""让我们看看前四个和最后两个特征，以及相应标签（房价）"""
Out[13]: '让我们看看前四个和最后两个特征，以及相应标签（房价）'
print(train_data.iloc[0:4, [0, 1, 2, 3, -3, -2, -1]])
   Id  MSSubClass MSZoning  LotFrontage SaleType SaleCondition  SalePrice
0   1          60       RL         65.0       WD        Normal     208500
1   2          20       RL         80.0       WD        Normal     181500
2   3          60       RL         68.0       WD        Normal     223500
3   4          70       RL         60.0       WD       Abnorml     140000
"""第一个特征是ID，这有助于模型识别每个训练样本。虽然这很方便，但它不携带任何用于预测的信息。因此，在将数据提供给模型之前，我们将其从数据集中删除"""
Out[15]: '第一个特征是ID，这有助于模型识别每个训练样本。虽然这很方便，但它不携带任何用于预测的信息。因此，在将数据提供给模型之前，我们将其从数据集中删除'
all_features = pd.concat((train_data.iloc[:, 1:-1], test_data.iloc[:, 1:]))
print(train_data.shape)
print(test_data.shape)
(1460, 81)
(1459, 80)
"""上面的函数吧两个标签去掉后合并到all_feature里"""
Out[18]: '上面的函数吧两个标签去掉后合并到all_feature里'
print(all_feature.shape)
Traceback (most recent call last):
  File "C:\ProgramData\Anaconda3\lib\site-packages\IPython\core\interactiveshell.py", line 3437, in run_code
    exec(code_obj, self.user_global_ns, self.user_ns)
  File "<ipython-input-19-12cdc88ee3cc>", line 1, in <module>
    print(all_feature.shape)
NameError: name 'all_feature' is not defined
print(all_features.shape)
(2919, 79)
"""所有特征如果不是object的话就是数值，对数值特征，每一列减去均值除以方差"""
Out[21]: '所有特征如果不是object的话就是数值，对数值特征，每一列减去均值除以方差'
numeric_features = all_features.dtypes[all_features.dtypes != 'object'].index
all_features[numeric_features] = all_features[numeric_features].apply(
    lambda x: (x - x.mean()) / (x.std()))
# 在标准化数据之后，所有数据都意味着消失，因此我们可以将缺失值设置为0
all_features[numeric_features] = all_features[numeric_features].fillna(0)
all_features = pd.get_dummies(all_features, dummy_na=True)
all_features.shape
Out[23]: (2919, 331)
"""根据独热编码，如果“MSZoning”的原始值为“RL”，则:“MSZoning_RL”为1，“MSZoning_RM”为0。pandas软件包会自动为我们实现这一点。"""
Out[24]: '根据独热编码，如果“MSZoning”的原始值为“RL”，则:“MSZoning_RL”为1，“MSZoning_RM”为0。pandas软件包会自动为我们实现这一点。'
# `Dummy_na=True` 将“na”（缺失值）视为有效的特征值，并为其创建指示符特征。
n_train = train_data.shape[0]
train_features = torch.tensor(all_features[:n_train].values, dtype=d2l.float32)
test_features = torch.tensor(all_features[n_train:].values, dtype=d2l.float32)
train_labels = torch.tensor(
    train_data.SalePrice.values.reshape(-1, 1), dtype=d2l.float32)
loss = nn.MSELoss()
in_features = train_features.shape[1]
"""单层线性回归"""
Out[28]: '单层线性回归'
def get_net():
    net = nn.Sequential(nn.Linear(in_features,1))
    return net
"""对于房价，就像股票价格一样，我们关心的是相对数量，而不是绝对数量。因此，我们更关心相对误差"""
Out[30]: '对于房价，就像股票价格一样，我们关心的是相对数量，而不是绝对数量。因此，我们更关心相对误差'
def log_rmse(net, features, labels):
    # 为了在取对数时进一步稳定该值，将小于1的值设置为1
    clipped_preds = torch.clamp(net(features), 1, float('inf'))
    rmse = torch.sqrt(loss(torch.log(clipped_preds),
                           torch.log(labels)))
    return rmse.item()
def train(net, train_features, train_labels, test_features, test_labels,
          num_epochs, learning_rate, weight_decay, batch_size):
    train_ls, test_ls = [], []
    train_iter = d2l.load_array((train_features, train_labels), batch_size)
    # 这里使用的是Adam优化算法
    optimizer = torch.optim.Adam(net.parameters(),
                                 lr = learning_rate,
                                 weight_decay = weight_decay)
    for epoch in range(num_epochs):
        for X, y in train_iter:
            optimizer.zero_grad()
            l = loss(net(X), y)
            l.backward()
            optimizer.step()
        train_ls.append(log_rmse(net, train_features, train_labels))
        if test_labels is not None:
            test_ls.append(log_rmse(net, test_features, test_labels))
    return train_ls, test_ls
"""K 折交叉验证"""
Out[33]: 'K 折交叉验证'
def get_k_fold_data(k, i, X, y):
    assert k > 1
    fold_size = X.shape[0] // k
    X_train, y_train = None, None
    for j in range(k):
        idx = slice(j * fold_size, (j + 1) * fold_size)
        X_part, y_part = X[idx, :], y[idx]
        if j == i:
            X_valid, y_valid = X_part, y_part
        elif X_train is None:
            X_train, y_train = X_part, y_part
        else:
            X_train = torch.cat([X_train, X_part], 0)
            y_train = torch.cat([y_train, y_part], 0)
    return X_train, y_train, X_valid, y_valid
def k_fold(k, X_train, y_train, num_epochs, learning_rate, weight_decay,
           batch_size):
    train_l_sum, valid_l_sum = 0, 0
    for i in range(k):
        data = get_k_fold_data(k, i, X_train, y_train)
        net = get_net()
        train_ls, valid_ls = train(net, *data, num_epochs, learning_rate,
                                   weight_decay, batch_size)
        train_l_sum += train_ls[-1]
        valid_l_sum += valid_ls[-1]
        if i == 0:
            d2l.plot(list(range(1, num_epochs + 1)), [train_ls, valid_ls],
                     xlabel='epoch', ylabel='rmse', xlim=[1, num_epochs],
                     legend=['train', 'valid'], yscale='log')
        print(f'fold {i + 1}, train log rmse {float(train_ls[-1]):f}, '
              f'valid log rmse {float(valid_ls[-1]):f}')
    return train_l_sum / k, valid_l_sum / k
"""模型选择"""
Out[36]: '模型选择'
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 5, 0, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.170329, valid log rmse 0.156803
fold 2, train log rmse 0.162168, valid log rmse 0.190858
fold 3, train log rmse 0.164145, valid log rmse 0.168385
fold 4, train log rmse 0.168205, valid log rmse 0.154574
fold 5, train log rmse 0.162800, valid log rmse 0.182962
5-折验证: 平均训练log rmse: 0.165529, 平均验证log rmse: 0.170716
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 0.1, 0, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 3.056470, valid log rmse 3.070585
fold 2, train log rmse 3.056685, valid log rmse 3.067474
fold 3, train log rmse 3.055573, valid log rmse 3.070779
fold 4, train log rmse 3.064620, valid log rmse 3.040255
fold 5, train log rmse 3.057942, valid log rmse 3.060880
5-折验证: 平均训练log rmse: 3.058258, 平均验证log rmse: 3.061995
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 0.1, 0.5, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 3.057000, valid log rmse 3.071048
fold 2, train log rmse 3.056671, valid log rmse 3.067579
fold 3, train log rmse 3.055965, valid log rmse 3.071027
fold 4, train log rmse 3.064961, valid log rmse 3.040616
fold 5, train log rmse 3.058636, valid log rmse 3.061857
5-折验证: 平均训练log rmse: 3.058647, 平均验证log rmse: 3.062426
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 2, 0.5, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.370613, valid log rmse 0.374139
fold 2, train log rmse 0.368042, valid log rmse 0.384315
fold 3, train log rmse 0.367445, valid log rmse 0.381951
fold 4, train log rmse 0.373619, valid log rmse 0.348599
fold 5, train log rmse 0.369029, valid log rmse 0.376345
5-折验证: 平均训练log rmse: 0.369750, 平均验证log rmse: 0.373070
k, num_epochs, lr, weight_decay, batch_size = 3, 50, 2, 0.5, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 1.030238, valid log rmse 1.043583
fold 2, train log rmse 1.034686, valid log rmse 1.038174
fold 3, train log rmse 1.040084, valid log rmse 1.030745
3-折验证: 平均训练log rmse: 1.035003, 平均验证log rmse: 1.037501
k, num_epochs, lr, weight_decay, batch_size = 3, 200, 2, 0.5, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.193460, valid log rmse 0.181236
fold 2, train log rmse 0.180577, valid log rmse 0.200099
fold 3, train log rmse 0.185564, valid log rmse 0.184747
3-折验证: 平均训练log rmse: 0.186534, 平均验证log rmse: 0.188694
k, num_epochs, lr, weight_decay, batch_size = 3, 200, 2, 2, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.398549, valid log rmse 0.203491
fold 2, train log rmse 0.198727, valid log rmse 0.526641
fold 3, train log rmse 0.395527, valid log rmse 0.202768
3-折验证: 平均训练log rmse: 0.330935, 平均验证log rmse: 0.310966
k, num_epochs, lr, weight_decay, batch_size = 3, 200, 1, 0.1, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.451707, valid log rmse 0.461684
fold 2, train log rmse 0.453614, valid log rmse 0.459693
fold 3, train log rmse 0.456515, valid log rmse 0.448323
3-折验证: 平均训练log rmse: 0.453945, 平均验证log rmse: 0.456566
k, num_epochs, lr, weight_decay, batch_size = 5, 100, 5, 0, 64
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.169823, valid log rmse 0.156799
fold 2, train log rmse 0.161673, valid log rmse 0.187564
fold 3, train log rmse 0.164084, valid log rmse 0.168445
fold 4, train log rmse 0.168282, valid log rmse 0.154567
fold 5, train log rmse 0.162482, valid log rmse 0.182689
5-折验证: 平均训练log rmse: 0.165269, 平均验证log rmse: 0.170013
k, num_epochs, lr, weight_decay, batch_size = 10, 100, 1, 0, 32
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 0.297825, valid log rmse 0.268380
fold 2, train log rmse 0.297619, valid log rmse 0.321685
fold 3, train log rmse 0.296151, valid log rmse 0.301120
fold 4, train log rmse 0.293633, valid log rmse 0.317876
fold 5, train log rmse 0.294891, valid log rmse 0.328760
fold 6, train log rmse 0.298303, valid log rmse 0.293056
fold 7, train log rmse 0.297149, valid log rmse 0.273156
fold 8, train log rmse 0.296419, valid log rmse 0.272533
fold 9, train log rmse 0.292034, valid log rmse 0.324450
fold 10, train log rmse 0.298338, valid log rmse 0.276839
10-折验证: 平均训练log rmse: 0.296236, 平均验证log rmse: 0.297786
k, num_epochs, lr, weight_decay, batch_size = 3, 100, 0.1, 0.1, 128
train_l, valid_l = k_fold(k, train_features, train_labels, num_epochs, lr,
                          weight_decay, batch_size)
print(f'{k}-折验证: 平均训练log rmse: {float(train_l):f}, '
      f'平均验证log rmse: {float(valid_l):f}')
fold 1, train log rmse 3.902883, valid log rmse 3.917665
fold 2, train log rmse 3.907660, valid log rmse 3.910068
fold 3, train log rmse 3.911198, valid log rmse 3.903748
3-折验证: 平均训练log rmse: 3.907247, 平均验证log rmse: 3.910493