二手车价格预测task03:特征工程

二手车价格预测task03:特征工程

1.学习了operator模块operator.itemgetter()函数
2.学习了箱线图
3.了解了特征工程的方法 (内容介绍)
4.敲代码学习,加注解

以下是代码

代码示例

1.导入数据

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')
%matplotlib inline
# 这个之前没见过
from operator import itemgetter

train = pd.read_csv('car_train_0110.csv', sep=' ')
test = pd.read_csv('car_testA_0110.csv',sep= ' ')

train.shape, test.shape

((250000, 40), (50000, 39))

train.columns # 看下列名

Index(['SaleID', 'name', 'regDate', 'model', 'brand', 'bodyType', 'fuelType',
       'gearbox', 'power', 'kilometer', 'notRepairedDamage', 'regionCode',
       'seller', 'offerType', 'creatDate', 'price', 'v_0', 'v_1', 'v_2', 'v_3',
       'v_4', 'v_5', 'v_6', 'v_7', 'v_8', 'v_9', 'v_10', 'v_11', 'v_12',
       'v_13', 'v_14', 'v_15', 'v_16', 'v_17', 'v_18', 'v_19', 'v_20', 'v_21',
       'v_22', 'v_23'],
      dtype='object')

2.异常值处理 (删除或者用最大值填充)

# 一个异常值处理的代码,可以随便调用
def outliers_proc(data,col_name,scale=3):
    """
    用于清洗异常值，默认用 box_plot（scale=3）进行清洗
    :param data: 接收 pandas 数据格式
    :param col_name: pandas 列名
    :param scale: 尺度
    :return:
    """
    def box_plot_outliers(data_ser, box_scale):
        """
        利用箱线图去除异常值
        :param data_ser: 接收 pandas.Series 数据格式
        :param box_scale: 箱线图尺度，
        :return:
        """
        iqr = box_scale * (data_ser.quantile(0.75) - data_ser.quantile(0.25)) # 像是一个阈值的定义
        val_low = data_ser.quantile(0.25) - iqr # 一列中的最小值
        val_up = data_ser.quantile(0.75) + iqr  # 一列中的最大值
        rule_low = (data_ser < val_low) # 小于最小值的设置为了0
        rule_up = (data_ser > val_up)   # 大于最小值的设置为了1
        return (rule_low, rule_up), (val_low, val_up)
    
    # 这里是删除异常值 -- 不建议删除吧..
    data_n = data.copy()
    data_series = data_n[col_name] # 得到了这一列数据
    rule, value = box_plot_outliers(data_series, box_scale=scale)
    index = np.arange(data_series.shape[0])[rule[0] | rule[1]]
    print("Delete number is: {}".format(len(index)))
    data_n = data_n.drop(index)
    data_n.reset_index(drop=True, inplace=True)
    print("Now column number is: {}".format(data_n.shape[0]))
    index_low = np.arange(data_series.shape[0])[rule[0]]
    outliers = data_series.iloc[index_low]
    print("Description of data less than the lower bound is:")
    print(pd.Series(outliers).describe())
    index_up = np.arange(data_series.shape[0])[rule[1]]
    outliers = data_series.iloc[index_up]
    print("Description of data larger than the upper bound is:")
    print(pd.Series(outliers).describe())
    
    
#     # 这里是异常值的填充 -- 实现这个替换上面直接删除的那个
#     data_n = data.copy()
#     data_series = data_n[col_name] # 得到了这一列数据
#     rule, value = box_plot_outliers(data_series, box_scale=scale)
#     index = np.arange(data_series.shape[0])[rule[0] | rule[1]]
#     print("Delete number is: {}".format(len(index)))
#     data_n = data_n.drop(index)
#     data_n.reset_index(drop=True, inplace=True)
#     print("Now column number is: {}".format(data_n.shape[0]))
#     index_low = np.arange(data_series.shape[0])[rule[0]]
#     outliers = data_series.iloc[index_low]
#     print("Description of data less than the lower bound is:")
#     print(pd.Series(outliers).describe())
#     index_up = np.arange(data_series.shape[0])[rule[1]]
#     outliers = data_series.iloc[index_up]
#     print("Description of data larger than the upper bound is:")
#     print(pd.Series(outliers).describe())
    
    
    
    fig, ax = plt.subplots(1, 2, figsize=(10, 7))
    sns.boxplot(y=data[col_name], data=data, palette="Set1", ax=ax[0])
    sns.boxplot(y=data_n[col_name], data=data_n, palette="Set1", ax=ax[1])
    return data_n

# 可以删掉一些异常数据，以 power 为例。  
# 这里删不删可以自行判断
# 但是要注意 test 的数据不能删 

train = outliers_proc(train, 'power', scale=3)

Delete number is: 1290
Now column number is: 248710
Description of data less than the lower bound is:
count    0.0
mean     NaN
std      NaN
min      NaN
25%      NaN
50%      NaN
75%      NaN
max      NaN
Name: power, dtype: float64
Description of data larger than the upper bound is:
count     1290.000000
mean      1104.958915
std       2373.619469
min        392.000000
25%        420.000000
50%        476.000000
75%        579.000000
max      20000.000000
Name: power, dtype: float64

# 删除所有数值型特征中的异常数据 -->这里最好不要删除,可以尝试用最大值和最小值替换异常值
numeric_features = ['power', 'kilometer', 'v_0', 'v_1', 'v_2', 'v_3', 'v_4', 'v_5', 'v_6', 'v_7', 'v_8', 'v_9', 'v_10'