二手车 数据工程

二手车 数据工程

import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import seaborn as sns
from operator import itemgetter

def outline(data,col_name,scale= 1):
    def box_plot(data_ser,box_scale):
        iqr = box_scale*(data_ser.quantile(0.75)-data_ser.quantile(0.25))
        val_low = data_ser.quantile(0.25)-iqr
        val_up = data_ser.quantile(0.75)+iqr
        rule_low = (data_ser<val_low)
        rule_up = (data_ser>val_up)
        return (rule_low,rule_up),(val_low,val_up)
    data_n = data.copy()
    data_series = data_n[col_name]
    rule,value = box_plot(data_series,box_scale=scale)
    index = np.arange(data_series.shape[0])[rule[0]|rule[1]]
    data_n = data_n.drop(index)
    data_n.rest_index(drop=True,inplace=True)
    print("Now column number is: {}".format(data_n.shape[0]))
    index_low = np.arange(data_series.shape[0])[rule[0]]
    outliers = data_series.iloc[index_low]
    print("Description of data less than the lower bound is:")
    print(pd.Series(outliers).describe())
    index_up = np.arange(data_series.shape[0])[rule[1]]
    outliers = data_series.iloc[index_up]
    print("Description of data larger than the upper bound is:")
    print(pd.Series(outliers).describe())

    fig, ax = plt.subplots(1, 2, figsize=(10, 7))
    sns.boxplot(y=data[col_name], data=data, palette="Set1", ax=ax[0])
    sns.boxplot(y=data_n[col_name], data=data_n, palette="Set1", ax=ax[1])
    return data_n


Train_data = pd.read_csv('used_car_train_20200313.csv',sep = ' ')
Test_data = pd.read_csv('used_car_testA_20200313.csv',sep = ' ')
# Train_data = outline(Train_data,'power',scale = 3)
#训练集和测试放在一起 方便构造特征
Train_data['train']=1
Test_data['test']=0
data = pd.concat([Train_data,Test_data],ignore_index = True)
data['city']=data['regionCode'.apply](lambda x:str(x)[:3])
data = data
Train_gb = Train_data.groupby('brand')
all_info = {}
for kind,kind_data in Train_data:
    info = {}
    kind_data = kind_data[kind_data['price']>0]
    info['brand_amount'] = len(kind_data)
    info['bran_price_max'] = kind_data.price.max()
    info['brand_prince_median'] = kind_data.price.median()
    info['brand_price_min'] = kind_data.price.min()
    info['brand_price_sum'] = kind_data.price.sum()
    info['brand_price_std'] = kind_data.price.std()
    info['brand_price_average'] = round(kind_data.price.sum() / (len(kind_data) + 1), 2)
    all_info[kind] = info
brand_fe = pd.DataFrame(all_info).T.reset_index().rename(columns={"index":"brand"})
bin = [i*10 for i in range(31)]
data['power_bin'] = pd.cut(data['power'], bin, labels=False)
data[['power_bin', 'power']].head()
data = data.drop(['creatData','regData','regionCode'],axis=1)