三. 特征工程

三.特征工程

本期特征提取接上一期数据清洗内容

test = pd.read_csv('C:/Users/86061/PycharmProjects/untitled/1/real.csv')
target_train = data_train.pop('tradeMoney')     #弹出的仍然为series数组，弹出的是经过数据处理后的训练集的交易金额
target_test = test.pop('pre')               #表示的是测试集里的待预测数据集的真实交易金额

def newfeature(data):
    # 将houseType转为'Room'，'Hall'，'Bath'
    def Room(x):
        Room = int(x.split('室')[0])
        return Room

    def Hall(x):
        Hall = int(x.split("室")[1].split("厅")[0])
        return Hall

    def Bath(x):
        Bath = int(x.split("室")[1].split("厅")[1].split("卫")[0])
        return Bath

    data['Room'] = data['houseType'].apply(lambda x: Room(x))
    data['Hall'] = data['houseType'].apply(lambda x: Hall(x))
    data['Bath'] = data['houseType'].apply(lambda x: Bath(x))
    data['Room_Bath'] = (data['Bath'] + 1) / (data['Room'] + 1)
    # 填充租房类型,将未知类型全部转化为已知类型
    data.loc[(data['rentType'] == '未知方式') & (data['Room'] <= 1), 'rentType'] = '整租'
    # print(data.loc[(data['rentType']=='未知方式')&(data['Room_Bath']>1),'rentType'])
    data.loc[(data['rentType'] == '未知方式') & (data['Room_Bath'] > 1), 'rentType'] = '合租'
    data.loc[(data['rentType'] == '未知方式') & (data['Room'] > 1) & (data['area'] < 50), 'rentType'] = '合租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] / data['Room'] < 20), 'rentType'] = '合租'
    # data.loc[(data['rentType']=='未知方式')&(data['area']>60),'rentType']='合租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 50) & (data['Room'] == 2), 'rentType'] = '合租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 2), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] <= 60) & (data['Room'] == 3), 'rentType'] = '合租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] > 60) & (data['Room'] == 3), 'rentType'] = '整租'
    data.loc[(data['rentType'] == '未知方式') & (data['area'] >= 100) & (data['Room'] > 3), 'rentType'] = '整租'

    # data.drop('Room_Bath', axis=1, inplace=True)
    # 提升0.0001
    #     data['pv/uv'] = data['pv'] / data['uv']
    #     data['房间总数'] = data['室'] + data['厅'] + data['卫']

    # 合并部分配套设施特征，自己为各种资源赋予权重
    data['trainsportNum'] = 5 * data['subwayStationNum'] / data['subwayStationNum'].mean() + data['busStationNum'] / \
                            data[
                                'busStationNum'].mean()
    data['all_SchoolNum'] = 2 * data['interSchoolNum'] / data['interSchoolNum'].mean() + data['schoolNum'] / data[
        'schoolNum'].mean() \
                            + data['privateSchoolNum'] / data['privateSchoolNum'].mean()
    data['all_hospitalNum'] = 2 * data['hospitalNum'] / data['hospitalNum'].mean() + \
                              data['drugStoreNum'] / data['drugStoreNum'].mean()
    data['all_mall'] = data['mallNum'] / data['mallNum'].mean() + \
                       data['superMarketNum'] / data['superMarketNum'].mean()
    data['otherNum'] = data['gymNum'] / data['gymNum'].mean() + data['bankNum'] / data['bankNum'].mean() + \
                       data['shopNum'] / data['shopNum'].mean() + 2 * data['parkNum'] / data['parkNum'].mean()

    data.drop(['subwayStationNum', 'busStationNum',
               'interSchoolNum', 'schoolNum', 'privateSchoolNum',
               'hospitalNum', 'drugStoreNum', 'mallNum', 'superMarketNum', 'gymNum', 'bankNum', 'shopNum', 'parkNum'],
              axis=1, inplace=True)
    # 提升0.0005
    #     data['houseType_1sumcsu']=data['Bath'].map(lambda x:str(x))+data['month'].map(lambda x:str(x))
    #     data['houseType_2sumcsu']=data['Bath'].map(lambda x:str(x))+data['communityName']
    #     data['houseType_3sumcsu']=data['Bath'].map(lambda x:str(x))+data['plate']
    data.drop('houseType', axis=1, inplace=True)
    data["area"] = data["area"].astype(int)
    return data, data['trainsportNum']


data_all = newfeature(data_all)

# 计算统计特征
def featureCount(data_all):

    def feature_count(data_all, features=[]):
        new_feature = 'count'
        for i in features:
            new_feature += '_' + i
        temp = data_all.groupby(features).size().reset_index().rename(columns={0: new_feature})
        data_all = data_all.merge(temp, 'left', on=features)
        return data_all

    data_all = feature_count(data_all, ['communityName'])
    data_all = feature_count(data_all, ['buildYear'])
    data_all = feature_count(data_all, ['totalFloor'])
    data_all = feature_count(data_all, ['communityName', 'totalFloor'])
    data_all = feature_count(data_all, ['communityName', 'newWorkers'])
    data_all= feature_count(data_all, ['communityName', 'totalTradeMoney'])

    return  data_all

data_all = featureCount(data_all)

添加新的统计特征，譬如所属小区有多少个租房记录，属于某个修建时间的租房有多少个等等

def gourpby(data_all):

    data_all['region'] = LabelEncoder().fit_transform(data_all['region'])

    temp = data_all.groupby('communityName')['area'].agg({'com_area_mean': 'mean', 'com_area_std': 'std'})
    temp.fillna(0, inplace=True)
    data_all = data_all.merge(temp, on='communityName', how='left')

    data_all['price_per_area'] = data_all.tradeMeanPrice / data_all.area * 100
    temp = data_all.groupby('communityName')['price_per_area'].agg(
        {'comm_price_mean': 'mean', 'comm_price_std': 'std'})
    temp.fillna(0, inplace=True)
    data_all = data_all.merge(temp, on='communityName', how='left')

    temp = data_all.groupby('plate')['price_per_area'].agg(
        {'plate_price_mean': 'mean', 'plate_price_std': 'std'})
    temp.fillna(0, inplace=True)
    data_all = data_all.merge(temp, on='plate', how='left')
    data_all.drop('price_per_area', axis=1, inplace=True)

    temp = data_all.groupby('plate')['area'].agg({'plate_area_mean': 'mean', 'plate_area_std': 'std'})
    temp.fillna(0, inplace=True)
    data_all = data_all.merge(temp, on='plate', how='left')

    temp = data_all.groupby(['plate'])['buildYear'].agg({'plate_year_mean': 'mean', 'plate_year_std': 'std'})
    data_all = data_all.merge(temp, on='plate', how='left')
    data_all.plate_year_mean = data_all.plate_year_mean.astype('int')
    data_all['comm_plate_year_diff'] = data_all.buildYear - data_all.plate_year_mean
    data_all.drop('plate_year_mean', axis=1, inplace=True)

    temp = data_all.groupby('plate')['trainsportNum'].agg('sum').reset_index(name='plate_trainsportNum')
    data_all = data_all.merge(temp, on='plate', how='left')
    temp = data_all.groupby(['communityName', 'plate'])['trainsportNum'].agg('sum').reset_index(name='com_trainsportNum')
    data_all = data_all.merge(temp, on=['communityName', 'plate'], how='left')
    data_all['trainsportNum_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,
                                           data_all['com_trainsportNum'], data_all['plate_trainsportNum']))
    data_all = data_all.drop(['com_trainsportNum', 'plate_trainsportNum'], axis=1)

    temp = data_all.groupby('plate')['all_SchoolNum'].agg('sum').reset_index(name='plate_all_SchoolNum')
    data_all = data_all.merge(temp, on='plate', how='left')
    temp = data_all.groupby(['communityName', 'plate'])['all_SchoolNum'].agg('sum').reset_index(name='com_all_SchoolNum')
    data_all = data_all.merge(temp, on=['communityName', 'plate'], how='left')
    data_all = data_all.drop(['com_all_SchoolNum', 'plate_all_SchoolNum'], axis=1)

    temp = data_all.groupby(['communityName', 'plate'])['all_mall'].agg('sum').reset_index(name='com_all_mall')
    data_all = data_all.merge(temp, on=['communityName', 'plate'], how='left')

    temp = data_all.groupby('plate')['otherNum'].agg('sum').reset_index(name='plate_otherNum')
    data_all = data_all.merge(temp, on='plate', how='left')
    temp = data_all.groupby(['communityName', 'plate'])['otherNum'].agg('sum').reset_index(name='com_otherNum')
    data_all = data_all.merge(temp, on=['communityName', 'plate'], how='left')
    data_all['other_ratio'] = list(map(lambda x, y: round(x / y, 3) if y != 0 else -1,
                                    data_all['com_otherNum'], data_all['plate_otherNum']))
    data_all = data_all.drop(['com_otherNum', 'plate_otherNum'], axis=1)

    temp = data_all.groupby(['month', 'communityName']).size().reset_index(name='communityName_saleNum')
    data_all = data_all.merge(temp, on=['month', 'communityName'], how='left')
    temp = data_all.groupby(['month', 'plate']).size().reset_index(name='plate_saleNum')
    data_all = data_all.merge(temp, on=['month', 'plate'], how='left')

    data_all['sale_ratio'] = round((data_all.communityName_saleNum + 1) / (data_all.plate_saleNum + 1), 3)
    data_all['sale_newworker_differ'] = 3 * data_all.plate_saleNum - data_all.newWorkers
    data_all.drop(['communityName_saleNum', 'plate_saleNum'], axis=1, inplace=True)
    return data_all


data_all = gourpby(data_all)

增加各种属性，都可能对预测产生影响

#聚类
def cluster(data_all):
    from sklearn.mixture import GaussianMixture

    col = ['totalFloor',
           'houseDecoration', 'communityName', 'region', 'plate', 'buildYear',

           'tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',
           'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',

           'landTotalPrice', 'landMeanPrice', 'totalWorkers',
           'newWorkers', 'residentPopulation', 'lookNum',
           'trainsportNum',
           'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']

    # EM
    gmm = GaussianMixture(n_components=3, covariance_type='full', random_state=0)
    data_all['cluster'] = pd.DataFrame(gmm.fit_predict(data_all[col]))        #高斯混合，通过多个参数将数据集最终分为几类

    col1 = ['totalFloor', 'houseDecoration', 'communityName', 'region', 'plate', 'buildYear']
    col2 = ['tradeMeanPrice', 'tradeSecNum', 'totalNewTradeMoney',
            'totalNewTradeArea', 'tradeNewMeanPrice', 'tradeNewNum', 'remainNewNum',
            'landTotalPrice', 'landMeanPrice', 'totalWorkers',
            'newWorkers', 'residentPopulation', 'lookNum',
            'trainsportNum',
            'all_SchoolNum', 'all_hospitalNum', 'all_mall', 'otherNum']
    for feature1 in col1:
        for feature2 in col2:
            temp = data_all.groupby(['cluster', feature1])[feature2].agg('mean').reset_index(
                name=feature2 + '_' + feature1 + '_cluster_mean')
            temp.fillna(0, inplace=True)

            data_all = data_all.merge(temp, on=['cluster', feature1], how='left')

    return data_all


data_all = cluster(data_all)

本例中通过 GassianMixture()方法最终分为三类，通过分类后，添加关于各种类的有关新的特征数据

#结果评估
#相关系数法特征选择
new_train = data_all[data_all['Type'] == 'Train']      #去除一些数据后的训练集，测试集不变
new_train ['Type'] = 0
Test = data_all[data_all['Type'] == 'Test']
Test ['Type'] = 1
sk=SelectKBest(k=150)       #default=10，所选择的topK个特征
new_train=sk.fit_transform(new_train,target_train)        #拟合数据并生成新的数据
print(new_train.shape)

# 获取对应列索引
select_columns=sk.get_support(indices = True)
# print(select_columns)

#获取对应列名

select_columns_name=Test.columns[select_columns]
print(Test.columns[select_columns])
new_test=Test[select_columns_name]
print(new_test.shape)
# Lasso回归
from sklearn.linear_model import Lasso

lasso=Lasso(alpha=0.1)
lasso.fit(new_train,target_train)
#预测测试集和训练集结果
y_pred_train=lasso.predict(new_train)
print(y_pred_train)
alldata_na = pd.DataFrame(new_test.isnull().sum(), columns={'missingNum'})
print(alldata_na)
new_test['tradeMoney'] = 0      #两个集合合并，缺失的数据由NAN来填充，需要设置为零
y_pred_test=lasso.predict(new_test)

#对比结果
from sklearn.metrics import r2_score
score_train=r2_score(y_pred_train,target_train)
print("训练集结果：",score_train)
score_test=r2_score(y_pred_test, target_test)
print("测试集结果：",score_test)

特征评估时确保每一个列均为数值，不能为字符型，有点懵，出现了负数？？