17IOT 房价回归分析

import pandas as pd

import numpy as np

import seaborn as sns

import statsmodels.api as sm

import matplotlib.pyplot as plt

import scipy.stats as sci

from sklearn.model_selection import train_test_split

from statsmodels.formula.api import ols

from statsmodels.sandbox.regression.predstd import wls_prediction_std

import statsmodels.formula.api as smf

from sklearn.preprocessing import OneHotEncoder,LabelEncoder

#copyright by Interstellar-Ark-AI on 2020_5_23


    

#读取数据集-查看数据形状和类型以及分布
    
data = pd.read_csv("house_train.csv")

data.head() #查看数据

data.shape #查看数据集形状

print(data['price'].describe()) #查看价值目标类型和分布、有无异常值

sns.distplot(data['price']) # 查看价格分布的散点图

plt.show()


#预处理和数据清洗，去除价格-面积的离群点

sns.jointplot(x='area',y='price',data=data) #查看价格和面积的分布图

plt.show()

index_del = data[(data['area'] > 100) & (data['price'] < 700)].index

data.drop(index=index_del, inplace=True)    #噪声去除

sns.jointplot(x='area',y='price',data=data) #再次查看价格和面积的分布图

plt.show()


#预处理和数据清洗，去除价格-PM2.5的离群点

sns.jointplot(x='pm25',y='price',data=data) #查看价格和PM2.5的分布图

plt.show()

index_del = data[(data['pm25'] > 80) & (data['price'] < 400)].index

data.drop(index=index_del, inplace=True)    #噪声去除

sns.jointplot(x='pm25',y='price',data=data) #再次查看价格和PM2.5的分布图

plt.show()


#预处理和数据清洗，去除犯罪率-价格的离群点

sns.jointplot(x='crime_rate',y='price',data=data) #查看犯罪率和价格的分布图

plt.show()

index_del = data[(data['crime_rate'] > 7.5) & (data['price'] < 500)].index

data.drop(index=index_del, inplace=True)    #噪声去除

sns.jointplot(x='crime_rate',y='price',data=data) #再次查看犯罪率和价格的分布图

plt.show()


#进行预处理数据清洗，查看什么特征拥有空值,去除空值列

total = data.isnull().sum().sort_values(ascending=False)

percent = (data.isnull().sum() / data.isnull().count()).sort_values(ascending = False)

missing_data = pd.concat([total,percent],axis = 1,keys = ['Total','Percent'])

data['distirct'] = data['distirct'].astype(str) #对于distirct数据，需要转化为object类型进行分类

print(missing_data.head(13)) #查看空值列

data1 = data.drop(['id'], axis = 1)  #删除id列，id列是无效的数据

data2 = data1.drop(data1.loc[data1['crime_rate'].isnull()].index) #删除犯罪率为空的行

data3 = data2.drop(data2.loc[data2['green_rate'].isnull()].index) #删除绿化率为空的行

print(data3.isnull().sum().max()) #检查最后数据

#可以使用热力图表示变量相关性

plt.rcParams['figure.figsize'] = (15, 10)# 计算相关系数

corrmatrix = data3.corr()

sns.heatmap(corrmatrix,square = True,vmax = 1,vmin = -1,center = 0.0,cmap = 'coolwarm')

plt.show()


#特征工程：求出各个特征之间的皮尔逊相关系数

k = 10  #十个特征

cols = corrmatrix.nlargest(k,'price')['price'].index

cm = np.corrcoef(data3[cols].values.T)

sns.set(font_scale = 1.25)

hm = sns.heatmap(cm,cmap = 'RdPu',annot = True,square = True,fmt = '.2f',annot_kws = {'size':10},yticklabels = cols.values,xticklabels = cols.values)

plt.show()


#算法1:ols回归分析数据建模

feature_data = data3.drop(['price'],axis = 1)

target_data = data3['price'] #预测目标列为price

X_train,X_test,y_train, y_test = train_test_split(feature_data, target_data, test_size = 0.3) #划分数据集和测试集进行交叉验证

df_train = pd.concat([X_train,y_train],axis=1)

print(data['distirct'].describe())

lr_model = ols("price~area+C(floor)+C(oriented)+crime_rate+pm25+C(distirct)",data = df_train).fit()