通过sklearn和pyspark机器学习模型训练房价预测数据集的计算时间比较

运行环境

CPU：i5-8265U 4核8线
内存：8G
操作系统：Windows 10
其他：Spark采用local[*]模式运行

数据集

kc_house_data.csv，大小为2.12MB，通过训练bedrooms, bathrooms, sqft_living, sqft_lot, floors, waterfront, view, condition, grade, sqft_above, sqft_basement, yr_built, yr_renovated, zipcode, lat, long, sqft_living15, sqft_lot15这几项来预测price这一项

运行结果

pyspark
读取文件时间3.3s
数据标准化时间170s
随机梯度下降线性回归计算时间8.6s
随机梯度下降线性回归模型评分：-1.4012390910433514
随机森林计算时间37s
随机森林回归模型评分：0.7020957800669471
决策树计算时间29s
决策树回归模型评分：0.7045587731756398
GBDT计算时间73s
GBDT回归模型评分：0.8667308339317286
平均绝对误差：79524.48389462061
均方根误差：18680226496.846706
模型评估时间3.3s

sklearn
读取文件时间0.052s
数据标准化时间0.01s
线性回归模型评分：0.7013292672895156
线性回归计算时间0.014s
随机森林回归模型评分：0.8815614718912635
随机森林计算时间19s
决策树回归模型评分：0.7104037868879087
决策树计算时间0.29s
GBDT回归模型评分：0.8710643694641974
GBDT计算时间3.4s
GBDT回归模型评估值：0.8710643694641974
平均绝对误差：78124.26028672243
均方根误差：16761398066.75135
模型评估时间0.004s

源代码

pyspark

from pyspark import SparkConf
from pyspark.sql import SparkSession
from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD
from pyspark.mllib.tree import DecisionTree, RandomForest, GradientBoostedTrees
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.feature import StandardScaler
from pyspark.mllib.evaluation import RegressionMetrics
from time import time

cfg = SparkConf()
cfg.setAppName('HousePriceML').setMaster('local[*]')
spark = SparkSession.builder.config(conf=cfg).getOrCreate()
sc = spark.sparkContext

read = time()
rawDf = spark.read.options(header='true', inferschema='true') \
    .csv(r'C:\Users\ASUS.LAPTOP-5O0LKO7M\Downloads\kc_house_data\kc_house_data.csv') #原始数据
print('读取文件时间%.2gs'%(time() - read))

rdd = rawDf.drop('id', 'date').rdd.map(lambda row: LabeledPoint(row[0], Vectors.dense(row[1:])))
train, test = rdd.randomSplit([0.7, 0.3], 10)
std = time()
scaler = StandardScaler(True).fit(train.map(lambda point: point.features))
#sdTrain = train.map(lambda point: LabeledPoint(point.label, scaler.transform(point.features))) \\报错：无法序列化
sdTrain = sc.parallelize(list(map(lambda point: LabeledPoint(point.label, scaler.transform(point.features)),
                                  train.collect()))).cache()
sdTest = sc.parallelize(list(map(lambda point: LabeledPoint(point.label, scaler.transform(point.features)),
                                  test.collect()))).cache()
print('数据标准化时间%.2gs'%(time() - std))
lr = time()
model = LinearRegressionWithSGD.train(sdTrain)
predsAndValues = sdTest.map(lambda point: (float(model.predict(point.features)), point.label)).cache()
print('线性回归计算时间%.2gs'%(time() - lr))
metrics = RegressionMetrics(predsAndValues)
print('随机梯度下降线性回归模型评分：%s'%metrics.r2)

rf = time()
model = RandomForest.trainRegressor(sdTrain, {}, 500)
#predsAndValues = sdTest.map(lambda point: (float(model.predict(point.features)), point.label)) \\报错：无法序列化
predsAndValues = sc.parallelize(list(map(lambda point: (model.predict(point.features), point.label), sdTest.collect()))).cache()
print('随机森林计算时间%.2gs'%(time() - rf))
metrics = RegressionMetrics(predsAndValues)
print('随机森林回归模型评分：%s'%metrics.r2)

dt = time()
model = DecisionTree.trainRegressor(sdTrain, {})
#predsAndValues = sdTest.map(lambda point: (float(model.predict(point.features)), point.label)) \\报错：无法序列化
predsAndValues = sc.parallelize(list(map(lambda point: (model.predict(point.features), point.label), sdTest.collect()))).cache()
print('决策树计算时间%.2gs'%(time() - dt))
metrics = RegressionMetrics(predsAndValues)
print('决策树回归模型评分：%s'%metrics.r2)

gbdt = time()
model = GradientBoostedTrees.trainRegressor(sdTrain, {})
#predsAndValues = sdTest.map(lambda point: (float(model.predict(point.features)), point.label)) \\报错：无法序列化
predsAndValues = sc.parallelize(list(map(lambda point: (model.predict(point.features), point.label), sdTest.collect()))).cache()
print('GBDT计算时间%.2gs'%(time() - gbdt))
score = time()
metrics = RegressionMetrics(predsAndValues)
print('GBDT回归模型评分：%s'%metrics.r2)
print('平均绝对误差：%s'%metrics.meanAbsoluteError)
print('均方根误差：%s'%metrics.meanSquaredError)
print('模型评估时间%.2gs'%(time() - score))

sklearn

import numpy as np
import pandas as pd
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score
from time import time

pd.set_option('display.max_columns', 20)
pd.set_option('display.max_rows', 20)
read = time()
df = pd.read_csv(r'C:\Users\ASUS.LAPTOP-5O0LKO7M\Downloads\kc_house_data\kc_house_data.csv')
print('读取文件时间%.2gs'%(time() - read))
df.drop(['id'], axis=1, inplace=True)
x = df.as_matrix(df.columns.drop(['price','date']))
y = df['price']
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.3, random_state=10)
std = time()
scaler = StandardScaler()
scaler.fit(x_train)
x_train = scaler.transform(x_train)
x_test = scaler.transform(x_test)
print('数据标准化时间%.2gs'%(time() - std))
lr = time()
model = LinearRegression(n_jobs=-1)
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
print('线性回归模型评分：%s'%model.score(x_test, y_test))
print('线性回归计算时间%.2gs'%(time() - lr))
rf = time()
model = RandomForestRegressor(500,n_jobs=-1)
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
print('随机森林回归模型评分：%s'%model.score(x_test, y_test))
print('随机森林计算时间%.2gs'%(time() - rf))
dt = time()
model = DecisionTreeRegressor()
model.fit(x_train, y_train)
y_pred = model.predict(x_test)
print('决策树回归模型评分：%s'%model.score(x_test, y_test))
print('决策树计算时间%.2gs'%(time() - dt))
gbdt = time()
model = GradientBoostingRegressor(n_estimators=100)
model.fit(x_train, y_train)
y_predicted = model.predict(x_test)
print('GBDT回归模型评分：%s'%model.score(x_test, y_test))
print('GBDT计算时间%.2gs'%(time() - gbdt))
score = time()
print('GBDT回归模型评估值：%s'%r2_score(y_test, y_predicted))
print('平均绝对误差：%s'%mean_absolute_error(y_test, y_predicted))
print('均方根误差：%s'%mean_squared_error(y_test, y_predicted))
print('模型评估时间%.2gs'%(time() - score))

总结

pyspark的计算时间比sklearn长很多，这应该有部分原因是因为有些模型无法序列化，从而无法在RDD的map算子中进行计算，我通过把训练集转换成python列表再用机器学习模型进行计算，然后再转换回RDD解决，但这样就是用python来处理了，效率降低。但是没有这样做的随机梯度下降线性回归模型同样计算时间长。应该还有些原因是Driver与Executor之间通信太多，留待以后寻求解决方法。

参考资料

华为《HCIE-Big_Data-Data_Mining_V2.0_实验手册》