spark机器学习构建回归模型

from pyspark import SparkContext


sc=SparkContext("local[4]","first spark app")
raw_data=sc.textFile("E:\sparkLearning\Bike-Sharing-Dataset\hour.csv")
records=raw_data.map(lambda x:x.split(","))
num_data=raw_data.count()
first=records.first()
print(first)
print(num_data)
records.cache()
def get_mapping(rdd,idx):
    return rdd.map(lambda fields:fields[idx]).distinct().zipWithIndex().collectAsMap()
print("Mapping of first categorical feature column: %s"%get_mapping(records,2))

mappings=[get_mapping(records,i) for i in range(2,10)]
cat_len=sum(map(len,mappings))
num_len=len(records.first()[11:15])
total_len=num_len+cat_len

print("Feature vector length for categorical features: %d"%cat_len)
print("Feature vector length for numerical features: %d"%num_len)
print("Total feature vector length: %d"%total_len)

from pyspark.mllib.regression import  LabeledPoint
import numpy as np
def extract_features(record):
    cat_vec=np.zeros(cat_len)
    i=0
    step=0
    for field in record[2:9]:
        m=mappings[i]
        idx=m[field]
        cat_vec[idx+step]=1
        i=i+1
        step=step+len(m)
    num_vec=np.array([float(field) for field in record[10:14]])
    return np.concatenate((cat_vec,num_vec))

def extract_label(record):
    return float(record[-1])

data=records.map(lambda r:LabeledPoint(extract_label(r),extract_features(r)))
first_point=data.first()
print("原始特征向量Raw data: "+str(first[2:]))
print("原始特征标签Label: "+str(first_point.label))
print("对特征进行热编码之后的特征向量Linear Model feature vector :\n"+str(first_point.features))
print("对特征进行热编码之后的特征向量的长度Linear Model feature vector length:"+str(len(first_point.features)))

def extract_features_dt(record):
    return np.array(record[2:14])

data_dt = records.map(lambda point: LabeledPoint(extract_label(point), extract_features_dt(point)))
first_point_dt=data_dt.first()
print("决策树特征向量：\n"+str(first_point_dt.features))
print("决策树特征向量长度:"+str(len(first_point_dt.features)))

from pyspark.mllib.regression import LinearRegressionWithSGD
from pyspark.mllib.tree import DecisionTree
liner_model=LinearRegressionWithSGD.train(data,iterations=10,step=0.1,intercept=False)
true_vs_predicted=data.map(lambda p:(p.label,liner_model.predict(p.features)))
print("线性回归对前五个样本的预测：\n"+str(true_vs_predicted.take(5)))

dt_model=DecisionTree.trainRegressor(data_dt,{})
preds=dt_model.predict(data_dt.map(lambda p:p.features))
actual=data.map(lambda p:p.label)
true_vs_predicted_dt=actual.zip(preds)
print(" 决策树预测值："+str(true_vs_predicted_dt.take(5)))
print("决策树深度："+str(dt_model.depth()))
print("决策树节点个数："+str(dt_model.numNodes()))

def squared_error(actual,pred):
    return (pred-actual)**2

def abs_error(actual,pred):
    return np.abs(pred-actual)

def squared_log_error(pred,actual):
    return (np.log(pred+1)-np.log(actual+1))**2


mse=true_vs_predicted.map(lambda point:squared_error(point[0],point[1])).mean()
mae=true_vs_predicted.map(lambda p:abs_error(p[0],p[1])).mean()
rmsle=np.sqrt(true_vs_predicted.map(lambda p:squared_log_error(p[0],p[1])).mean())
print("Linear Model - Mean Squared Error : %2.4f"%mse)
print("Linear Model - Mean Absolute Error : %2.4f"%mae)
print("Linear Model - Root Mean Squared Log Error : %2.4f"%rmsle)


mse_dt=true_vs_predicted_dt.map(lambda point:squared_error(point[0],point[1])).mean()
mae_dt=true_vs_predicted_dt.map(lambda p:abs_error(p[0],p[1])).mean()
rmsle_dt=np.sqrt(true_vs_predicted_dt.map(lambda p:squared_log_error(p[0],p[1])).mean())
print("Decision Tree - Mean Squared Error : %2.4f"%mse_dt)
print("Decision Tree - Mean Absolute Error : %2.4f"%mae_dt)
print("Decision Tree - Root Mean Squared Log Error : %2.4f"%rmsle_dt)

import matplotlib.pyplot as plt
# targets=records.map(lambda r:float(r[-1])).collect()
# plt.hist(targets,bins=40,color="lightblue",normed=True)
# fig=plt.gcf()
# fig.set_size_inches(16,10)
# plt.show()

# log_targets=records.map(lambda r: np.log(float(r[-1]))).collect()
# plt.hist(log_targets,bins=40,color="lightblue",normed=True)
# fig=plt.gcf()
# fig.set_size_inches(12,6)
# plt.show()

# sqrt_targets=records.map(lambda r: np.sqrt(float(r[-1]))).collect()
# plt.hist(sqrt_targets,bins=40,color="lightblue",normed=True)
# fig=plt.gcf()
# fig.set_size_inches(16,10)
# plt.show()

data_log=data.map(lambda lp:LabeledPoint(np.log(lp.label),lp.features))
model_log=LinearRegressionWithSGD.train(data_log,iterations=10,step=0.1)
r=true_vs_predicted_log=data_log.map(lambda p:(np.exp(p.label),np.exp(model_log.predict(p.features))))
mse_log=true_vs_predicted_log.map(lambda point:squared_error(point[0],point[1])).mean()
mae_log=true_vs_predicted_log.map(lambda p:abs_error(p[0],p[1])).mean()
rmsle_log=np.sqrt(true_vs_predicted_log.map(lambda p:squared_log_error(p[0],p[1])).mean())
print("Linear Model - Mean Squared Error : %2.4f"%mse_log)
print("Linear Model - Mean Absolute Error : %2.4f"%mae_log)
print("Linear Model - Root Mean Squared Log Error : %2.4f"%rmsle_log)
print("Non log-transfromed predictions:\n"+str(true_vs_predicted.take(3)))
print("Log_transformed predictions:\n"+str(true_vs_predicted_log.take(3)))


data_dt_log=data_dt.map(lambda lp:LabeledPoint(np.log(lp.label),lp.features))
dt_model_log=DecisionTree.trainRegressor(data_dt_log,{})
preds_log=dt_model_log.predict(data_dt_log.map(lambda p:p.features))
actual_log=data_dt_log.map(lambda p:p.label)
true_vs_predicted_dt_log=actual_log.zip(preds_log).map(lambda p:(np.exp(p[0]),np.exp(p[1])))
#均方误差
mse_log_dt=true_vs_predicted_dt_log.map(lambda point:squared_error(point[0],point[1])).mean()
#平均绝对误差
mae_log_dt=true_vs_predicted_dt_log.map(lambda p:abs_error(p[0],p[1])).mean()
#均方根对数误差
rmsle_log_dt=np.sqrt(true_vs_predicted_dt_log.map(lambda p:squared_log_error(p[0],p[1])).mean())

print("Decision Tree - Mean Squared Error : %2.4f"%mse_log_dt)
print("Decision Tree  - Mean Absolute Error : %2.4f"%mae_log_dt)
print("Decision Tree - Root Mean Squared Log Error : %2.4f"%rmsle_log_dt)
print("Non log-transfromed predictions:\n"+str(true_vs_predicted_dt.take(3)))
print("Log_transformed predictions:\n"+str(true_vs_predicted_dt_log.take(3)))

#划分测试集和训练集
data_with_idx=data.zipWithIndex().map(lambda k_v:(k_v[1],k_v[0]))
test=data_with_idx.sample(False,0.2,42)
train=data_with_idx.subtractByKey(test)

train_data=train.map(lambda idx_p:idx_p[1])
test_data=test.map(lambda idx_p:idx_p[1])
train_size=train_data.count()
test_size=test_data.count()
print("训练集大小：%d"%train_size)
print("测试集大小： %d"%test_size)
print("总数据大小: %d"%num_data)
print("训练集+测试集大小：%d"%(train_size+test_size))

data_with_idx_dt=data_dt.zipWithIndex().map(lambda k_v:(k_v[1],k_v[0]))
test_dt=data_with_idx_dt.sample(False,0.2,42)
train_dt=data_with_idx_dt.subtractByKey(test)

train_data_dt=train_dt.map(lambda idx_p:idx_p[1])
test_data_dt=test_dt.map(lambda idx_p:idx_p[1])

#评估参数

def evaluate(train,test,iterations,step,regParam,regType,intercept):
    model=LinearRegressionWithSGD.train(train,iterations,step,regParam=regParam,regType=regType,intercept=intercept)
    tp=test.map(lambda p:(p.label,model.predict(p.features)))
    rmsle=np.sqrt(tp.map(lambda p:squared_log_error(p[0],p[1])).mean())
    return rmsle
#迭代次数变化
params=[1,5,10,20,50,100,125,150,175,200]
metrics=[evaluate(train_data,test_data,param,0.01,0.0,'l2',False) for param in params]
print(params)
print(metrics)

# plt.plot(params,metrics)
# fig=plt.gcf()
# plt.xscale('log')
# plt.show()
#步长变化
params=[0.01,0.025,0.05,0.075,0.1,1.0]
metrics=[evaluate(train_data,test_data,10,param,0.0,'l2',False) for param in params]
print(params)
print(metrics)
# plt.plot(params,metrics)
# fig=plt.gcf()
# plt.xscale('log')
# plt.show()

#L2正则化
params=[0.0,0.01,0.1,1.0,5.0,10.0,20.0]
metrics=[evaluate(train_data,test_data,10,0.1,param,'l2',False) for param in params]
print(params)
print(metrics)
# plt.plot(params,metrics)
# fig=plt.gcf()
# plt.xscale('log')
# plt.show()

#L1正则化
params=[0.0,0.01,0.1,1.0,10.0,100.0,1000.0]
metrics=[evaluate(train_data,test_data,10,0.1,param,'l1',False) for param in params]
print(params)
print(metrics)
# plt.plot(params,metrics)
# fig=plt.gcf()
# plt.xscale('log')
# plt.show()

#截距
params=[False, True]
metrics=[evaluate(train_data,test_data,10,0.1,10,'l2',param) for param in params]
print(params)
print(metrics)
# plt.bar(params,metrics,color="lightblue")
# fig=plt.gcf()
# plt.show()


#决策树性能分析

def evaluate_dt(train,test,maxDepth,maxBins):
    model=DecisionTree.trainRegressor(train,{},impurity='variance',maxDepth=maxDepth,maxBins=maxBins)
    preds=model.predict(test.map(lambda p:p.features))
    actual=test.map(lambda p:p.label)
    tp=actual.zip(preds)
    rmsle=np.sqrt(tp.map(lambda p:squared_log_error(p[0],p[1])).mean())
    return  rmsle
#树深度
params=[1,2,3,4,5,10,20]
metrics=[evaluate_dt(train_data_dt,test_data_dt,param,32) for param in params]
print(params)
print(metrics)
# plt.plot(params,metrics)
# fig=plt.gcf()
# plt.show()

#最大划分树
params=[2,4,8,16,32,64,100]
metrics=[evaluate_dt(train_data_dt,test_data_dt,5,param) for param in params]
print(params)
print(metrics)
plt.plot(params,metrics)
fig=plt.gcf()
plt.show()