pyspark 实现随机森林并且连接端口

m0_50959639

于 2022-01-08 14:02:33 发布

阅读量1.1k

点赞数

文章标签：随机森林算法机器学习

本文链接：https://blog.csdn.net/m0_50959639/article/details/122379649

版权

该博客介绍了如何利用PySpark构建随机森林信用风险分类模型，并通过创建一个服务器，实现实时接收外部请求进行预测。首先，加载数据，预处理缺失值，然后使用VectorAssembler组合特征，训练RandomForestClassifier模型。接着，定义了一个预测函数rfpredict，用于对外部输入进行预测。最后，启动一个服务器，监听特定端口，接收预测请求，返回预测概率。

摘要由CSDN通过智能技术生成

模型建立

import json
import socket
import findspark
findspark.init('/usr/local/spark') #之前找到的spark存放路径
from pyspark import SparkConf, SparkContext
from pyspark.sql import SparkSession
from pyspark import SparkContext,SparkConf
from pyspark.sql import Row
from pyspark.sql.types import *
from pyspark.sql import functions
from pyspark.sql.functions import avg
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.feature import VectorAssembler
from pyspark.ml.feature import StringIndexer
import pandas as pd

def fill_with_mean(df, exclude=set()):
stats = df.agg(*(
avg(c).alias(c) for c in df.columns if c not in exclude
))
return df.na.fill(stats.first().asDict())

filename = "input/data.csv"
spark = SparkSession.builder.config(conf = SparkConf()).getOrCreate()
df = spark.read.format("csv").option("header","true").load(filename)
for c in df.columns:
df = df.withColumn(c,df[c].cast('Double'))
df = df.drop("_c0")
df = df.drop_duplicates()
df = fill_with_mean(df, ["SeriousDlqin2yrs"])

dfInfoAssembler = VectorAssembler(inputCols=['RevolvingUtilizationOfUnsecuredLines','age','NumberOfTime30-59DaysPastDueNotWorse','DebtRatio','MonthlyIncome','NumberOfOpenCreditLinesAndLoans','NumberOfTimes90DaysLate','NumberRealEstateLoansOrLines','NumberOfTime60-89DaysPastDueNotWorse','NumberOfDependents'],outputCol='features')
df = dfInfoAssembler.transform(df)
df = df.select(['features', 'SeriousDlqin2yrs'])
training,test = df.randomSplit([0.75,0.25])
rfModel = RandomForestClassifier(labelCol='SeriousDlqin2yrs').fit(training)

def rfpredict(x):
x = [tuple(x)]
input = spark.createDataFrame(x,['RevolvingUtilizationOfUnsecuredLines','age','NumberOfTime30-59DaysPastDueNotWorse','DebtRatio','MonthlyIncome','NumberOfOpenCreditLinesAndLoans','NumberOfTimes90DaysLate','NumberRealEstateLoansOrLines','NumberOfTime60-89DaysPastDueNotWorse','NumberOfDependents']).collect()
input = dfInfoAssembler.transform(input)
res = rfModel.transform(input)
return res.select('probability').limit(1).collect()[0][0]

连接外部端口

import json
import socket
class Server:
def __init__(self):
self.buffer_size = 1024
self.server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.server.bind(('192.168.19.135', 11223)) # 绑定端口

def predict(self, client): # 拿模型做预测
recvData = client.recv(self.buffer_size).decode()
print(recvData)
X = json.loads(recvData, encoding='utf-8')

pred = rfpredict(X)[0]
pred = round(pred,2)

sentData = json.dumps(pred, ensure_ascii=False)
client.send(sentData.encode())
print('已发送预测结果：' + str(sentData))

def start(self):
print('启动预测服务器')
self.server.listen(1) # 监听
try:
while True:
client, addr = self.server.accept() # 等待客户端连接
self.predict(client)
client.close()
except BaseException as e:
print(repr(e))
finally:
print("关闭预测服务器")
self.server.close() # 关闭连接

if __name__ == '__main__':
server = Server()
server.start()