[校企合作]淡水的质量预测

sklearnex：借助面向 Scikit-learn* 的英特尔® 扩展，加速 Scikit-learn ，并且仍然完全符合所有 Scikit-Learn API 和算法。英特尔® Extension for Scikit-learn是一款免费软件 AI 加速器，可为各种应用程序带来超过 10-100 倍的加速。

1.4.3 daal4py组件

daal4py:可将训练好的xgboost转换为 daal4py 模型，以便进一步改进预测时间性能, 利用底层的英特尔® 高级矢量扩展指令集（英特尔® AVX-512）硬件，最大限度地提高英特尔® 至强® 处理器上的梯度提升性能。

2 数据集处理

2.1 导库导包

import os
import xgboost
from xgboost import XGBClassifier
import time
import warnings

import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import plotly.io as pio
import plotly.graph_objects as go
from sklearn.utils import resample

from sklearn.model_selection import train_test_split, StratifiedKFold, GridSearchCV, RandomizedSearchCV
from sklearn.preprocessing import RobustScaler
from sklearn.metrics import roc_auc_score, roc_curve, auc, accuracy_score, f1_score
from sklearn.preprocessing import StandardScaler
import sklearn
from sklearn.metrics import precision_recall_curve, average_precision_score
import daal4py as d4p

import modin.pandas as pd
from modin.config import Engine
Engine.put("dask")

from sklearnex import patch_sklearn

patch_sklearn()

2.2 读入数据集和显示

df = pd.read_csv('data/dataset.csv')
print("Data shape: {}\n".format(df.shape))
display(df.head())

df.info()

2.3 数据转化

为后续处理做准备，将离散数据转化为连续数据

display(df.head())
factor = pd.factorize(df['Color'])
print(factor)
df.Color = factor[0]
factor = pd.factorize(df['Source'])
print(factor)
df.Source = factor[0]
factor = pd.factorize(df['Month'])
print(factor)
df.Month = factor[0]

df.describe()

2.4 相关性分析及图像

import matplotlib.pyplot as plt
import numpy as np

# 获取相关系数并排序
bar = df.corr()['Target'].abs().sort_values(ascending=False)[1:]

# 设置图形大小
plt.figure(figsize=(15, 5))

# 绘制柱状图
plt.bar(bar.index, bar, width=0.4, color='skyblue')

# 设置 x 轴刻度标签的旋转角度和字体大小
plt.xticks(rotation=-60, fontsize=12)

# 设置 y 轴标签的字体大小
plt.yticks(fontsize=12)

# 设置 x 轴和 y 轴的标签
plt.xlabel('Features', fontsize=14)
plt.ylabel('Correlation', fontsize=14)

# 添加网格线
plt.grid(axis='y', linestyle='--', alpha=0.7)

# 显示图形
plt.show()

舍去相关性小的特性

2.5 缺失，重复，偏差处理

display(df.isna().sum())
missing = df.isna().sum().sum()
duplicates = df.duplicated().sum()
print("\nThere are {:,.0f} missing values in the data.".format(missing))
print("There are {:,.0f} duplicate records in the data.".format(duplicates))

做如下处理

df = df.fillna(df.interpolate(method='linear'))
df = df.drop_duplicates()
display(df.isna().sum())
missing = df.isna().sum().sum()
duplicates = df.duplicated().sum()
print("\nThere are {:,.0f} missing values in the data.".format(missing))
print("There are {:,.0f} duplicate records in the data.".format(duplicates))

检查皮尔逊相关系数与缺失值的比例和数值型特征的方差值来筛选符合要求的特征。

最后剩下15个特征

from scipy.stats import pearsonr

variables = df.columns
df = df

var = df.var()
numeric = df.columns
df = df.fillna(df.interpolate())
for i in range(0, len(var) - 1):
    if var[i] <= 0.1:  # 方差大于10％
        print(variables[i])
        df = df.drop(numeric[i],axis=1)
variables = df.columns

for i in range(0, len(variables)):
    x = df[variables[i]]
    y = df[variables[-1]]
    if pearsonr(x, y)[1] > 0.05:
        print(variables[i])
        df = df.drop(variables[i],axis=1)

variables = df.columns
print(variables)
print(len(variables))

2.6 检查数据平衡性

import matplotlib.pyplot as plt

# 获取目标列的值计数
target_counts = df.Target.value_counts()

# 重命名索引标签
target_counts.rename(index={1: 'state 1', 0: 'state 0'}, inplace=True)

# 绘制饼图
plt.figure(figsize=(8, 6))
plt.pie(target_counts, labels=target_counts.index, autopct='%1.1f%%', startangle=140, colors=['lightblue', 'lightgreen'])
plt.axis('equal')  # 使饼图呈正圆形
plt.title('Distribution of Target')
plt.show()

2.7 将数据进行正态分布变化

df.hist(bins=50,figsize=(16,12))
# 针对不规则分布的变量进行非线性变换，一般进行log
log_col = ['Iron', 'Zinc', 'Turbidity', 'Copper', 'Manganese']
show_col = []
for col in log_col:
    df[col + '_log'] = np.log(df[col])
    show_col.append(col + '_log')
    
df[show_col].hist(bins=50,figsize=(16,12))

3 模型训练

3.1 划分数据

X = df.drop( ['Target','Iron', 'Zinc', 'Turbidity', 'Copper', 'Manganese'], axis=1)
y = df['Target']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=21)
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)
print("Train Shape: {}".format(X_train_scaled.shape))
print("Test Shape: {}".format(X_test_scaled.shape))

X_train, X_test = X_train_scaled, X_test_scaled

3.2 模型参数定义

xgb = XGBClassifier(
    learning_rate=0.1,
    n_estimators=15,
    max_depth=12,
    min_child_weight=6,
    gamma=0,
    subsample=1,
    colsample_bytree=1,
    objective='binary:logistic', 
    nthread=4,
    alpha=4,
    scale_pos_weight=1,
    seed=27)
param_grid = {
    'max_depth': [10, 15, 20],
    "gamma": [0, 1, 2], 
    "subsample": [0.9, 1], 
    "colsample_bytree": [0.3, 0.5, 1], 
    'min_child_weight': [4, 6, 8], 
    "n_estimators": [10,50, 80, 100],
    "alpha": [3, 4, 5]
}

3.3 进行训练

训练同时打印相关数据,最优F1值

refit_score = "f1_score"

start_time = time.time()
print(start_time)
rd_search = RandomizedSearchCV(xgb, param_grid, n_iter=10, cv=3, refit=refit_score, scoring=sklearn.metrics.make_scorer(f1_score), verbose=10, return_train_score=True)
rd_search.fit(X_train, y_train)
print(rd_search.best_params_)
print(rd_search.best_score_)
print(rd_search.best_estimator_)
print(time.time() - start_time)

3.4 训练结果验证

将训练好的模型对统一测试集test_data.csv进行测试

test_data = pd.read_csv('data/test_data.csv')

test_data = test_data.drop(
    columns=['Index', 'Day', 'Time of Day', 'Month', 'Water Temperature', 'Source', 'Conductivity', 'Air Temperature'])
pd.factorize(test_data['Color'])
test_data.Color = factor[0]
test_data = test_data.fillna(test_data.interpolate(method='linear'))
test_data = test_data.drop_duplicates()

log_col = ['Iron', 'Zinc', 'Turbidity', 'Copper', 'Manganese']
show_col = []
for col in log_col:
    test_data[col + '_log'] = np.log(test_data[col])
    show_col.append(col + '_log')
test_data = test_data.drop( ['Iron', 'Zinc', 'Turbidity', 'Copper', 'Manganese'], axis=1)

y_true = test_data['Target']
X_test = test_data.drop(
    columns=['Lead', 'Target'])

scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)

from datetime import datetime
start = datetime.now()
y_pred = xgb.predict(X_test)
time = datetime.now() - start
f1 = f1_score(y_true, y_pred)


print('\n模型在测试集上的耗时: {}'.format(time))
print('模型在测试集上的f1分数: {}'.format(f1))

4 思考总结

在本次项目中，基于预测淡水质量这一实际问题，我结合了以往在机器学习和深度学习的知识，运用oneAPI组件来提升项目的运行速度和结果可靠度，自己也在这一方面有了更多的理解，尤其是daal4py的加速效率极高，使得项目模型训练速度大幅度提升，让我受益匪浅。

Envyes

关注

20
点赞
踩
25

收藏

觉得还不错? 一键收藏
1
评论
[校企合作]淡水的质量预测

gamma=0,nthread=4,alpha=4,seed=27)在本次项目中，基于预测淡水质量这一实际问题，我结合了以往在机器学习和深度学习的知识，运用oneAPI组件来提升项目的运行速度和结果可靠度，自己也在这一方面有了更多的理解，尤其是daal4py的加速效率极高，使得项目模型训练速度大幅度提升，让我受益匪浅。
复制链接

扫一扫