项目——数据预处理2+xgboost+knn

本文链接：https://blog.csdn.net/qq_31150399/article/details/72831824

本文介绍了如何在2017年的项目中进行数据预处理，然后结合XGBoost和KNN算法进行预测分析。通过数据清洗、特征工程和模型融合，提升了预测效果。

摘要由CSDN通过智能技术生成

20170410

#!/usr/bin/env python
# -*- coding: utf-8 -*-

import numpy as np
import pandas as pd
import seaborn as sns
sns.set_style('whitegrid')
import matplotlib.pyplot as plt
from scipy.cluster.hierarchy import linkage,fclusterdata
from scipy.sparse import coo_matrix, bmat
from sklearn import preprocessing

PLOT = False  #是否画图
N = 1    #输出结果个数

data = pd.read_csv(r'E:\7 Python\data\t2.csv')
print data.head()
print data.info()
print data.columns

data.columns = ['ID', 'FAULT_TYPE_3', 'work_months', 'save_months',
       'FAULT_MONTH', 'INST_MONTH', 'PS_MONTH', 'SYNC_ORG_NO',
       'SPEC_CODE', 'COMM_MODE', 'ARRIVE_BATCH_NO', 'MANUFACTURER',
       'TYPE_CODE', 'EXCHG_TYPE_CODE', 'LC_FLAG', 'TL_SHARE_FLAG',
       'MP_CAP', 'TRADE_CODE', 'ELEC_TYPE_CODE', 'RUN_CAP',
       'CUST_STATUS_CODE', 'TRANSFER_CODE', 'TMP_FLAG']
#故障作图
print data['FAULT_TYPE_3'].value_counts()
fig, axis0 = plt.subplots(1, 1)
sns.countplot(x='FAULT_TYPE_3', data=data, ax=axis0)
# plt.show()

# SYNC_ORG_NO
print data['SYNC_ORG_NO'].describe()
# plot
def plot_fun(name_fea, name_fault, figsize=None, fontsize=None):
    plt.figure(figsize=figsize)
    fig, axis1 = plt.subplots(1, 1)
    sns.countplot(x=name_fea, data=data, ax=axis1)

    plt.figure(figsize=figsize)
    fig, axis2 = plt.subplots(1, 1)
    c = data[name_fea].value_counts()
    s = c.cumsum() / c.sum()
    axis2.plot(np.arange(s.shape[0]) + 1, s.values * 100)
    axis2.set_title('precent of %s' % name_fea)

    plt.figure(figsize=figsize)
    fig, axis3 = plt.subplots(1, 1)
    sns.countplot(x=name_fea, hue=name_fault, data=data, ax=axis3)
    plt.legend(loc=2)
    plt.figure(figsize=figsize)
    fig, axis4 = plt.subplots(1, 1)
    sns.countplot(x=name_fault, hue=name_fea, data=data, ax=axis4)
    plt.legend(loc=2, fontsize=fontsize)
    # calculate similar score
    from scipy.cluster.hierarchy import dendrogram, linkage
    # clustermap
    fault_num1 = data.groupby([name_fault, name_fea])[data.columns[0]].count().unstack()
    ratio = fault_num1 / fault_num1.sum()
    g1 = sns.clustermap(ratio,
                        cmap=plt.get_cmap('RdBu'),
                        vmax=1,
                        vmin=-1,
                        linewidth=0,
                        figsize=(10, 10),
                        row_cluster=False,
                        col_cluster=False
                        )
    plt.title('fault ratio')
# 聚类函数
def cluster_encoding(name):
    global data
    fault_num = data.groupby(['FAULT_TYPE_3', name])[data.columns[0]].count().unstack()
    MAN_ratio = fault_num / fault_num.sum()
    MAN_ratio_T = MAN_ratio.T

    clusters = fclusterdata(np.array(MAN_ratio_T), 1)
    print clusters.shape
    clusters_mapping = {label: idx for label, idx in zip(MAN_ratio.columns, clusters)}
    data[name] = data[name].map(clusters_mapping)
# 编码函数
def onehot_pre(name):
    global data
    le = preprocessing.LabelEncoder()
    le.fit(data[name])
    cat_name = list(le.classes_)
    data[name] = le.transform(data[name])
    return cat_name
# plot
if PLOT:
    plot_fun('SYNC_ORG_NO', 'FAULT_TY