关于sklearn算法学习的一部分总结

前言

关于一部分算法实践的代码整理

关于算法实践的几个步骤

###  载入python相关的包
```python
# 基础
import pandas as pd
import numpy as np
from collections import defaultdict
import matplotlib.pyplot as plt
import seaborn as sns
import gc

# import sklearn
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import StandardScaler
 
from sklearn.model_selection import train_test_split
# X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size = 0.25, random_state = 0)


from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error as mse
from lightgbm.sklearn import LGBMRegressor

# import boosting algorithm
import lightgbm as lgb
import xgboost as xgb
import catboost as ctb

载入数据

df=pd.read_csv(filepath)
print(df.info())

数据清洗

0、数据描述

trainset.describe().apply(lambda x:round(x,2) if isinstance(x,'int') else x)

1、查看特征空值情况

df_nulldata='\n'.join(['{} feature have: {} null data'.format(col,df[col].isnull().sum()) for col in df.columns])
print(df_nulldata)

2、查看特征构成

df_sturcture='\n'.join(['{} feature hvae {} data'.format(col,df[col].nunique()) for col in df.columns])
print(df_sturcture)

筛选特征的时候，剔除部分特征只有单一值的特征

3、缺省值填充

# 插入固定值
df['Embarked']=df['Embarked'].fillna('S')
# 插入中位数
df['Fare']=df['Fare'].fillna(df['Fare'].median())
# 插入区间内的随机数
age_avg=df['age'].mean()
age_std=df['age'].std()
age_null_count=df['age'].isnull().sum()
age_null_random_list=np.random.randint(age_avg-age_std,age_avg+age_std,size=age_null_count)
df['age'][np.isnan(df['age'])]=age_null_random_list
df['age']=df['age'].astype(int)

4、类型转换code

# labelEncoder from sklearn
cate_features=[col1,col2]
for col in cate_features:
    colname='{}id'.format(col)
    
    le =LabelEncoder()
    le.fit(list(df[col].unique()))
    df[colname]=le.transform(df[col].values.tolist())
    # 输出item对应的序号
    print([tuple(item[0]) for item in zip(df[[col,colname]].drop_duplicates().values.tolist())])
    
# trans has to 0/1
df[col].apply(lambda x: 0 if type(x) == float else 1)

# qut data to box
pd.cut(train['age'],5)
pd.qcut(train['age'],5)

# trans male/female to 0/1
df['Sex']=df['Sex'].map({'female':0,'male':1}.astype(int))

5、构建映射字典

def transtodict(key_list,value_list):
    res=dict(zip(key_list,value_list))
    return res
transtodict(list1,list2)

6、数据标准化

# 数据标准化

sc=StandardScaler()
num_feature=['a','b','c']
sc.fit_transform(df[num_feature])
sc.transform(df[num_feature])

# z-score标准化
def z_score(data):
    mean=np.nanmean(data)
    std=np.nanstd(data)
    res=list(map(lambda x:(x-mean)/std if not np.isnan(x) else x,data))
    return res

# 归一化
def normalization(data):
    s=np.nansum(data)
    res=list(map(lambda x:x/s if not np.isnan(x) else x,data))
    return res

def log(data):
    res=list(map(lambda x:np.log(x) if not np.isnan(x) else x,data))
    return res

7、数据观测

# 皮尔逊相关系数
colormap=plt.cm.viridis
plt.figure(figsize=(12,12))
plt.title('Pearson  Correlation of Features',y=1.05,size=15)
sns.heatmap(df_iris.astype(float).corr(),linewidths=0.1,
            vmax=1.0,square=True,cmap=colormap,
           linecolor='white',annot=True)
           
# Pairplots
g=sns.pairplot(df_iris,hue='Survived',palette='seismic',size=1.2,diag_kind='kde',
              diag_kws=dict(shade=True),plot_kws=dict(s=10))
g.set(xticklabels=[])

n、内存优化

def reduce_mem_usage(df, verbose=True):
    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']
    start_mem = df.memory_usage().sum() / 1024**2    
    for col in df.columns:
        col_type = df[col].dtypes
        if col_type in numerics:
            c_min = df[col].min()
            c_max = df[col].max()
            if str(col_type)[:3] == 'int':
                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:
                    df[col] = df[col].astype(np.int8)
                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:
                    df[col] = df[col].astype(np.int16)
                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:
                    df[col] = df[col].astype(np.int32)
                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:
                    df[col] = df[col].astype(np.int64)  
            else:
                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:
                    df[col] = df[col].astype(np.float16)
                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:
                    df[col] = df[col].astype(np.float32)
                else:
                    df[col] = df[col].astype(np.float64)    
    end_mem = df.memory_usage().sum() / 1024**2
    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))
    return df

特征选择

1、删除特征

可以删除的几类：

• 对每个特征进行unique，标记只有1或者0个值的特征；
• 含NULL值大于90%的特征；
• 相关性高；
• 同一指向性的多个特征；
• 通过value_counts统计频数，标记top1元素出现频次占比大于90%的特征

drop_elements = ['PassengerId', 'Name', 'Ticket', 'Cabin', 'SibSp']
train = train.drop(drop_elements, axis = 1)

选择模型