Scikit-learn Cookbook (三) --- Postmodel Workflow

一、K-fold cross validation

from sklearn import datasets                          
from sklearn import ensemble
rf = ensemble.RandomForestRegressor(max_features = 'auto')
X, y = datasets.make_regression(10000, 10)
from sklearn import cross_validation
scores = cross_validation.cross_val_score(rf, X, y)
print(scores)
#[ 0.86310151  0.87504517  0.86335846]
scores = cross_validation.cross_val_score(rf, X, y, verbose=3, cv=4)
#[CV] no parameters to be set .........................................
#[CV] ................ no parameters to be set, score=0.867883 -   0.4s
#[CV] no parameters to be set .........................................
#[CV] ................ no parameters to be set, score=0.878830 -   0.4s
#[CV] no parameters to be set .........................................
#[CV] ................ no parameters to be set, score=0.869258 -   0.4s
#[CV] no parameters to be set .........................................
#[CV] ................ no parameters to be set, score=0.872930 -   0.4s
#[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:    1.9s finished

二、Stratified k-fold

The class representation was unbalanced in some manner. Stratified k-fold is specifically designed to maintain the class proportions.

1. Create datasets

from sklearn import datasets
X, y = datasets.make_classification(n_samples=int(1e3),weights=[1./11])
y.mean()  
# 0.90600000000000003  

从以上数字可以看出，90.6%的样本为正，其余为负。

2. 对比Stratified k-fold and shuffle_split k-fold

from sklearn import cross_validation
n_folds = 50
strat_kfold = cross_validation.StratifiedKFold(y, n_folds = n_folds)
shuff_split = cross_validation.ShuffleSplit(n=len(y),n_iter=n_folds)

kfold_y_props = []
shuff_y_props = []

for (k_train,k_test), (s_train,s_test) in zip (strat_kfold,shuff_split):
    kfold_y_props.append(y[k_train].mean())
    shuff_y_props.append(y[s_train].mean())

import matplotlib.pyplot as plt
f, ax = plt.subplots(figsize=(7, 5))
ax.plot(range(n_folds), shuff_y_props, label = "ShuffleSplit")
ax.plot(range(n_folds),kfold_y_props,label = "Stratified", color='k', ls='--')
ax.set_title("Comparing class proportions")
ax.legend(loc='best')  

We can see that the proportion of each fold for stratified k-fold is stable across folds

3. for multiclasses

import numpy as np
three_classes = np.random.choice([1,2,3],p=[.1,.4,.5],size=1000) 

import itertools as it
for train, test in cross_validation.StratifiedKFold(three_classes, 5):
    print(np.bincount(three_classes[train]))
# [  0  80 291 428]
# [  0  80 291 429]
# [  0  80 291 429]
# [  0  80 291 429]
# [  0  80 292 429]

三、Poor man’s grid search

1.选择两个参数维度

criteria = {gini, entropy}
max_features = {auto, log2, None}
parameter space = criteria * max_features

# Poor man's grid search
from sklearn import datasets
X, y = datasets.make_classification(n_samples = 2000, n_features=10)

criteria = {'gini','entropy'}
max_features = {'auto','log2',None}
import itertools as it
parameter_space = it.product(criteria, max_features)

import numpy as np
train_set = np.random.choice([True, False],size=len(y))
from sklearn.tree import DecisionTreeClassifier
accuracies = {}
for criterion, max_feature in parameter_space:
    print(criterion,max_feature)
    dt = DecisionTreeClassifier(criterion=criterion, max_features = max_feature)
    dt.fit(X[train_set],y[train_set])
    accuracies[(criterion,max_feature)] = (dt.predict(X[~train_set]) == y[~train_set]).mean()

print(accuracies)

2. Visualize the results

# Visualize
from matplotlib import pyplot as plt
from matplotlib import cm

cmap = cm.RdBu_r
f, ax = plt.subplots(figsize=(7,4))
ax.set_xticklabels([''] + list(criteria))
ax.set_yticklabels([''] + list(max_features))
plot_array = []
for max_feature in max_features:
    m=[]
    for criterion in criteria:  
        m.append(accuracies[(criterion, max_feature)])
    plot_array.append(m)
colors = ax.matshow(plot_array, vmin=np.min(list(accuracies.values()))-0.001,
                    vmax=np.max(list(accuracies.values()))+0.001,cmap=cmap)        
f.colorbar(colors)

注意：python3中，accuracies.values()为字典格式，需要通过list()函数将其转化为列表形式的，否则会报错。

四、Brute force grid search

1. Create some classification data

# Brute force grid search
from sklearn.datasets import make_classification
X, y = make_classification(1000, n_features=5)

2.Create logistic regression object

from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(class_weight='auto')

3. Grid Search

#lr.fit(X,y)
grid_search_params = {'penalty':['l1','l2'],
                      'C':[1,2,3,4]}
import scipy.stats as st
random_search_params = {'penalty':['l1','l2'],
                      'C':st.randint(1,4)}
#fit the classifier
from sklearn.grid_search import GridSearchCV, RandomizedSearchCV
gs = GridSearchCV(lr, grid_search_params)
gs.fit(X, y)
rs = RandomizedSearchCV(lr, random_search_params)
rs.fit(X, y)
gs.grid_scores_
rs.grid_scores_
max(gs.grid_scores_,key=lambda x : x[1])
max(rs.grid_scores_,key=lambda x : x[1])