Python - 机器学习的分类方法（KNN、朴素贝叶斯 和 决策树）

1. KNN：K最近邻法，把所有的训练集数据都加载到内存中，当它需要对测试实例进行分类时，它衡量这个实例的所有训练实例之间的距离，基于距离，它选择训练集里的K个最近的实例。
2. 朴素贝叶斯分类器：贝叶斯是基于“独立假定”的概念，即分类实例之间是相互独立的，例如文档里出现的词是相互独立的，并基于此假定来计算过概率。相关方程在文末有张图表示。文档内正负词分类相对复杂，这里不做详细介绍。
3. 决策树：if-then语句分层组织来构建决策树。优点是易于解释，支持多分类问题，但容易过拟合，类别不平衡的影响十分严重，因此要注意剪枝和按相同分类比例划分数据集。

1. KNN

# -*- coding: utf-8 -*-
"""
Created on Wed Apr 25 17:15:34 2018

@author: Alvin AI
"""

from sklearn.cross_validation import StratifiedShuffleSplit
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import classification_report
import numpy as np
from sklearn.datasets import make_classification
import matplotlib.pyplot as plt
import itertools


def get_data():
    """
    Make a sample classification dataset
    Retrun ： Independent variable y, dependent variable x
    """
    x,y = make_classification(n_features=4)#任意分类100条测试数据（含4种特征值）
    return x,y

#得到训练集和测试集
def get_train_test(x,y):
    train_size = 0.8
    test_size = 1-train_size
    input_dataset = np.column_stack([x,y])
    stratified_split = StratifiedShuffleSplit(input_dataset[:,-1],\
                test_size=test_size,n_iter=1,random_state = 77)
    
    for train_indx,test_indx in stratified_split:
        train_x = input_dataset[train_indx,:-1]
        train_y = input_dataset[train_indx,-1]
        test_x = input_dataset[test_indx,:-1]
        test_y = input_dataset[test_indx,-1]
        
    return train_x,train_y,test_x,test_y

#构造模型
def build_model(x,y):
    knn = KNeighborsClassifier(n_neighbors=k)
    knn.fit(x,y)
    return knn

def test_model(x,y,knn_model):
    y_predicted = knn_model.predict(x)
    print classification_report(y,y_predicted)

def plot_data(x,y):
    """
    plot a scatter plot for all variable combinations
    """
    subplot_start = 321
    col_numbers = range(0,4)
    col_pairs = itertools.combinations(col_numbers,2)#0 1 2 3 四个数字两两组合
    
    for col_pair in col_pairs:
        plt.subplot(subplot_start)
        plt.scatter(x[:,col_pair[0]],x[:,col_pair[1]],c=y)
        title_string = str(col_pair[0]) + "-" +str(col_pair[1])
        plt.title(title_string)
        x_label = str(col_pair[0])
        y_label = str(col_pair[1])
        plt.xlabel(x_label)
        plt.ylabel(y_label)
        subplot_start+=1
    plt.show()
    
x,y = get_data()
plot_data(x,y)
    
if __name__ == "__main__":
    k=3
    x,y = get_data()
    plot_data(x,y)
    train_x,train_y,test_x,test_y = get_train_test(x,y)
    knn_model = build_model(train_x,train_y)
    print "\n model evaluation on training set"
    print "==================================\n"
    test_model(train_x,train_y,knn_model)
    
    print "\n model evaluation on test set"
    print "==================================\n"
    test_model(test_x,test_y,knn_model)    

2. 贝叶斯

# -*- coding: utf-8 -*-
"""
Created on Tue Apr 03 19:05:18 2018

@author: Alvin AI
"""

from nltk.corpus import movie_reviews
from sklearn.cross_validation import StratifiedShuffleSplit
import nltk
from string import punctuation
from nltk.corpus import stopwords
from nltk.collocations import BigramCollocationFinder
from nltk.metrics import BigramAssocMeasures

#--------载入数据----------------------------
def get_data():
    """
    Get movie review data
    """
    dataset = []
    y_labels =[]
    #抽取分类
    for cat in movie_reviews.categories():
        #对于每个类别下的文件
        for fileid in movie_reviews.fileids(cat):
            #获取属于这个分类的词语
            words = list(movie_reviews.words(fileid))
            dataset.append((words,cat))
            y_labels.append(cat)
    return dataset,y_labels

#-------将数据整理成训练集和测试集---------------
def get_train_test(input_dataset,ylabels):
    """
    perpare a stratified train and test split
    """
    train_size = 0.7
    test_size = 1-train_size
    stratified_split = StratifiedShuffleSplit(ylabels,\
                                              test_size=test_size,n_iter=1,\
                                              random_state=77)
    for train_idx,test_idx in stratified_split:
        train = [input_dataset[i] for i in train_idx]
        train_y = [ylabels[i] for i in train_idx]
        test = [input_dataset[i] for i in test_idx]
        test_y = [ylabels[i] for i in test_idx]
    return train,train_y,test,test_y

#---------生成特征----------------------------
def build_word_features(instance):
    """
    build feature dictionary
    features are binary, name of feature is word itself
    and value is 1. features are stored in a dictionary called feature_set
    """
    feature_set = {}#用字典来保持特征值
    words = instance[0]#词列表的实例元组里的第1个子项
    #把特征保存到字典中
    for word in words:
        feature_set[word] = 1
    return(feature_set,instance[1])#实例元组里的第二个子项是类别标签
    
def build_negate_features(instance):
    """
    if a word is preceeded by either 'not' or 'no'
    this function adds a prefix 'Not_' to that word
    it will also not insert the previous negation word
    'not' or 'no' in feature dictionary
    """
    #对词进行检索，即实例元组中的第1个子项
    words = instance[0]
    final_words =[]
    negate = False#用一个布尔变量追踪上一个词是不是负面词
    negate_words = ['no','not']#生成负面词列表
    #对词进行循环的时候，遇到一个负面词，负面标识变量值变为True
    #否定词不会加入到特征字典中
    #当负面标识变量值为True时，词的前面加上一个‘Not_’prefix
    for word in words:
        if negate:
            word = 'Not_' + word
            negate = False#即每次返回false,以便if语句调用
        if word not in negate_words:
            final_words.append(word)
        else:
            negate = True
    #特征字典
    feature_set = {}
    for word in final_words:
        feature_set[word] = 1
    return(feature_set,instance[1])
    
#-----------移除停留词-----------------------------------
def remove_stop_words(in_data):
    """
    Utility function to remove stop words from the given list of words
    """
    stopword_list = stopwords.words('english')
    negate_words = ['no','not']
    #我们不希望删除负面词
    #我们创建一个新的停用词列表来排除负面词
    new_stopwords = [word for word in stopword_list if word not in negate_words]
    label = in_data[1]
    #删除停用词和标点符号
    words = [word for word in in_data[0] if word not in new_stopwords and word\
             not in punctuation]
    return (words,label)
    
def build_keyphrase_features(instance):
    """
    a function to extract key phrases from the given text.
    key phrases are words of importance according to a measure
    in this key out phrase of is our length 2, i.e two words of bigrams
    """
    feature_set = {}
    instance = remove_stop_words(instance)
    words = instance[0]
    
    bigram_finder = BigramCollocationFinder.from_words(words)
    #我们采用二元特征的原始频率计数
    #例如，二元特征按出现的频率降序排序，选择前400个
    bigrams = bigram_finder.nbest(BigramAssocMeasures.raw_freq,400)
    for bigram in bigrams:
        feature_set[bigram] = 1
    return (feature_set,instance[1])

#---------------建模------------------------------------
def build_model(features):
    """
    build a naive bayes model with the given feature set
    """
    model = nltk.NaiveBayesClassifier.train(features)
    return model

def probe_model(model,features,dataset_type='Train'):
    """
    A utility function to check the goodness of our model
    """
    accuracy = nltk.classify.accuracy(model,features)
    print "\n" + dataset_type + "Accuracy = %0.2f" % (accuracy*100) + "%"
    
def show_features(model,no_features = 5):
    """
    a utility function to see how important various features are for out model
    """
    print "\nFeature importance"
    print "===================\n"
    print model.show_most_informative_features(no_features)
    
#---------------改善模型------------------------------
def build_model_cycle_1(train_data,dev_data):
    """
    first pass at trying out our model
    """
    #为训练集建立特征
    train_features = map(build_word_features,train_data)
    #为测试集建立特征
    dev_features = map(build_word_features,dev_data)
    model = build_model(train_features)#建模
    probe_model(model,train_features)
    probe_model(model,dev_features,'Dev')
    return model

def build_model_cycle_2(train_data,dev_data):
    """
    Second pass at trying out our model
    """
    #为训练集建立特征
    train_features = map(build_negate_features,train_data)#每一行做映射
    #为测试集建立特征
    dev_features = map(build_negate_features,dev_data)
    model = build_model(train_features)#建模
    probe_model(model,train_features)
    probe_model(model,dev_features,'Dev')
    return model

def build_model_cycle_3(train_data,dev_data):
    """
    Third pass at trying out our model
    """
    #为训练集建立特征
    train_features = map(build_keyphrase_features,train_data)
    #为测试集建立特征
    dev_features = map(build_keyphrase_features,dev_data)
    model = build_model(train_features)#建模
    probe_model(model,train_features)
    probe_model(model,dev_features,'Dev')
    test_features = map(build_keyphrase_features,test_data)
    probe_model(model,test_features,'Test')
    return model

#调用上面定义的各个函数
if __name__ == "__main__":
    #加载数据
    input_dataset,y_labels = get_data()
    #训练数据
    train_data,train_y,all_test_data,\
    all_test_y=get_train_test(input_dataset,y_labels)
    #Dev数据
    dev_data,dev_y,test_data,test_y=get_train_test(all_test_data,all_test_y)
    #查看不同数据集的大小
    print "\nOriginal data size =", len(input_dataset)
    print "\ntraining data size =", len(train_data)
    print "\ndev data size =", len(dev_data)
    print "\ntest data size =", len(test_data)
    
    #建模的不同过程
    model_cycle_1 = build_model_cycle_1(train_data,dev_data)
    #打印输出模型的信息
    show_features(model_cycle_1)
    model_cycle_2 = build_model_cycle_2(train_data,dev_data)
    show_features(model_cycle_2)
    model_cycle_3 = build_model_cycle_3(train_data,dev_data)
    show_features(model_cycle_3)    

3. 决策树

# -*- coding: utf-8 -*-
"""
Created on Wed Apr 04 14:47:27 2018

@author: Alvin AI
"""

from sklearn.datasets import load_iris
from sklearn.cross_validation import StratifiedShuffleSplit
import numpy as np
from sklearn import tree
from sklearn.metrics import accuracy_score,classification_report,\
confusion_matrix
import pprint
#import graphviz



#载入数据
def get_data():
    """Get iris data"""
    data = load_iris()
    x = data['data']
    y = data['target']
    label_names = data['target_names']
    return x,y,label_names.tolist()

#得到训练集和测试集
def get_train_test(x,y):
    train_size = 0.8
    test_size = 1-train_size
    input_dataset = np.column_stack([x,y])
    stratified_split = StratifiedShuffleSplit(input_dataset[:,-1],\
                test_size=test_size,n_iter=1,random_state = 77)
    
    for train_indx,test_indx in stratified_split:
        train_x = input_dataset[train_indx,:-1]
        train_y = input_dataset[train_indx,-1]
        test_x = input_dataset[test_indx,:-1]
        test_y = input_dataset[test_indx,-1]
        
    return train_x,train_y,test_x,test_y

#构造模型
def build_model(x,y):
    model = tree.DecisionTreeClassifier(criterion="entropy")
    model = model.fit(x,y)
    return model

#测试模型
def test_model(x,y,model,label_names):
    y_predicted = model.predict(x)
    print "Model accuracy = %0.2f" % (accuracy_score(y,y_predicted)*\
                                      100) + "%\n"
    print "\nConfusion Matrix"
    print "=================="
    #pprint用于打印出任何python数据结构类和方法，输出比较整齐美观
    print pprint.pprint(confusion_matrix(y,y_predicted))
    print "\nclassification report"
    print "=================="
    
    print classification_report(y,y_predicted,target_names=label_names)
                                      
def get_feature_names():
    data = load_iris()
    return data['feature_names']

def probe_model(x,y,model,label_names):
    feature_names = get_feature_names()
    feature_importance = model.feature_importances_
    print "\nfeature importance\n"
    print "======================\n"
    for i,feature_name in enumerate(feature_names):
        print "%s = %0.3f" % (feature_name,feature_importance[i])
    
    #将决策树导出成图
    tree.export_graphviz(model,out_file='tree.png')
    
    #graphviz.view('E:\\python training\\python data science cookbook\\tree.dot')
    
    
if __name__ == "__main__":
    #加载数据
    x,y,label_names = get_data()
    #将数据分割为训练集和测试集
    train_x,train_y,test_x,test_y = get_train_test(x,y)
    #建模
    model = build_model(train_x,train_y)
    #在训练集上评估模型
    test_model(train_x,train_y,model,label_names)
    #在测试集上评估模型
    test_model(test_x,test_y,model,label_names)
    probe_model(x,y,model,label_names)