利用TfidfVectorizer+朴素贝叶斯进行文本分类

处理流程一般为：

     提取特征数据集 标签 并向量化

     选择合适的分类器构建模型

     对模型进行验证

案例：

from sklearn.datasets import fetch_20newsgroups
from sklearn.cross_validation import cross_val_score
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
import matplotlib.pyplot as plt
from sklearn.naive_bayes import MultinomialNB

news=fetch_20newsgroups(subset='all')
X,Y=news.data,news.target
k=list(range(10000,180000,10000))
k_count_score=[]
k_tfidf_score=[]
for i in k:
    #tfidf分类器
    tfidf=TfidfVectorizer(analyzer='word',stop_words='english' ,max_features=i)
    X_tfidf=tfidf.fit_transform(X)
    mnb_tfidf=MultinomialNB()
    scores_tfidf=cross_val_score(mnb_tfidf,X_tfidf,Y,cv=10,scoring='accuracy')
    score_tfidf=scores_tfidf.mean()
    k_tfidf_score.append(score_tfidf)
    #tf分类器
    count=CountVectorizer(analyzer='word',stop_words='english' ,max_features=i)
    X_count=count.fit_transform(X)
    mnb_count=MultinomialNB()
    scores_count=cross_val_score(mnb_count,X_count,Y,cv=10,scoring='accuracy')
    score_count=scores_count.mean()
    print(score_count)
    d=()
    d=X_count.get_shape()
    print("维数",d[1])
    k_count_score.append(score_count)
plt.xlabel('dimension')
plt.ylabel('accuracy')
plt.plot(k,k_count_score)
plt.plot(k,k_tfidf_score,color='red')
plt.legend()
plt.show()

 输出：

D:\F\Anaconda3\lib\site-packages\sklearn\cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.