对文章的分类和聚类

以之前抓取的“科技”和“娱乐”文章为例

‘articls.csv’中的内容如图：

分类

import jieba
import joblib
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
import numpy as np
from sklearn.metrics import accuracy_score
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn import tree
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import AdaBoostClassifier

#获取停用词
def get_stop_words():
    content=list()
    with open(r'stop.txt',encoding='utf-8') as f:
        for line in f:
            content.append(line.strip())
    return content
stop_hanzi=get_stop_words()
print(' '.join(stop_hanzi))
#'$ 0 1 2 3 4 5 6 7 8 9 ? _ “ ” 、 。 《 》 一 一些 一何 一切......'

#将文章进行清洗，转换
article_path=r'articls.csv'
def get_TFIDF():
    labels = list()  #标签库，做验证用
    corpus = list()  # 语料库 空格连接
    # 读取语料  一行为一个文档
    for line in open(article_path, 'r',encoding='utf8').readlines():
        label, content=line.strip().split(',')
        content=[x for x in jieba.cut(content) if x not in stop_hanzi]
        #如果文章不为空
        if content:
            corpus.append(' '.join(content))
            labels.append(label)    
    labels=list(map(lambda x:0 if x=='news_tech' else 1,labels))    

    # 将文本中的词语转换为词频矩阵 矩阵元素a[i][j] 表示j词在i类文本下的词频
    vectorizer = CountVectorizer()
    # 该类会统计每个词语的tf-idf权值
    transformer = TfidfTransformer()
    # 第一个fit_transform是计算tf-idf（词频-逆文章频率）， 第二个fit_transform是将文本转为词频矩阵
    tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus))
    # 获取词袋模型中的所有词语
    word = vectorizer.get_feature_names()  
    # 将tf-idf矩阵抽取出来，元素w[i][j]表示j词在i类文本中的tf-idf权重
    weight = tfidf.toarray()    
    return weight,np.array(labels)

weight,labels=get_TFIDF()
print('weight.shape: {}, lables.shape: {}'.format(weight.shape,labels.shape))
# weight.shape: (884, 44870), lables.shape: (884,),表示有884篇文章，提取了44870个特征词

#训练集与测试集的比例为6:4
x_train, x_test, y_train, y_test=train_test_split(weight,labels,test_size=0.4, random_state=0)

#使用SVM分类器
kernels=['sigmoid','linear','rbf']
for kernel in kernels:
    clf = svm.SVC(kernel=kernel).fit(x_train, y_train)
    print('svm_{}_accuracy:{}'.format(kernel,clf.score(x_test, y_test)))

#svm_sigmoid_accuracy:0.576271186440678
#svm_linear_accuracy:0.9971751412429378
#svm_rbf_accuracy:0.576271186440678

#使用决策树分类
clf = tree.DecisionTreeClassifier().fit(x_train, y_train)
print('{}:{}'.format('DecisionTreeClassifier', clf.score(x_test, y_test)))
#DecisionTreeClassifier:0.9293785310734464

#使用随机森林分类
clf = RandomForestClassifier(n_estimators=10, max_depth=None,min_samples_split=2, random_state=0).fit(x_train, y_train)
print('{}:{}'.format('RandomForestClassifier', clf.score(x_test, y_test)))
#RandomForestClassifier:0.9717514124293786

#使用AdaBoostClassifier
clf = AdaBoostClassifier(n_estimators=100).fit(x_train, y_train)
print('{}:{}'.format('AdaBoostClassifier', clf.score(x_test, y_test)))
#AdaBoostClassifier:0.9717514124293786

#使用GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingClassifier
clf = GradientBoostingClassifier(n_estimators=100,random_state=0).fit(x_train, y_train)
print('{}:{}'.format('GradientBoostingClassifier', clf.score(x_test, y_test)))
#GradientBoostingClassifier:0.9548022598870056

#使用交叉验证集
from sklearn.model_selection import cross_val_score
clf = svm.SVC(kernel='linear')
scores = cross_val_score(clf, weight, labels, cv=5)
scores
#array([ 0.98876404,  0.97740113,  0.97740113,  0.97727273,  0.98863636])

#使用GridSearchCV调参
from sklearn.model_selection import GridSearchCV
parameters = {'kernel':('linear', 'rbf'), 'C':[1, 5]}
clf=GridSearchCV(estimator =svm.SVC(),param_grid =parameters,scoring='accuracy',cv=5)  
clf.fit(weight,labels)

clf.best_params_ 
#{'C': 1, 'kernel': 'linear'}
clf.best_score_ 
#0.98190045248868774

parameters ={'n_estimators':range(10,61,10)} 
clf = GridSearchCV(estimator =RandomForestClassifier(min_samples_split=2, random_state=0),param_grid =parameters,scoring='accuracy',cv=5)
clf.fit(weight,labels)

clf.best_params_ 
#{'n_estimators': 40}
clf.best_score_ 
#0.98190045248868774

#使用 kaggle神器xgboost
#下载地址https://www.lfd.uci.edu/~gohlke/pythonlibs/#xgboost

import xgboost as xgb
from xgboost import XGBClassifier

clf = XGBClassifier()
clf.fit(x_train, y_train)
print('{}_accuracy:{}'.format('xgboost',clf.score(x_test, y_test)))
#xgboost_accuracy:0.9745762711864406

#使用GridSearchCV对XGBClassifier调参
parameters = {'learning_rate' : [0.001, 0.01, 0.1, 0.4, 0.7]}
clf = GridSearchCV(estimator = XGBClassifier(), param_grid = parameters, scoring="neg_log_loss", n_jobs=-1, cv=5)
clf.fit(weight,labels)

clf.best_params_ 
#{'learning_rate': 0.1}
clf.best_score_ 
0.97285067873303166

聚类

from sklearn.cluster import KMeans

weight,labels=get_TFIDF()

# 选择2个中心点
clf = KMeans(n_clusters=2)
# clf.fit(X)可以把数据输入到分类器里
clf.fit(weight)

# 打印2个中心点
print('cluster_center:')
print(clf.cluster_centers_)
#cluster_center:
#[[  2.65940140e-04   8.00510130e-05   7.36860595e-05 ...,   5.35179931e-05
#    3.78464085e-05   6.36585103e-05]
# [  2.48911399e-04   5.80102231e-05   1.45779433e-04 ...,  -8.80914265e-20
#    5.75982404e-20   0.00000000e+00]]

# 保存模型
joblib.dump(clf, 'kmeans.pkl')
# 载入保存的模型
clf = joblib.load('kmeans.pkl')
#预测
clf.fit_predict(weight)

#打印分类结果
print(clf.labels_)
#[1 0 1 0 1 1 0 0 1 1 0 0 0 0 1 1 0 0 1 0 1 1 1 0 ......]

#对聚类结果打分
score = accuracy_score(clf.labels_, labels)
#因为聚类对0, 1的不确定性，结果越趋近0或1越好，0.5左右就是乱猜了
score = max(score,1-score)
score
#0.95248868778280538