我使用的数据集是从网上下载的,下载地址和具体内容直接看网址吧
Home Page for 20 Newsgroups Data Set (qwone.com)
这里是下载,然后解压就可以用了
import os
import sys
# 加载数据
from sklearn import datasets
twenty_train = datasets.load_files("C:/Users/Rick/Desktop/20news-bydate/20news-bydate-train")
twenty_test = datasets.load_files("C:/Users/Rick/Desktop/20news-bydate/20news-bydate-test")
#打印一下文件数量
print(len(twenty_train.target_names),len(twenty_train.data),len(twenty_train.filenames),len(twenty_test.data))
#计算词频
from sklearn.feature_extraction.text import CountVectorizer
count_vect = CountVectorizer(stop_words="english",decode_error='ignore')
X_train_counts = count_vect.fit_transform(twenty_train.data)
X_train_counts.shape
#使用TF-IDF进行特征提取
from sklearn.feature_extraction.text import TfidfTransformer
tf_transformer = TfidfTransformer(use_idf = False).fit(X_train_counts)
X_train_tf = tf_transformer.transform(X_train_counts)
X_train_tf.shape
tfidf_transformer = TfidfTransformer()
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
X_train_tfidf.shape
from sklearn.naive_bayes import MultinomialNB
clf = MultinomialNB().fit(X_train_tfidf,twenty_train.target)
docs_new = ['God is love','OpenGL on the GPU is fast']
X_new_counts = count_vect.transform(docs_new)
X_new_tfidf = tfidf_transformer.transform(X_new_counts)
predicted = clf.predict(X_new_tfidf)
for doc,category in zip(docs_new,predicted):
print(("%r => %s") %(doc,twenty_train.target_names[category]))
#建立管道
from sklearn.pipeline import Pipeline
text_clf = Pipeline([('vect',CountVectorizer(stop_words="english",decode_error='ignore')),
('tfidf',TfidfTransformer()),
('clf',MultinomialNB()),
])
text_clf = text_clf.fit(twenty_train.data,twenty_train.target)
#朴素贝叶斯分类器
import numpy as np
docs_test = twenty_test.data
predicted = text_clf.predict(docs_test)
np.mean(predicted == twenty_test.target)
#线性核支持向量机
from sklearn.linear_model import SGDClassifier
text_clf_2 = Pipeline([('vect',CountVectorizer(stop_words='english',decode_error='ignore')),
('tfidf',TfidfTransformer()),
('clf',SGDClassifier(loss = 'hinge',penalty = 'l2',
alpha = 1e-3,n_iter_no_change = 5, random_state = 42)),
])
_ = text_clf_2.fit(twenty_train.data,twenty_train.target)
predicted = text_clf_2.predict(docs_test)
np.mean(predicted == twenty_test.target)
通过朴素贝叶斯和向量机两种方式进行分类结果如下
可以看到向量机是优于朴素贝叶斯分类的结果的