python文本挖掘包_利用python的sklearn开源包进行文本挖掘

从网站抓取训练样本数据，代码：

import urllib2

from BeautifulSoup import BeautifulSoup

import sys

import re

import time

import sys

reload(sys)

sys.setdefaultencoding('utf-8')

url=['http://news.baidu.com/n?cmd=4&class=mil&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=finannews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=internet&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=housenews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=autonews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=sportnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=enternews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=gamenews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=edunews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=healthnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=technnews&pn=1&from=tab' ,'http://news.baidu.com/n?cmd=4&class=socianews&pn=1&from=tab']

ff=['E:/baidu/军事.txt' ,'E:/baidu/财经.txt' ,'E:/baidu/互联网.txt' ,'E:/baidu/房产.txt' ,'E:/baidu/汽车.txt' ,'E:/baidu/体育.txt' ,'E:/baidu/娱乐.txt' ,'E:/baidu/游戏.txt' ,'E:/baidu/教育.txt' ,'E:/baidu/女人.txt' ,'E:/baidu/科技.txt' ,'E:/baidu/社会.txt']

for j in range(7,8):

soup=BeautifulSoup(urllib2.urlopen(url[j]).read())

main=soup.find('div',{'class':'p2'})

index=main.findAll('a')

len_0=len(index)

a=[]

for i in range(len_0):

a.append(index[i]['href'])

for i in range(len_0):

try:

soup=BeautifulSoup(urllib2.urlopen(a[i]).read())

txt=soup.findAll(text=re.compile(ur'[\u4e00-\u9fa5] '))

txt_=''.join(txt)

f= open(ff[j],'a')

print >>f,txt_

f.close()

except:

continue

有监督学习的文本分类代码：

import jieba

import os

import sys

import codecs

from sklearn import feature_extraction

from sklearn import svm

from sklearn.neighbors import KNeighborsClassifier

from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.feature_extraction.text import CountVectorizer

from sklearn import tree

from sklearn.naive_bayes import MultinomialNB

#--------------#

def load_data():

corpus_train=[]

target_train=[]

filepath='E:\python_pananteng/程序6：文本挖掘/文本分类/实例2/train'

filelist = os.listdir(filepath)

for num in range(len(filelist)):

filetext=filepath '/' filelist[num]

filename=os.path.basename(filetext)

myfile = codecs.open(filetext, 'r','utf-8')

temp=myfile.readlines()

myfile.close()

for i in range(0,100):

len_0=len(temp)

seg_list=jieba.cut(','.join(temp[int(i*len_0/100):int((i 1)*len_0/100)]), cut_all=False)

words=' '.join(seg_list)

target_train.append(filename)

corpus_train.append(words)

#--------------#

corpus_test=[]

target_test=[]

filepath='E:\python_pananteng/程序6：文本挖掘/文本分类/实例2/test'

filelist = os.listdir(filepath)

for num in range(len(filelist)):

filetext=filepath '/' filelist[num]

myfile = open(filetext, 'r')

temp=myfile.readlines()

myfile.close()

seg_list=jieba.cut(','.join(temp[1:]), cut_all=False)

words=' '.join(seg_list)

target_test.append(temp[0])

corpus_test.append(words)

return [[corpus_train,target_train],[corpus_test,target_test]]

#--------------#

def data_pro():

[[corpus_train,target_train],[corpus_test,target_test]]=load_data()

count_v1=CountVectorizer()

#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频

counts_train=count_v1.fit_transform(corpus_train)

#fit_transform是将文本转为词频矩阵

transformer=TfidfTransformer()

#该类会统计每个词语的tf-idf权值

tfidf_train=transformer.fit(counts_train).transform(counts_train)

#fit_transform是计算tf-idf

weight_train=tfidf_train.toarray()

#weight[i][j],第i个文本，第j个词的tf-idf值

count_v2=CountVectorizer(vocabulary=count_v1.vocabulary_)

#让两个CountVectorizer共享vocabulary

counts_test=count_v2.fit_transform(corpus_test)

#fit_transform是将文本转为词频矩阵

transformer=TfidfTransformer()

#该类会统计每个词语的tf-idf权值

tfidf_test=transformer.fit(counts_train).transform(counts_test)

#fit_transform是计算tf-idf

weight_test=tfidf_test.toarray()

#weight[i][j],第i个文本，第j个词的tf-idf值

return [[weight_train,target_train],[weight_test,target_test]]

#--------------#

[[weight_train,target_train],[weight_test,target_test]]=data_pro()

#---------------------------------------------#

knnclf = KNeighborsClassifier()

knnclf.fit(weight_train,target_train)

knn_pred = knnclf.predict(weight_test)

#knn模型

#---------------------------------------------#

#---------------------------------------------#

#svm模型

svc = svm.SVC(kernel='linear')

svc.fit(weight_train,target_train)

svc_pred = svc.predict(weight_test)

#---------------------------------------------#

#---------------------------------------------#

#tree模型

tre = tree.DecisionTreeClassifier()

tre.fit(weight_train,target_train)

tre_pred = tre.predict(weight_test)

#---------------------------------------------#

#---------------------------------------------#

#bayes模型

bayes = MultinomialNB(alpha = 0.01)

bayes.fit(weight_train,target_train)

bayes_pred = bayes.predict(weight_test)

#---------------------------------------------#

调用两个开源库，分别是

1、结巴中文分词库，运用该库对网页抓取的中文文章进行分词

2、sklearn机器学习库，调用里面的算法有：tf-idf算法，将文本转换为特征数字矩阵；及knn算法、svm算法、naivebeyes算法、cart算法，这三个算法都是分类的算法，作用是对网页抓取的文章进行有监督的分类学习

效果：

训练：样本 1001 个，其中有3类文章，

第一类，互联网类，样本数量 300个

第二类，军事类，样本数量309个

第三类，财经类，样本数量302个

测试：150个测试样本

KNN算法，命中118个，错误32个

SVM算法，命中125个，错误25个

CART算法，命中122个，错误28个

Bayes算法，命中130个，错误20个