基于Python结巴分词（调用自定义词库已经去除停用词）

# -*- coding: utf-8 -*-
import time
import jieba
import jieba.posseg as pseg#用于词性标注
#分词

#停用词过滤
def stop_word(fid1,fid2,fid3):
    stopword=[]
    for j in fid2.readlines():
        stopword.append(j.strip().decode("utf-8"))#储存停用词表
        #print j
    for i in fid1.readlines():
        data_line=i.strip()
        wordList = jieba.cut(data_line.decode("utf-8"))#wordlist是一个生成器
        outStr=''
        for word in wordList:
            if word not in stopword:
                outStr+=word
                outStr+=' '
        fid3.write(outStr.strip().encode('utf-8') + '\n')

#主文件
def main():
    jieba.enable_parallel()
    # 加入自定义词库
    jieba.load_userdict("/Users/zhuxinquan/Desktop/mykeyword.txt")
    t1 = time.time()
    fid1=open('/Users/zhuxinquan/Desktop/合并3.txt','r')#读取文件
    fid2=open('/Users/zhuxinquan/Desktop/stopword.txt','r')#读取停用词表
    fid3=open('/Users/zhuxinquan/Desktop/文本检索语料库5.txt','w')#将要写入的文件
    stop_word(fid1,fid2,fid3)#停用词过滤

    fid1.close()
    fid2.close()
    fid3.close()
    t2 = time.time()
    tm_cost = t2-t1
    print tm_cost

main()