使用pymmseg进行中文分词
python分词模块,基于mmseg算法编写,核心代码c++,提供python接口。
最近在做BUG质量分析的时候,要提取BUG关键词,正好用到的pymmseg这分词模块,分词的效率还不错。
Code Example:
#-*- coding: utf-8 -*- from pymmseg import mmseg import os import sys def CWS_pymmseg(shortdeslist,wordlist): if os.path.isfile(shortdeslist): mmseg.dict_load_defaults() sd = open(shortdeslist,'r') word = open(wordlist,'w') for bugdes in sd.readlines(): algor = mmseg.Algorithm(bugdes) wlist = [] for tok in algor: wlist.append(tok.text + "\n") word.writelines(wlist) sd.close() word.close() print "CWSeg_pymmseg is OK ! %s ==> %s" % (shortdeslist,wordlist) else: print "ERROR : The file ,shortdeslist doesn't exist!" if __name__ == '__main__': if len(sys.argv) == 3: CWS_pymmseg(sys.argv[1],sys.argv[2]) else: print "Usage: python CWS_pymmseg.py [shortdeslist] [wordlist]"