简单python逆向分词

使用python进行逆向最大长度分词

# Python 3.4.3

import time
start = time.clock()
#说明：分词效果好坏在于词典中词项的优劣
# 为了方便，词典直接写在程序里。
#lexicon = ('发展','中国','国家','领导人','正在','开会')                    # ---对应于test1.txt
#lexicon = ('共同','创造','美好','的','新','生活')                          # ---对应于test1.txt
lexicon = ('词序列','分词','过程','合成','就是','连续','重新','字序列','组合')  # ---对应于test2.txt
#lexicon = ('计算语言学','课程','课时')                                      # ---对应于test3.txt
#lexicon = ('南京','市长','大桥')                                          # 搞笑词典1---对应于test4.txt
#lexicon = ('南京市','长江大桥')                                           # 正规词典1---对应于test4.txt
outi = 0
wordSeg = []    # 新建列表存放切分好的词
maxWordLen = 3  # 最大词长设为3，记录一下最大词长为3的分词结果
#maxWordLen = 5  # 最大词长设为5---测试“计算机语言学”示例时将最大词长设置为5
with open('test3.txt','r', encoding='utf-8') as src:
    sentence = src.read()
    #sentence = '我和你共同创造美好的新生活'
    sentenceLen = len(sentence)
    wordLen = min(maxWordLen, sentenceLen)
    startPoint = 0
    while startPoint < sentenceLen:  # 从第一个字符循环到最后一个字符
        matched = False    # 假设找不到匹配的词
        for i in range(maxWordLen, 0, -1):  # 从最大词长3递减到1
            string = sentence[sentenceLen-startPoint-i:sentenceLen-startPoint]  # 取startPoint开始到startPoint+i-1的切片
            print(string)
            if string in lexicon:
                wordSeg.append(string)
                matched = True
                break
        if not matched:    # 假如在词典中找不到匹配
            i = 1          
            wordSeg.append(sentence[sentenceLen-startPoint-1])   # 全部切分为单字词
        startPoint += i

with open('WordSeg.txt', 'w', encoding='utf-8') as des:
    while outi < len(wordSeg):
        des.write(wordSeg[len(wordSeg)-outi-1]+'/')
        outi += 1
    #for word in wordSeg:
    #    des.write(word+'/')
end = time.clock()
TotalTime = 1000*(end - start)
print("---Processing time:%s ms ---" % TotalTime)
#print("--- %s seconds ---" % (time.time() - start_time))