"""
功能:PyNLPIR文本预处理
过程:文本分词,词频统计
时间:2020年02月16日
作者:Angel
import pynlpir
pynlpir.open()
#文本分词、词频统计
p = open(r'test.txt', 'r', encoding = 'utf-8')
q = open(r'test_result.txt', 'w', encoding = 'utf-8')
counts = {} #定义空字典
#直接打印出结果
for line in p.readlines():
words = pynlpir.segment(line, pos_english=False) # 把词性标注语言变更为汉语
for word,flag in words:
q.write(str(word) + str(flag) + " ")
if len(word) == 1:
continue
else:
rword = word
s = rword+','+flag #将分词和词性进行拼接
counts[s] = counts.get(s,0) + 1 #同时统计分词和词性的词频,存储在字典counts中
q.write('\n')
#遍历字典输出
for count in counts:
print(count+':'+str(counts[count]))
#将字典转换为列表排序后遍历输出
items = list(counts.items())
items.sort(key=lambda x:x[1],reverse = True)
for item in items:
print(item)
p.close()