完整的中英文词频统计

#读取fire文本文件
f = open('fire.txt','r',encoding='utf-8')
fire = f.read()
f.close()
print(fire)
#预处理
print(fire.lower())  #全部字母小写
a = ",.;:'`"   #特殊符号归集为字符串
for b in a:
    fire.replace(b,' ')   #利用for循环语句把特殊符号替换成空格
    print(fire)

#分别从空格提取单词
firelist = fire.split()
print(firelist)

#统计每个单词出现的次数
fireset = set(firelist)  #把列表firelist转换成集合，使得单词不会重复出现
#排除语法型词汇，代词、冠词、连词等无语义词
se = {'a','the','and','we','you','of'}
fireset =fireset-se
firedict = {}  #创建一个空字典，用于存储
# #遍历fireset集合，放入单词为键，统计的次数为值的strdict字典
for word in fireset:
    firedict[word] = firelist.count(word)
print(len(firedict),firedict)
wordlist = list(firedict.items())
#按单词的频数排序
wordlist.sort(key=lambda x:x[1],reverse=True)
print(wordlist)
#输出TOP(20)
for i in range(20):
    print(wordlist[i])

import jieba
f=open('cry.txt','r',encoding='utf-8')
cry=f.read()
f.close()
crylist=list(jieba.cut(cry))
crydict={}
for word in crylist:
    if len(word)==1:
        continue
    else:
     crydict[word]=crydict.get(word,0)+1

wordlist=list(crydict.items())
wordlist.sort(key=lambda x:x[1],reverse=True)

for a in range(15):
    print(wordlist[a])

 

 

 

转载于:https://www.cnblogs.com/xsxsx/p/9775957.html