NLP之词频统计实战

# -*- coding: gb2312 -*-
'''
2020/3.15 词频统计
'''
#读入数据
raw_content=[]
with open('3635.txt',"r",encoding='UTF8') as fin:
    [raw_content.append(line) for line in fin.readlines() if line.strip()!='']
    for num in range(1,10):
        print(raw_content[num])
    print(len(raw_content))

import re
pattern1=r"[【】,.\"\"“”：:（）()《》<>！；，。？...！？]+"
pattern2=re.compile(r'[1-9]\d*')
content=[]
for each_line in raw_content:
    each_line=re.sub(pattern1,"",each_line)
    each_line=re.sub(pattern2,'',each_line)
    sentences=each_line.split("\n")
    [content.append(e) for e in sentences if e!='']
print(len(content))
for num in range(20):
    print(content[num])

#使用jieba的精确模式进行分词
import jieba
sent_list=[]
for each_sentence in content:
    sent_seg=jieba.cut(each_sentence,cut_all=False,HMM=True)
    sent_seg=''.join(sent_seg)
    sent_list.append(sent_seg)
print(len(sent_list))

#词频统计
word_list=[]
word_dict={}
for each_sent in sent_list:
    [word_list.append(each_word) for each_word in each_sent.split(' ')]
print(len(word_list))
print(word_list)
for word in word_list:
    if len(word)<2:
        continue
    if word_dict.get(word):
        word_dict[word]+=1
    else:
        word_dict[word]=1
print(len(word_dict))

import matplotlib.pyplot as plt
import matplotlib
from matplotlib.font_manager import FontProperties
font_name='KaiTi_GB2312'
#用来正常显示中文标签
matplotlib.rcParams['font.family']=font_name
top10words=list(word_dict.items())
top10words.sort(key=lambda x:x[1],reverse=True)
#打印出现频率最高的前十个词，并将其存入list方便可视化
x=[]
y=[]
for i in range(10):
    print(top10words[i][0])
    print(top10words[i][1])
    x.append(top10words[i][0])
    y.append(top10words[i][1])
#画出分布直方图
plt.bar(x,height=y,width= .7)
plt.xticks(rotation=75)
plt.show()
plt.pie(y,labels=x,autopct='%.0f%%')
plt.show()

下面就是词频最高的10个词。