实践
对给定语料进行分析,包括:字数(中文)、词数(中文、英文(原词、词干))、高频词、低频词(Top50),句子个数,文章个数
提示:
中文需要分词,英文需要做词干化处理(Porter算法:nltk.stem.porter.PorterStemmer)
取词干
PorterStemmer.stem(w)
分词
jieba.cut(sentence,cut_all=False,HMM=True)
代码
from collections import Counter
class myWord:
def __init__(self,text):
self.dict = []
self.count = {}
self.flushDict = []
self.text = text
with open(text, 'r', encoding='utf-8') as f1:
self.list = f1.readlines()
def readDict(self):
""" 读取语料生成词典 """
for l in self.list:
l = l.strip()
if len(l) == 0:
continue
l = l.split(' ')
# 不计日期
for i in range(1,len(l)):
l[i] = l[i].split('/')[0]
self.dict.append(l[i])
def printDict(self):
""" 打印词典 """
for i in range(50):
print(self.dict[i])
def countWords(self):
""" 计算词数(未去重) """
for i in self.dict:
if len(i)!=1:
self.flushDict.append(i)
print("总词数:", len(self.flushDict))
def countWord(self):
""" 计算词数(去重) """
print("去重词数:", len(set(self.flushDict)))
def formatShow(self,list,st,fn):
""" 格式化输出(词频) """
t = 0
for l in list[st:fn]:
t += 1
word = l[0]
times = l[1]
print("word:{} time:{} ".format(word, times).ljust(15), end="")
if t % 4 == 0:
print()
print()
def countTimes(self):
""" 高频词低频词的提取 """
countDict = Counter(self.flushDict)
listDict = sorted(countDict.items(), key = lambda x: x[1], reverse=True)
print("高频词(Top50):")
self.formatShow(listDict,0,50)
print()
print("低频词(Top50):")
self.formatShow(listDict,-50,len(listDict))
def countChar(self):
""" 计算字数 """
c = 0
for l in self.list:
l = l.strip()
if len(l) == 0:
continue
l = l.split(" ")
for i in range(len(l)):
l[i] = l[i].split("/")[0]
l = l
l = "".join(l)
c += len(l)
print("总字数:",c)
def countArticles(self):
""" 计算文章数 """
c = 0
for l in self.list:
l = l.strip()
if len(l) == 0:
c += 1
print("文章总数:",c+1)
def countSentences(self):
""" 计算句子数 """
with open(self.text, 'r', encoding='utf-8') as f1:
list = f1.read()
word = list.split("。")
print("句子总数:",len(word))
if __name__ == '__main__':
test_text = '199801.txt'
myWord = myWord(text=test_text)
myWord.readDict()
print("读取语料成功!")
print("正在对语料 %s 进行分析..."%test_text)
print()
# myWord.printDict()
print("正在统计字数...")
myWord.countChar()
print()
print("正在统计词数...")
myWord.countWords()
myWord.countWord()
print()
print("正在计算高频词与低频词...")
myWord.countTimes()
print()
print("正在统计文章数...")
myWord.countArticles()
print()
print("正在统计句子数...")
myWord.countSentences()
print()
print("语料分析完毕!", end="")
附件:
199801.txt