[语料库的构建] 第一讲 语料库分析方法

实践

对给定语料进行分析，包括：字数（中文）、词数（中文、英文(原词、词干)）、高频词、低频词（Top50），句子个数，文章个数

提示：

中文需要分词，英文需要做词干化处理(Porter算法：nltk.stem.porter.PorterStemmer)
取词干
PorterStemmer.stem(w)
分词
jieba.cut(sentence,cut_all=False,HMM=True)

代码

from collections import Counter

class myWord:
    def __init__(self,text):
        self.dict = []
        self.count = {}
        self.flushDict = []
        self.text = text
        with open(text, 'r', encoding='utf-8') as f1:
            self.list = f1.readlines()


    def readDict(self):

        """ 读取语料生成词典 """

        for l in self.list:
            l = l.strip()
            if len(l) == 0:
                continue
            l = l.split('  ')
            # 不计日期
            for i in range(1,len(l)):
                l[i] = l[i].split('/')[0]
                self.dict.append(l[i])

    def printDict(self):

        """ 打印词典 """

        for i in range(50):
            print(self.dict[i])

    def countWords(self):

        """ 计算词数（未去重） """

        for i in self.dict:
            if len(i)!=1:
                self.flushDict.append(i)
        print("总词数：", len(self.flushDict))

    def countWord(self):

        """ 计算词数（去重） """

        print("去重词数：", len(set(self.flushDict)))

    def formatShow(self,list,st,fn):

        """ 格式化输出（词频） """

        t = 0
        for l in list[st:fn]:
            t += 1
            word = l[0]
            times = l[1]
            print("word:{}  time：{}  ".format(word, times).ljust(15), end="")
            if t % 4 == 0:
                print()
        print()

    def countTimes(self):

        """ 高频词低频词的提取 """

        countDict = Counter(self.flushDict)
        listDict = sorted(countDict.items(), key = lambda x: x[1], reverse=True)
        print("高频词(Top50):")
        self.formatShow(listDict,0,50)
        print()
        print("低频词(Top50):")
        self.formatShow(listDict,-50,len(listDict))

    def countChar(self):

        """ 计算字数 """

        c = 0
        for l in self.list:
            l = l.strip()
            if len(l) == 0:
                continue
            l = l.split("  ")
            for i in range(len(l)):
                l[i] = l[i].split("/")[0]
            l = l
            l = "".join(l)
            c += len(l)
        print("总字数：",c)

    def countArticles(self):

        """ 计算文章数 """

        c = 0
        for l in self.list:
            l = l.strip()
            if len(l) == 0:
                c += 1
        print("文章总数：",c+1)

    def countSentences(self):

        """ 计算句子数 """

        with open(self.text, 'r', encoding='utf-8') as f1:
            list = f1.read()
        word = list.split("。")
        print("句子总数：",len(word))


if __name__ == '__main__':
    test_text = '199801.txt'

    myWord = myWord(text=test_text)
    myWord.readDict()
    print("读取语料成功！")
    print("正在对语料 %s 进行分析..."%test_text)
    print()
    # myWord.printDict()
    print("正在统计字数...")
    myWord.countChar()
    print()
    print("正在统计词数...")
    myWord.countWords()
    myWord.countWord()
    print()
    print("正在计算高频词与低频词...")
    myWord.countTimes()
    print()
    print("正在统计文章数...")
    myWord.countArticles()
    print()
    print("正在统计句子数...")
    myWord.countSentences()
    print()
    print("语料分析完毕！", end="")

附件：
199801.txt