data-understan

这篇文章是做数据处理时输出的数据

第一列：词（包括n-gram); 第二列：DF，第三列：全部文章中的 total TF； 第四列：total TF/全部文章不去重的总字数，第五列：第四列/DF； 第六列：total TF/该词出现的文章中不去重的总字数；第7列：第六列/DF

from collections import defaultdict

def get_count(fPath):
    invertedIndex = defaultdict(list)
    docNumber = 0
    text=[]
    with open(fPath, 'r',encoding='utf-8') as f:
        line = f.readline()
        while line:
            line = line.strip('\n').split(' ')#这里输出的line是切好词的list
            text.append(line)
            lengthOfDocument = len(line) # 读出文章的长度，也就是每个line的长度
            docNumber += 1 # 计算文档索引，也就是line的索引
            if len(line) == 0:# 文本中的空行也要读取。在我看来有点多余。
                line = f.readline()
                continue
            docIndex = line[0] # 文章标签
            
            for term in set(line):
                count = line.count(term) #计算每行中去重单词的个数
                invertedIndex[term].append([docIndex,count,lengthOfDocument]) 
            line = f.readline()
    f.close()

    # print(invertedIndex) # 输出的是文章标签、词对应的在文章中的词频、文章的长度
    
    countTime = defaultdict(list)
    # df_dict = defaultdict(list)
    for k, v in invertedIndex.items():
        frequence = 0
        NumberOfDocuWord = 0
        for i in range(len(v)):
            frequence += v[i][1] # 计算词在整个文章中的词频
            NumberOfDocuWord += v[i][2]  # 计算词出现的文章的文章长度之和
        countTime[k] = [len(v),frequence, NumberOfDocuWord] # len(v) 代表词出现在了多少篇文章当中
    # print(countTime)
    return invertedIndex,countTime,text

#这个function是将词写到txt文件里
def search_word(countTime,text2,total_words):
    with open('data_understanding.csv','w',newline='',encoding='utf_8_sig') as fw:
        df=[]
        total_tf=[]
        total_tf_total_Word=[]
        total_word_in_document=[]
        for searchword in text2:
            if searchword in countTime:
                df.append(countTime.get(searchword)[0]/2060)
                total_tf.append(countTime.get(searchword)[1])
                total_tf_total_Word.append(countTime.get(searchword)[1]/len(total_words))
                total_word_in_document.append(countTime.get(searchword)[1]/countTime.get(searchword)[2])
        fw.write('word' + ',' + 'DF' + ',' + 'total TF' + ',' + 'totalTF/total_Word' + ',' + '4/DF' + ',' + 'total TF/total_word_in_document' + ',' + '6/DF'+'\n')
        for i in range(len(text2)):
            fw.write(text2[i]+','+str(df[i])+','+str(total_tf[i])+','+str(total_tf_total_Word[i])+',' +str(total_tf_total_Word[i]/df[i])+','+str(total_word_in_document[i])+','+ str(total_word_in_document[i]/df[i])+'\n')
    fw.close()