bag_words 练习

import jieba
import pandas as pd
from gensim import corpora
import os
from itertools import chain
from collections import Counter
#返回一个总的列表
def get_list(root):
    stopwords=pd.read_csv("C:/stopwords.txt",index_col=False,quoting=3,sep="\t",names=['stopword'], encoding='utf-8')
    # quoting=3全不引用
    stopwords=stopwords['stopword'].values
    paths=[]   
    try:
        dirs=os.listdir(root)
        print(dirs)
        for dirx in dirs:
            paths.append(root+'/'+dirx)
            print(root+'/'+dirx)
    except:
        print('error')
    all_list=[]
    for path in paths:
        df = pd.read_csv(path,encoding='gbk')
        df = df.dropna()#删除缺失数据
        content=df.content.values.tolist()
        sentence=[]
        #分词
        for line in content:
            try:
                segs=jieba.lcut(line)
                segs = list(filter(lambda x:len(x)>1, segs))
                segs = list(filter(lambda x:x not in stopwords, segs))
                sentence.append(segs)
            except:
                print(line)
                continue
        sentence=list(chain.from_iterable(sentence))#转换为一维的列表  
        all_list.append(sentence)
    return all_list

#写入文件，返回词袋模型，格式：词，序号，词频
def write(all_list):
    root="C:/NLP_project/NLP_project/result/"
    corpus = set(list(chain.from_iterable(all_list)))
    print(len(corpus))
    corpus_dict = dict(zip(corpus, range(len(corpus))))
    vector=[]
    i=0 #计数
    for sentence in sentences:
        line=root+str(i)+'.txt'
        f=open(line,'w+')
        vec = []
        for key in corpus_dict.keys():
            if key in sentence:
                f.write(key+','+str(corpus_dict[key])+','+str(sentence.count(key))+'\n')
                vec.append((corpus_dict[key], sentence.count(key)))
            else:
                vec.append((corpus_dict[key], 0))
                f.write(key+','+str(corpus_dict[key])+','+'0'+'\n')
        vec = sorted(vec, key= lambda x: x[0]) #利用第一个元素进行排序
        vector.append(vec)
        f.close
        i=i+1
    return vector

sentences=get_list("C:/NLP_project/NLP_project/train")
vec=write(sentences)