利用停顿词进行分词

只是为了记录避免忘记

import pandas as pd
import jieba

"""
去停用词stopwords
:param list content_lines: 将被处理文本的data
:param list new_content_lines: 存储处理好的data，默认=[]
:return list new_content_lines:  存储处理好的data
"""
def preprocess_text(content_lines, new_content_lines=[]):
    stopwords = pd.read_csv("data/stopwords.txt", index_col=False, quoting=3, sep='\t', names=['stopword'], encoding='utf-8')
    stopwords = list(stopwords['stopword'].values)
    
    for line in content_lines:
        segs_line = jieba.lcut(line)
        new_line = [word for word in segs_line if word not in stopwords]
        new_content_lines.append(new_line)
    return new_content_lines

# 数据准备
df_sports = pd.read_csv("./data/sports_news.csv", encoding='utf-8')
df_sports = df_sports.dropna()
sports = df_sports.content.values.tolist()[:200] # 取前200条数据。实际中可用20000条数据

# 数据处理
sentences = []
preprocess_text(sports, sentences)

# test
print(sentences)

运行结果

[ ['中新网', '深圳', '日电', ' ', '郑小红', '2019', '深圳'（此处省略）],
  ['王蔷', '岁', '天津', '纯真', '性格', '颇受欢迎',（此处省略）],
  ['寒冷', '冬季', '并未', '挡住', '奔跑', '热情'', （此处省略）],
  ......（此处省略两万字）......

参考连接：参考的博客