中文自然语言处理--Gensim 构建词袋模型

import jieba
from gensim import corpora
import gensim

# 首先，引入 jieba 分词器、语料和停用词。
# 定义停用词、标点符号
punctuation = ["，", "。", "：", "；", "？"]
# 定义语料
content = ["机器学习带动人工智能飞速的发展。",
           "深度学习带动人工智能飞速的发展。",
           "机器学习和深度学习带动人工智能飞速的发展。"]

# 对语料进行分词操作，这里用到 lcut() 方法
# 分词
segs_1 = [jieba.lcut(con) for con in content]
print(segs_1)

# 去停用词和标点符号
tokenized = []
for sentence in segs_1:
    words = []
    for word in sentence:
        if word not in punctuation:
            words.append(word)
    tokenized.append(words)
print(tokenized)

# tokenized是去标点之后的
dictionary = corpora.Dictionary(tokenized)
print(dictionary)
# 查看词典和下标 id 的映射
print(dictionary.token2id)
print(dictionary.dfs)
# 保存词典
dictionary.save('deerwester.dict')

# 得到词袋模型的特征向量
corpus = [dictionary.doc2bow(sentence) for sentence in segs_1]
print(corpus)

原文：
https://soyoger.blog.csdn.net/article/details/108729409