jieba分词
ubuntu下安装jieba
pip install jieba
使用
import jieba
import jieba.posseg as psg
content = "张三非常喜欢打篮球,他认为打篮球和有趣."
print("精确分词模式结果:")
segs_1 = jieba.cut(content, cut_all=False)
print("/".join(segs_1))
print("全模式分词模式结果:")
segs_3 = jieba.cut(content, cut_all = True)
print("/".join(segs_3))
print("搜索引擎模式结果:")
segs_4 = jieba.cut_for_search(content)
print("/".join(segs_4))
print("获取分词后的list:")
segs_5 = jieba.lcut(content)
print(segs_5)
print("获取词性")
print([(x.word, x.flag) for x in psg.lcut(content)])
gensim
利用gensim和jieba生成中文词向量
https://blog.csdn.net/shuihupo/article/details/85162237
针对中文文档生成分词得到的每个单词对应的词向量。
import os
import jieba
import jieba.analyse
from gensim.models import Word2Vec
from gensim.models.word2vec import LineSentence
class ChineseW2V:
def __init__(self, filename, outputPath="./output/", size=300, window=10, min_count=1, hs=1):
self.filename = filename # 文件路径
self.outputPath = outputPath # 文件输出路径
self.size = size
self.window = window
self.min_count = min_count
self.hs = hs
self.jieba_cut_outfile = None
def readFile(self):
with open(self.filename, encoding="utf-8") as f:
document = f.read()
document_cut = jieba.cut(document)
result = ' '.join(document_cut)
if not os.path.exists(self.outputPath):
os.mkdir(self.outputPath)
with open(self.outputPath+"jieba_cut.txt", "w", encoding="utf-8") as outf:
outf.write(result)
self.jieba_cut_outfile = self.outputPath+"jieba_cut.txt"
def genEmbedding(self):
self.readFile()
sentences = LineSentence(self.jieba_cut_outfile)
model = Word2Vec(sentences, hs=self.hs, min_count=self.min_count, window=self.window, size=self.size)
model.save("./output/Chinese_word2vec.model")
print("=="*3+"Done"+"=="*3)
if __name__ == '__main__':
filename = './data/三体txt.txt'
genmodel = ChineseW2V(filename)
genmodel.genEmbedding()