java 分词词频_文本分析——分词、统计词频、词云

导入包

import os

import sys

import numpy as np

import pandas as pd

import matplotlib.pyplot as plt

import re

from pandas import Series, DataFrame

import string

import re

import jieba

import jieba.analyse

import datetime

from wordcloud import WordCloud, ImageColorGenerator

import codecs

导入文件和数据

gongdan = pd.read_excel('Gongdan.xlsx')

数据预处理

gongdan['content'] = [str(i) for i in gongdan['content']]

gongdan['content'] = [''.join(re.findall(u'[\u4e00-\u9fff]+', i)) for i in gongdan['content']]

indexs = list(gongdan['content'][pd.isnull(gongdan['content'])].index)

gongdan = gongdan.drop(indexs)

indexs = list(gongdan['content'][gongdan['content']==''].index)

gongdan = gongdan.drop(indexs)

content = gongdan['content']

cont = ''.join(content)

cont = ''.join(re.findall(u'[\u4e00-\u9fa5]+', cont))

分词并去除停用词

stopwords = set()

fr = codecs.open('stopwords.txt', 'r', 'utf-8')

for word in fr:

stopwords.add(str(word).strip())

fr.close()

jieba.load_userdict("dict.txt")

text = list(jieba.cut(cont, cut_all=False, HMM=True))

text = list(filter(lambda x: x not in stopwords, text))

text = [str(i) for i in text if i != ' ']

Tfidf 算法

from sklearn import feature_extraction

from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.feature_extraction.text import CountVectorizer

test = ' '.join(text)

tlist = []

tlist.append(test)

vectorizer=CountVectorizer()#该类会将文本中的词语转换为词频矩阵，矩阵元素a[i][j] 表示j词在i类文本下的词频

transformer = TfidfTransformer()#该类会统计每个词语的tf-idf权值

tfidf = transformer.fit_transform(vectorizer.fit_transform(tlist)) #第一个fit_transform是计算tf-idf，第二个fit_transform是将文本转为词频矩阵

word=vectorizer.get_feature_names()#获取词袋模型中的所有词语

weight=tfidf.toarray()#将tf-idf矩阵抽取出来，元素a[i][j]表示j词在i类文本中的tf-idf权重

tfidf_list = {}

for i in range(len(weight)):#打印每类文本的tf-idf词语权重，第一个for遍历所有文本，第二个for便利某一类文本下的词语权重

for j in range(len(word)):

tfidf_list[word[j]] = weight[i][j]

词云

font_path = 'yahei.ttf'

from PIL import Image

back_coloring = np.array(Image.open('circle.jpg'))

wc = WordCloud(font_path=font_path, # 设置字体

background_color="white", # 背景颜色

max_words=60, # 词云显示的最大词数

mask=back_coloring, # 设置背景图片

stopwords=stopwords,

max_font_size=100, # 字体最大值

random_state=42,

width=1000, height=860, margin=2,# 设置图片默认的大小,但是如果使用背景图片的话,那么保存的图片大小将会按照其大小保存,margin为词语边缘距离

# prefer_horizontal=1,

)

wc.generate_from_frequencies(tfidf_list)

plt.imshow(wc, interpolation="bilinear")

plt.axis("off")

plt.figure()

wc.to_file("w.png")

# create coloring from image

image_colors = ImageColorGenerator(back_coloring)

# recolor wordcloud and show

# we could also give color_func=image_colors directly in the constructor

plt.imshow(wc.recolor(color_func=image_colors), interpolation="bilinear")

plt.axis("off")

plt.figure()

plt.imshow(back_coloring, cmap=plt.cm.gray, interpolation="bilinear")

plt.axis("off")

plt.show()