本文在mac环境中运行,如果是windows,在字体设置上可能会有区别
完整代码如下:
from src.common import baseClass
import pandas as pd
import jieba
from wordcloud import WordCloud
import matplotlib
# matplotlib.use('TkAgg') OS报错macOS 12 (1207) or later required, have instead 12 (1206) !
matplotlib.use('Qt5Agg')
import matplotlib.pyplot as plt
import collections
import numpy as np
# 获取execl需要的列的数据
def execl_get():
df = pd.read_csv(execl_path) # 读取爬取的数据
data = pd.DataFrame(df)
# 删除重复行:
df1 = data.drop_duplicates()
# print(df1)
return df1
# 将execl数据转化成list
def excel_list(arg):
# data = datatmsp[['title', 'price', 'sale', 'city']]
datatmsp = execl_get()
data = datatmsp[['标题', '价格', '销量', '城市']]
# print(data)
data.head() # 默认查看前5行数据
arg_list = datatmsp[arg].values.tolist()
# title = datatmsp.title.values.tolist() # 转为list
# print(title)
return arg_list
# 对获取的execl数据的list列表进行处理
def filter_same(arg):
# 对每个标题进行分词: 使用lcut函数
title_s = []
title = excel_list(arg)
for line in title:
title_cut = jieba.lcut(line)
title_s.append(title_cut)
# print(title_s)
'''
对 title_s(list of list 格式)中的每个list的元素(str)进行过滤 剔除不需要的词语,
即 把停用词表stopwords中有的词语都剔除掉:
'''
# 导入停用词表:
stopwords = pd.read_excel('/Users/mac/Documents/study23/ios_auto/test_case/stopword.xlsx')
stopwords = stopwords.stopword.values.tolist()
# print(stopwords)
# 剔除停用词:
title_clean = []
for line in title_s:
line_clean = []
for word in line:
# 设置停用词并去除单个词
if word not in stopwords and len(word) > 1:
line_clean.append(word)
title_clean.append(line_clean)
'''
因为下面要统计每个词语的个数,所以 为了准确性 这里对过滤后的数据 title_clean 中的每个list的元素进行去重,
即 每个标题被分割后的词语唯一。
'''
title_clean_dist = []
for line in title_clean:
line_dist = []
for word in line:
if word not in line_dist:
line_dist.append(word)
title_clean_dist.append(line_dist)
return title_clean_dist
# 单词统计
def word_counts():
title_clean_dist = filter_same('标题')
# 将 title_clean_dist 转化为一个list: allwords_clean_dist
allwords_clean_dist = []
for line in title_clean_dist:
for word in line:
allwords_clean_dist.append(word)
# print(allwords_clean_dist)
# 筛选后统计
word_counts = collections.Counter(allwords_clean_dist)
# print(word_counts)
return word_counts
# 对统计的单词添加列名
def df_word_count():
word_count = word_counts()
word_counts_top100 = word_count.most_common(200)
df = pd.DataFrame(word_counts_top100, columns=['word', 'count'])
return df
# 统计的单词列表化
def word_count_list():
df = df_word_count()
data = []
for i in range(0, len(df)):
data.append(df['word'][i])
# print(data)
return data
# 高频词的处理
def high_hang():
word_count = word_counts()
# # 获取前100最高频的词
word_counts_top100 = word_count.most_common(100)
# print(word_counts_top100)
# 使用pandas库中的DataFrame类可以将数据转换为数据框,这是一个二维表格结构
df = pd.DataFrame(word_counts_top100, columns=['word', 'count'])
# print(df)
# df.to_excel('word_count.xlsx', index=False)
# '''
# 观察 word_count 表中的词语,发现jieba默认的词典 无法满足需求:
# 有的词语(如 可用、不可用等)却被cut,这里根据需求对词典加入新词
# (也可以直接在词典dict.txt里面增删,然后载入修改过的dict.txt)
# '''
#
# # add_words = pd.read_excel('add_words.xlsx') # 导入整理好的待添加词语
# add_words = pd.DataFrame(word_count, columns=['word', 'count'])
# # print(add_words)
# # add_words.to_csv('add_words.csv', index=False) # 生成文档
#
# # 添加词语:
# for w in add_words.word:
# jieba.add_word(w, freq=1000)
#
# # 注:再将上面的 分词_过滤_去重_汇总 等代码执行一遍,得到新的 word_count表
#
# # word_count.to_excel('word_count.xlsx', index=False) # 导出数据
return df
# 生成词云可视图
def could_word():
word_count = word_counts()
my_cloud = WordCloud(
background_color='white', # 设置背景颜色 默认是black
width=900, height=600,
max_words=100, # 词云显示的最大词语数量
font_path='/Users/mac/Documents/SourceHanSansCNHeavy.otf', # 设置字体 显示中文(字体需要优先下载到mac 中)
max_font_size=99, # 设置字体最大值
min_font_size=16, # 设置子图最小值
random_state=50 # 设置随机生成状态,即多少种配色方案
).generate_from_frequencies(word_count)
# 显示生成的词云图片
plt.imshow(my_cloud, interpolation='bilinear')
# 显示设置词云图中无坐标轴
plt.axis('off')
plt.savefig('wordcloud.png', dpi=600) # 保存为图片
# plt.show()
# 单词 word对应销量sale的分析:排序图
def key_acount():
word_count = df_word_count()
print(word_count)
top_key = word_count.head(25) # 取排行最前面的25个
index = np.arange(top_key['count'].size)
plt.figure(figsize=(8, 4))
plt.barh(index, top_key['count'], color='indigo', height=0.9) # 显示在y轴
# plt.bar(index, top_key['count'], color='indigo', width=0.8) # 显示在x轴
plt.yticks(index, top_key['word'], fontsize=11, rotation=0)
# 添加数据标签:
for y, x in zip(index, top_key['count']):
plt.text(x, y, '%.0f' % x, ha='left', va='center', fontsize=11)
# plt.grid(True) # 打开网格
plt.xlabel('商品数量')
plt.ylabel('关键词')
plt.title('关键词的商品数量统计')
plt.savefig('key_acount.png', dpi=600) # 保存为图片
plt.show()
# va参数 ('top', 'bottom', 'center', 'baseline') # 数据标签显示的位置
# ha参数('center', 'right', 'left')
if __name__ == '__main__':
# could_word() # 生成词云排序图
wsa() # 关键词销量排序图
运行后,生成如下排序图: