豆瓣读书数据分析实战

数据爬取请参考:
Python分布式爬虫实战 - 豆瓣读书

本次分析内容:

1. 分析所有书籍评分情况
2. 热门书籍TOP20
3. 书名高频词汇
4. 作者出版书数量TOP20
5. 每年出版书籍数量分布
6. 热评作者TOP20
7. 每年出版最受欢迎的类别
8. 书籍最多的分类TOP20
9. 热评分类TOP20

为了每段代码都可复制直接使用,所以每段代码都重复导入一次pyecharts 模块…

# 首先导入需要用到的模块
import pandas as pd
import numpy as np
import pymysql,re
import matplotlib.pyplot as plt
%matplotlib inline

• 读取数据库数据

conn = pymysql.connect("localhost","root","123456","douban")
douban_data = pd.read_sql("select * from information",conn)

• 去重

douban_data.drop_duplicates("name",keep="first",inplace=True)

1.分析所有书籍评分情况

• 对评分进行分组,并对评分进行统计

score_count = douban_data.groupby("score")["score"].count()

• 散点图

from pyecharts.charts import Scatter
from pyecharts import options as opts
from pyecharts.globals import ThemeType
# 对评分进行分组,并对评分进行统计
score_count = douban_data.groupby("score")["score"].count()
# 创建散点图对象,并设置主题模式为ESSOS
scatter = Scatter(init_opts=opts.InitOpts(theme=ThemeType.ESSOS))
# 添加x轴数据,要求类型为python列表类型
scatter.add_xaxis(score_count.index.tolist())
# 添加标签名和y轴数据类型同上
scatter.add_yaxis("score rating",score_count.values.tolist())
# 全局配置
scatter.set_global_opts(
    # 配置x轴为数值轴(默认为cate类型轴)
    xaxis_opts=opts.AxisOpts(type_="value"),
    # 添加可拖动滚动条
    datazoom_opts=opts.DataZoomOpts(is_show=True)
)

# 系列配置
scatter.set_series_opts(
    # 设置不显示每个点上的数字
    label_opts=opts.LabelOpts(is_show=False),
    # 设置3条虚线 分别是最大值,最小值,平均值
    markline_opts=opts.MarkLineOpts(data=[
        opts.MarkLineItem(type_="max",name="最大值"),
        opts.MarkLineItem(type_="min",name="最小值"),
        opts.MarkLineItem(type_="average",name="平均值")
])
)
# 生成图像,可以使用render(patch="c:\123.html") 生成HTML文件
scatter.render_notebook()

• 使用柱状图展示更为清晰

from pyecharts.charts import Bar
from pyecharts import options as opts
from pyecharts.globals import ThemeType
# 这次配置不一样的主题
bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.CHALK))
# 添加x轴数据
bar.add_xaxis(score_count.index.tolist())
# 添加y轴数据(同样是列表类型)
bar.add_yaxis("score rating",score_count.values.tolist())
bar.set_global_opts(
    #　不多作解释了，就是开启图片最底下可以拖动的那个滚动条
    datazoom_opts=opts.DataZoomOpts(is_show=True)
)
bar.set_series_opts(
    label_opts=opts.LabelOpts(is_show=False),
    markline_opts=opts.MarkLineOpts(data=[
        opts.MarkLineItem(type_="max",name="最大值"),
        opts.MarkLineItem(type_="min",name="最小值"),
        opts.MarkLineItem(type_="average",name="平均值")
])
)
bar.render_notebook()

可以看出评分大部分集中在8.0-8.9之间

2.热门书籍TOP20

• 柱状图展示,按热门评论排行

 # 准备数据
datum = douban_data.loc[:,["name","score","rating_count","comment_count"]]
temp = datum.sort_values("rating_count",ascending=False)[:20].sort_values("rating_count",ascending=True)
# 热评TOP20的评分 柱状图
from pyecharts.charts import Bar,Parallel,Grid
from pyecharts import options as opts
from pyecharts.globals import ThemeType
bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.PURPLE_PASSION))
bar.add_xaxis(temp.name.tolist())
bar.add_yaxis("Score",temp.score.tolist())
bar.set_global_opts(
    title_opts=opts.TitleOpts(title="评分Top20"),
    legend_opts=opts.LegendOpts(pos_left="20%")
)
bar.set_series_opts(
    label_opts=opts.LabelOpts(position="right")
)
bar.reversal_axis()
bar.render_notebook()

• 平行多维曲线图

from pyecharts.charts import Parallel
from pyecharts import options as opts
import random
# 准备数据
data = [i.tolist() for i in temp.iloc[:,1:].values]
schemas = [{"dim":i,"name":name} for i,name in enumerate(temp.columns[1:])]
# 创建多维曲线对象
parallel = Parallel(init_opts=opts.InitOpts(theme=ThemeType.CHALK))
parallel.add_schema(schema=schemas)
parallel.add(series_name="name",data=data)
parallel.set_global_opts(
    title_opts=opts.TitleOpts(title="Hot 20")
)
parallel.render_notebook()

3.书名高频词汇

import jieba,re
from wordcloud import WordCloud
from PIL import Image
from collections import Counter
# 所有书名合并成一条字符串
names = "".join(datum.name)
# 按非字符分割字符串,目的是去除字符以外的标点符号
names = "".join(re.split("\W",names))
# 分词
split_name = jieba.lcut(names)
# 为了更好的展示效果,这里把长度为1的词过滤掉
split_name = [i for i in split_name if len(i) > 1]

# 用空格分隔开每个词语
words = " ".join(split_name)
# 加载图片
img = np.array(Image.open(r"e:/files/11.jpg"))
w = WordCloud(
    # 注意中文需要配置字体
    font_path=r"e:/files/PingFang.otf",
    # 指定生成图片形状
    mask=img
)
# 生成词云
w.generate(words)
# 配置图像大小
plt.figure(figsize=(15,15),dpi=80)
# 关闭轴显示
plt.axis("off")
# 在 jupyter notebook中展示
plt.imshow(w)

• 最总效果
  
• 柱状图展示高频词TOP20

count_name = Counter(split_name)
word_top20 = count_name.most_common(20)
# 柱状图
x = [x[0] for x in word_top20]
y = [y[1] for y in word_top20]
top_word_bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.CHALK))
top_word_bar.add_xaxis(x)
top_word_bar.add_yaxis("top20",y)
top_word_bar.render_notebook()

4.作者出版书数量TOP20

# 复制数据
datum = douban_data.loc[:,["author","name"]]
# 删除空值
datum = datum.loc[datum["author"].notnull()]
# 按作者名称分组,统计数量,倒序排序,取前20名
temp = datum.groupby("author").count().sort_values("name",ascending=False)[:20]
# 这里再进行排序是为了图片展示时最大值排第一个,最小值排最后一个,实属为了展示效果
temp.sort_values("name",inplace=True)
# 柱状图(除了要更换数据,基本绘图代码没什么变化)
publish_bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.DARK))
publish_bar.add_xaxis(temp.index.tolist())
publish_bar.add_yaxis("publish",temp.name.values.tolist())
publish_bar.reversal_axis()
publish_bar.set_series_opts(
    # 每条柱子显示数值配置
    # 位置可选
    # top，left，right，bottom，inside，insideLeft，insideRight
    # insideTop，insideBottom， insideTopLeft，insideBottomLeft
    # insideTopRight，insideBottomRight
    label_opts=opts.LabelOpts(position="right")
)
publish_bar.render_notebook()

5.每年出版书籍数量分布

# 老规矩,先复制需要的数据
datum = douban_data.loc[:,["name","date"]]
# 删除空值数据
datum = datum.loc[datum["date"].notnull()]
# 按年份分组,排序,取前50个数据
temp = datum.groupby("date").count().sort_values("name",ascending=False)[:50]
# 按年份排序
temp.sort_values("date",inplace=True)
# 绘制曲线图
from pyecharts.charts import Line
publish_book_line = Line(init_opts=opts.InitOpts(theme=ThemeType.ESSOS))
publish_book_line.add_xaxis(temp.index.tolist())
publish_book_line.add_yaxis("Quantity for every year",temp.name.values.tolist())
publish_book_line.set_global_opts(
    title_opts=opts.TitleOpts(title="Publish books of year"),
    datazoom_opts=opts.DataZoomOpts(is_show=True),
)
publish_book_line.set_series_opts(
    markline_opts=opts.MarkLineOpts(data=[
        opts.MarkLineItem(type_="max",name="最大值"),
        opts.MarkLineItem(type_="min",name="最小值"),
        opts.MarkLineItem(type_="average",name="平均值")
])
)
publish_book_line.render_notebook()

6.热评作者TOP20

# 复制数据
datum = douban_data.loc[:,["score","author","rating_count"]]
# 删除作者为空的数据
datum = datum.loc[datum["author"].notnull()]
# 删除评分为空的数据
datum = datum.loc[datum["score"].notnull()]
# 按作者名称分组取评分均值
temp = datum.groupby("author").mean()
# 删除评分均值为空的行
temp = temp.loc[temp.score.notnull()]
top20 = temp.sort_values("rating_count",ascending=False)[:20]
top20.sort_values("rating_count",ascending=False)
# 柱状图
hot_rating_bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.PURPLE_PASSION))
hot_rating_bar.add_xaxis(top20.index.tolist())
hot_rating_bar.add_yaxis("Rating agerage",top20.rating_count.round().tolist())

hot_rating_bar.set_series_opts(
    # 标签的位置。可选
    # top，left，right，bottom，inside，insideLeft，insideRight
    # insideTop，insideBottom， insideTopLeft，insideBottomLeft
    # insideTopRight，insideBottomRight
    label_opts=opts.LabelOpts(position="insideRight")
)
hot_rating_bar.reversal_axis()
hot_rating_bar.render_notebook()

7.每年出版最受欢迎的类别

# 复制数据
datum = douban_data.loc[:,["date","rating_count","type","name"]]
# 删除年份和总评论空值
datum.dropna(axis=0,inplace=True)
temp = datum.groupby("date").max().sort_values("rating_count",ascending=False)[:20]
temp.sort_values("date",ascending=False)

8.书籍最多的分类TOP20

# 复制资料
datum = douban_data.loc[:,["type","name"]]
temp = datum.groupby("type").count().sort_values("name",ascending=False)[:20]
# 柱状图
from pyecharts.charts import Bar
from pyecharts import options as opts
from pyecharts.globals import ThemeType
type_top20_bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.DARK))
type_top20_bar.add_xaxis(temp.index.tolist())
type_top20_bar.add_yaxis("Most books categories",temp.name.values.tolist())
type_top20_bar.set_global_opts(
    datazoom_opts=opts.DataZoomOpts(is_show=True)
)
type_top20_bar.set_series_opts(
    # 关闭标签显示
    label_opts=opts.LabelOpts(is_show=False),
    # 设置标记选项
    markpoint_opts=opts.MarkPointOpts(
        data=[
            opts.MarkPointItem(type_="max",name="最大值"),
            opts.MarkPointItem(type_="min",name="最小值"),
            opts.MarkPointItem(type_="average",name="平均值")
        ]
    )
)
type_top20_bar.render_notebook()

9.热评分类TOP20

# 复制资料
datum = douban_data.loc[:,["type","rating_count"]]
# 删除空值
datum.dropna(inplace=True)
temp = datum.groupby("type").mean().round(2).sort_values("rating_count",ascending=False)[:20]
# 制图
from pyecharts.charts import Bar
from pyecharts import options as opts
from pyecharts.globals import ThemeType
rating_top20_bar = Bar(init_opts=opts.InitOpts(theme=ThemeType.ESSOS))
rating_top20_bar.add_xaxis(temp.index.tolist())
rating_top20_bar.add_yaxis("Hot type top 20",temp.rating_count.tolist())
rating_top20_bar.set_global_opts(
    title_opts=opts.TitleOpts(title="热评分类TOP20"),
    datazoom_opts=opts.DataZoomOpts(is_show=True)
)
rating_top20_bar.set_series_opts(
    markline_opts=opts.MarkLineOpts(data=[
        opts.MarkLineItem(type_="max",name="最大值"),
        opts.MarkLineItem(type_="min",name="最小值"),
        opts.MarkLineItem(type_="average",name="平均值")
    ])
)
rating_top20_bar.render_notebook()

最后总结:
1.大部分书籍评分都在8.0-8.9之间
2.评论最多的书包括追风筝人、解忧的杂货店、白夜行等,评分都在8.1分以上
3.书名中高频热词前3甲中国、世界、艺术
4.出书前3甲第一名是加拿大人士亦舒先生,出书243本(可能还不止),五体投地啊!
5.可以看出每年出书量都在稳步上涨
6.作者热评前3甲:[清]曹雪芹著/高鹗续、柴静、[美]卡勒德·胡赛尼
7.最多书籍的分类前3:建筑,诗歌,数学
8.评论最高实属中国文学类
最后附上数据库文件
链接：https://pan.baidu.com/s/19VNdu7n8F7JpuqXQYPbPDg
提取码：nlv5