python爬取小说以及数据分析

最新推荐文章于 2023-12-27 07:30:00 发布

じ耐人

最新推荐文章于 2023-12-27 07:30:00 发布

阅读量435

点赞数

文章标签： python 数据分析开发语言

本文链接：https://blog.csdn.net/qq_55894976/article/details/130279800

版权

from pyecharts import options as opts
from pyecharts.charts import Bar, Grid, Line, Liquid, Page, Pie
from pyecharts.commons.utils import JsCode
from pyecharts.components import Table
from pyecharts.faker import Faker

import requests
from bs4 import BeautifulSoup
import csv
header={"User-Agent":"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Mobile Safari/537.36 Edg/112.0.1722.48"}
for i in range (1):
    URL="https://www.qidian.com/rank/readindex/page{}/".format(i)
    resp=requests.get(URL)
    soup=BeautifulSoup(resp.content,"html.parser")
    big_div=soup.find("div",attrs={"class":"rank-body"})


#名字
name=[]
names=big_div.find_all("h2")
for i in names:
    n=i.text
    name.append(n)
    # print(name)
#小说类型
lx=[]
lxs=big_div.find_all("a",attrs={"data-eid":"qd_C42"})
    # print(lxs)
for i in lxs:
    l=i.text
    lx.append(l)
#数据清洗，变成[(”a","b")]格式
tuples=dict(zip(name,lx))
my_list = list(zip(tuples.keys(),tuples.values()))
# print(my_list)

#将数据写入csv中
with open("xiaoshuo.csv.csv","w",encoding="utf-8",newline="") as file:
    f=csv.writer(file)
    con=["名字","类型"]
    f.writerow(con)
    f.writerows(my_list)

#作图

#去掉类型中的第一行 获得所有类型
lx2=[]

with open("xiaoshuo.csv.csv","r",encoding="utf-8") as file:
    bb=csv.reader(file)
    for i in bb:
        # print(i[1])
        if i[1] not in lx:
            pass
        else:
            lx2.append(i[1])
# print(lx2)
#获得类型的数目
def getcount(shujv):
    count=0
    for i in lx2:
        if i== shujv:
            count+=1
        else:
            continue
    return count
a=getcount("奇幻")
b=getcount("武侠")
c=getcount("都市")
d=getcount("诸天无限")
e=getcount("仙侠")
f=getcount("游戏")
g=getcount("军事")
h=getcount("悬疑")
i=getcount("轻小说")
j=getcount("体育")
k=getcount("历史")
l=getcount("科幻")
m=getcount("玄幻")
#获得所有类型的种类 将类型列表转换成集合
# print(set(lx))
shuliang=[a,b,c,d,e,f,g,h,i,j,k,l,m]
leixing=["奇幻","武侠","都市","诸天无限","仙侠","游戏","军事","悬疑","轻小说","体育","历史","科幻","玄幻"]
tupless=dict(zip(leixing,shuliang))
# print(tupless)
# print(a+b+c+d+e+f+g+h+i+k+j+l+m)

# 画图

def bar_datazoom_slider() -> Bar:
    c = (
        Pie()
        .add("", [list(z) for z in zip(tupless.keys(), tupless.values())])
        .set_colors(
            ["blue", "green", "yellow", "red", "pink", "orange", "purple", "DoderBlue", "Auqamarin", "	Honeydew",
             "BrulyWood", "Maroon", "Tomato", "DarkKhaki"])
        .set_global_opts(title_opts=opts.TitleOpts(title=""))
        .set_series_opts(label_opts=opts.LabelOpts(formatter="{b}: {c}"))
    )
    return c
def table_base() -> Table:
    table = Table()

    headers = ["type", "count"]
    rows = [
        ["奇幻", 13],
        ["武侠", 3],
        ["都市", 54],
        ["诸天无限", 16],
        ["仙侠", 47],
        ["游戏", 24],
        ["军事",2],
        ["悬疑",4],
        ["轻小说",72],
        ["体育",4],
        ["历史",21],
        ["科幻",30],
        ["玄幻",39]
    ]
    table.add(headers, rows)
    return table

page = Page()
page.add(
    table_base()
    )
page.render("page.html")

可以生成两个html 然后将两个htnl整合在一起

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
  <head>
    <title>all.html</title>
 
  </head>
  
  <frameset rows="300%,*" frameborder="no">
<!--  		<frame src="/HTML/example/top.html"/>-->
  		
  		<frameset cols="15%,*">
  				<frame src="page.html" noresize="noresize" />
  				
  				<!-- frame里面有一个name属性，用来表名属性来进行超链接替换 -->
  				<frame src="page1.html" noresize="noresize" name="right"/>
  		</frameset>
  		
  </frameset>
  
</html>

运行结果