豆瓣读书爬虫

搭建环境
import scrapy
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
加载scrapy框架
cd … # 跳转到上一层目录
scrapy startproject booktop
setting配置
ROBOTSTXT_OBEY = False # 君子协议 USER_AGENT = ‘Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36’ DOWNLOAD_DELAY = 0.5
spider编写

import scrapy

from booktop.items import BookItem

class BookSpider(scrapy.Spider):
name=“bookspider”
allowed_domains=[‘book.douban.com’]
start_urls[‘https://book.douban.com/top250’]
def parse(self,response, **kwargs):
print(response.text) # 测试页面

测试
cd booktop # 进入项目文件夹
scrapy crawl bookspider
获取数据（书名+出版社)

def parse(self, response, **kwargs):
print(response.text)
tables=response.xpath(’//table’)
print(‘书籍个数’,len(tables))
print(tables)
for t in tables: # 提取
extract()[0] tit=t.css(‘div.pl2 a::attr(title)’).extract()[0] #
print(title) 书名 pu=t.css(‘p.pl::text’).extract()[0]
pu=pu.split(’/’)[-3].strip() #
print(pub)
yield BookItem(title=tit,pub=pu)
需要使用item对象完成数据封装并传输
class BookItem(scrapy.Item):
title = scrapy.Field()
pub=scrapy.Field()
pass
pipeline 管道存储数据
ITEM_PIPELINES = { ‘booktop.pipelines.BooktopPipeline’: 300,
}
数据存储到txt文件下
class BooktopPipeline:
def process_item(self, item, spider):
file=open(‘result.txt’,‘a+’,encoding=‘utf-8’)
file.write(item[‘title’]+’,’+item[‘pub’]+’\n’)
return item
分析和可视化

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib #
font = {‘family’: ‘microsoft yahei’,
‘weight’: ‘bold’,
‘size’: 12}
matplotlib.rc(‘font’,**font) df=pd.read_csv(‘result.txt’,names=[‘title’,‘pub’])
print(df) # 福尔摩斯探案集 出版社有问题，手动修改
df.loc[8,‘pub’]=‘群众出版社’
print(df) # 按出版社不同分类汇总书数量，取出前5名 result=df[‘pub’].value_counts().head()
print(result) plt.pie(result,labels=result.index,autopct=’%3.1f%%’)
plt.show()