以下是所有的代码,如果想要改变爬取的书本类型可以在测试函数当中修改
# -*-coding:utf-8 -*-
# 豆瓣网爬书参考代码 日期:2018-08-19 参考代码
import sys
import time
from urllib.parse import quote
import requests
import numpy as np
from bs4 import BeautifulSoup
from openpyxl import Workbook
#User Agents
hds = [{
'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.1; enUS; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'},
{
'User-Agent': 'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/ 535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11'},
{
'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)'}]
#*********爬书虫函数**********
def book_spider(book_tag):
page_num = 0 # 页数
book_list = [] # 爬到的书籍列表
try_times = 0 # 尝试次数
while (page_num < 1): #只爬一页
# url='http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0' # 测试用
#每类爬15本书
url = 'http://www.douban.com/tag/' + quote(book_tag) + '/book?start=' + str(page_num*15)
time.sleep(np.random.rand() * 1)
#开始爬取
try:
req = requests.request('get',url, headers=hds[page_num % len(hds)])
source_code =req.text
plain_text = str(source_code)
excep