爬取豆瓣读书-用户所有阅读书籍名称、日期和书籍链接（Python爬虫实战）

获取用户的所有阅读的书籍、日期和链接，是对豆瓣用户画像刻画的基础，本文介绍如何利用BeautifulSoup框架对豆瓣读书用户阅读信息进行抓取。

import requests
from bs4 import BeautifulSoup
import re

headers = {
    "Host": "book.douban.com",
    "Referer":"https://www.douban.com/people/fayolee/collect",
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}

def GetBookList(link):
    res=requests.get(link,headers=headers)
    res.encoding='utf-8'
    soup=BeautifulSoup(res.text)
    #时间
    dates=[]
    for date in soup.select('span[class="date"]'):
         dates.append(str(date.text).replace('\n','').replace('读过','').replace(' ',''))
    #书名
    books=[]
    for book in soup.select('a[title]'):
        books.append(str(book.text).replace('\n','').replace(' ',''))

    soup=BeautifulSoup(res.text,"lxml")
    #书链接
    urls=[]
    for book_url in soup.find_all('a',class_='nbg'):
        url=str(book_url).split(" ")[2].replace("href","").replace("=","").replace('"','')
        urls.append(url)
    return books,urls,dates

#获取页数
def get_pageNum():
    res=requests.get("https://book.douban.com/people/fayolee/collect",headers=headers)
    res.encoding='utf-8'
    soup=BeautifulSoup(res.text)
    nums=soup.select('h1')
    pageNum=re.findall('\d+',str(nums).replace("h1",""))[0]
    return int(int(pageNum)/15)+1

if __name__=="__main__":
    start_link = 'https://book.douban.com/people/fayolee/collect?start='
    depth =get_pageNum()#页数
    for i in range(depth):
        link= start_link + str(15*i)
        books,urls,dates=GetBookList(link)
        print(books)
        print(dates)
        print(urls)