Python3之爬虫爬取豆瓣读书Top250

import requests
from bs4 import BeautifulSoup
from openpyxl import Workbook

excel_name = "豆瓣读书.xlsx"
wb = Workbook()
ws1 = wb.active
ws1.title = '豆瓣读书'


def get_html(url):
    header = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'}
    html = requests.get(url, headers=header).content
    return html


def get_con(html):
    soup = BeautifulSoup(html, 'html.parser')
    # 获取文章列表
    book_list = soup.find('div', attrs={'class': 'article'})
    # 获取当前页数
    page = soup.find('div', attrs={'class': 'paginator'})
    # 获取下一页
    next_page = page.find('span', attrs={'class': 'next'}).find('a')
    name = []
    for i in book_list.find_all('table'):
        # 找到所有的table标签，也就是每一个书目
        book_name = i.find('div', attrs={'class': 'pl2'})
        # m是获取a标签中的字符串，加入列表
        m = list(book_name.find('a').stripped_strings)
        # print(m)
        if len(m) > 1:
            x = m[0] + m[1]
            # print(x)
        else:
            x = m[0]
            # print(x)
        # print(x)
        name.append(x)
    if next_page:
        return name, next_page.get('href')
    else:
        return name, None


def main():
    url = 'https://book.douban.com/top250'
    name_list = []
    while url:
        html = get_html(url)
        name, url = get_con(html)
        name_list = name_list + name
    for i in name_list:
        location = 'A%s' % (name_list.index(i) + 1)
        # print(i)
        # print(location)
        ws1[location] = i
    wb.save(filename=excel_name)


if __name__ == '__main__':
    main()

别处看到的代码，写的不错，研究了一下