爬取起点推荐总榜前200本(requests)

import requests
from bs4 import BeautifulSoup



def get_book():
    re=requests.get(url,headers=headers)
    soup=BeautifulSoup(re.text,'html.parser')
    for i in range(1,21):
        title=soup.select(f'#book-img-text > ul > li:nth-child({i}) > div.book-mid-info > h2 > a')[0].text
        introduction=soup.select(f'#book-img-text > ul > li:nth-child({i}) > div.book-mid-info > p.intro')[0].text
        author=soup.select(f'#book-img-text > ul > li:nth-child({i}) > div.book-mid-info > p.author > a.name')[0].text
        category=soup.select(f'#book-img-text > ul > li:nth-child({i}) > div.book-mid-info > p.author > a.go-sub-type')[0].text
        print(f'{i+num}. {title}\t作者:{author}\t{category}\n{introduction}')
        print('----------------------------------')


if __name__ == '__main__':
    headers = {
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36"
    }
    for j in range(1,11):
        url = f'https://www.qidian.com/rank/recom/datetype3/page{j}/'
        num=(j-1)*20
        get_book()

先爬取源码，再用beautifulsoup解析，读者也可以用lxml里的etree来解析，不过在新版本中，etree模块被整合到html里面去了，所以要这么导入：

from lxml import html

etree=html.etree

下方的html里面的etree也许会高亮，但是是没有关系的。