python爬取糗事百科

2015/5/30 21:40测试有效
保留注释，代码仅供参考，浏览器为火狐浏览器

# 利用requests和bs4爬取糗事百科
import requests, bs4

orgUrl = dstUrl = 'https://www.qiushibaike.com/hot/'  # 糗事百科地址
savedBaiKe = open('savedBaiKe.txt', 'a', encoding='utf-8')  # 保存爬取的百科
page = 1 #当前页码
hd = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:67.0) Gecko/20100101 Firefox/67.0'}
while True:
    res = requests.get(dstUrl, headers=hd)
    try:
        res.raise_for_status()
    except Exception as exc:
        print('发生错误：%s' % exc)
    soup = bs4.BeautifulSoup(res.text, 'html.parser')
    # 目标元素 用户名 + 段子 利用continue跳过 有img标签便跳过
    divs = soup.select('div[id*="qiushi_tag_"]')  # type = <class 'list'> len = 0 -> cannot locate the element
    # img.illustration 能在该div下被locate到 就continue
    for i in range(len(divs)):
        imgEl = soup.select('div[id*="qiushi_tag_"]:nth-child(' + str(i) + ') img.illustration')
        if len(imgEl) > 0:
            continue
        uName = soup.select('h2')[i].get_text()  # 用户名
        uContent = soup.select('div.content')[i].select('span')[0].get_text()  # 段子
        savedBaiKe.write(uName + uContent)
    #进入下一页，继续爬
    page = page + 1
    dstUrl = orgUrl + 'page/' + str(page) + '/'
    if dstUrl == 'https://www.qiushibaike.com/hot/page/14/':
        break