爬虫学习之05-request模块获取某百科下的全部糗图

"""
    获取某百科下的全部糗图:https://www.***baike.com/imgrank/
    代码仅供学习，请勿作其他用途
    date:2020-05-25
"""
import requests
import re
import os

if __name__ == '__main__':
    requests.packages.urllib3.disable_warnings()
    # UA伪装
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36'
    }

    if not os.path.exists('05糗图'):
        os.mkdir('05糗图')

    # 1.指定URL
    url = 'https://www.***baike.com/imgrank/page/%d/'

    for pageNum in range(1,14):
        new_url = format(url%pageNum)
        #2.爬取页面
        page_data = requests.get(url=new_url, headers=headers, verify=False).text
        # print(page_data)

        #3. 使用聚焦爬虫对页面中所有的糗图进行解析
        """
        <div class="thumb">
        
        <a href="/article/123144519" target="_blank">
        <img src="//pic.***baike.com/system/pictures/12314/123144519/medium/763CX35F1320P2G1.jpg" alt="糗事#123144519" class="illustration" width="100%" height="auto">
        </a>
        </div>
        """
        # 用正则解析
        ex = '<div class="thumb">.*?<img src="(.*?)" alt=.*?</div>'
        img_src_list = re.findall(ex, page_data, re.S)

        for src in img_src_list:
            src = 'https:' + src
            img_data = requests.get(url=src, headers=headers, verify=False).content
            # 生成图片名称
            img_name = src.split('/')[-1]
            imgpath = './05糗图/' + img_name

            with open(imgpath, 'wb') as fp:
                fp.write(img_data)
                print(img_name, '下载完成')

运行结果如下：