Python爬取toutiao图片保存到本地

import re
import requests
import json
import os
from urllib import request


url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E7%BE%8E%E5%9B%BE&autoload=true&count=20&cur_tab=3&from=gallery'
offset = 0
# url = 'https://www.baidu.com/'

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
# 创建文件夹
if not os.path.exists('toutiao'):
    os.mkdir('toutiao')

# 获取三组ajax数据
for i in range(3):
    # 组合三次不同的url
    url = url.format(offset)
    offset = offset+20
    # 发送请求
    response = requests.get(url, headers=headers)
    # 接收数据并格式化
    data_str = response.text
    # 转成字典拿出有用的数据
    data_dict = json.loads(data_str)['data']
    # print(data)
    # 遍历每组数据
    for i in data_dict:
        # 拿出每组数据中的url
        url_b = i['article_url']
        # 发送请求
        response = requests.get(url_b, headers=headers)
        # 接收数据并格式化
        data_b_text= response.text
        # print(data_b_text)
        # 定义正则
        pat = r'parse\((.*)\),'
        # 匹配正则
        sea = re.search(pat, data_b_text)
        # 获取匹配的数据，并转换成str
        data_b_str =json.loads(sea.group(1))
        # 装换成字典 dict
        data_b_dict = json.loads(data_b_str)
        print(data_b_dict['sub_images'])
        for j in data_b_dict['sub_images']:
            img_url = j['url']
            # 文件路径 split以/分割去最后一个分组，toutiao/xx.jpg
            path = 'toutiao/'+img_url.split('/')[-1]+'.jpg'
            # 下载图片
            request.urlretrieve(img_url,path)