python爬虫实现今日头条街拍爬取

import requests
import re
import json
import os
from urllib import request
headers = {
    'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'
}

url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'
b = input('请输入页码')
c = int(b)*20
for i in range(0,c,20):
    #根据20一个隔断拼接出网页
    print(i)
    xiangxi_url = url.format(i)
    response = requests.get(xiangxi_url,headers=headers)

    re_json = response.content
    re_json = json.loads(re_json)
    #读取网页并转为json字典格式
    a = re_json['data']
    for item in a:
        # print(item)
        #获取json中的网址
        if 'share_url' in item:
            base_url = item['share_url']
            response = requests.get(base_url,headers=headers)
            re_json = response.text
            # with open('beiying.html','wb') as f:
            #     f.write(response.content)
            pattern = r'gallery: JSON\.parse\((.*)\),'
            k = re.search(pattern,re_json)
            #利用正则切割出每一个街拍集
            print(k.group(1))
            if not os.path.exists('download'):
                os.mkdir('download')

            if k:
                json_str = k.group(1)

                json_str = json.loads(json_str)

                json_dict = json.loads(json_str)
                #两次转码读取街拍集网页转为dict格式
                json_dict = json_dict['sub_images']
                # print(json_dict)
                for item in json_dict:
                    # print(item['url'])
                    image_url = item['url']
                    new_item = item['url'].split('/')[-1]
                    print(new_item)
                    file_name = 'download/' + new_item + '.jpg'
                    request.urlretrieve(image_url,file_name)