网络爬虫--头条街拍美女

import re
import requests
import json
import os
from urllib import request

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
# 个人主页url
url = 'https://www.toutiao.com/a6584980450106671620/#p=4'

response = requests.get(url,headers=headers)

# with open('jiepai,html','wb') as f:
#     f.write(response.content)

html_str = response.text

# 正则匹配
pattern = r'gallery: JSON\.parse\((.*)\),'

match_res = re.search(pattern,html_str)

# 新建文件夹
if not os.path.exists('download'):
    os.mkdir('download')
if match_res:
    # print(match_res.group(1))
    json_org = match_res.group(1)
    res_jj = json.loads(json_org)
    print(res_jj)
    print(type(res_jj))
    res_dict = json.loads(res_jj)
    # print(res_dict)
    # print(type(res_dict))

    sub_images_list = res_dict['sub_images']
    for image in sub_images_list:
        image_url = image['url']
        filename = 'download/' + image_url.split('/')[-1] + '.jpg'
        # 下载图片
        request.urlretrieve(image_url,filename)
        print(image_url)
else:
    print('下载失败')