获取街拍页面
import requests
from beiying import tou
# url
url = 'http://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'
# 请求
response = requests.get(url)
json_dict = response.json()
# print(json_dict)
data = json_dict['data']
# print(data)
for item in data:
if 'article_url' in item:
article_url = item['article_url']
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'
}
title = item['title']
# print(title)
tou(article_url,title)
图片下载封装函数
import re,json,os
import requests
from urllib import request
path = ''
def tou(url,title):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.84 Safari/537.36'
}
response = requests.get(url,headers=headers)
html = response.text
pattern = r'gallery: JSON\.parse\((.*)\),'
# 新建文件夹
if not os.path.exists('jiepai'):
os.mkdir('jiepai')
pattern_res = re.search(pattern,html)
if pattern:
pattern_res_html = pattern_res.group(1)
pattern_res_html = json.loads(pattern_res_html)
res_dict = json.loads(pattern_res_html)
sub_images = res_dict['sub_images']
# print(sub_images)
for i in sub_images:
image_url = i['url']
# 自动建立相应文件夹
path = 'jiepai/' + title + '/'
if not os.path.exists(path):
os.makedirs(path)
filename = path + image_url.split('/')[-1] + '.jpg'
# 下载图片
request.urlretrieve(image_url,filename)