python爬取今日头条街拍_python爬虫实现今日头条街拍爬取

import requests

import re

import json

import os

from urllib import request

headers = {

'User-Agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36'

}

url = 'https://www.toutiao.com/search_content/?offset={}&format=json&keyword=%E8%A1%97%E6%8B%8D&autoload=true&count=20&cur_tab=1&from=search_tab'

b = input('请输入页码')

c = int(b)*20

for i in range(0,c,20):

#根据20一个隔断拼接出网页

print(i)

xiangxi_url = url.format(i)

response = requests.get(xiangxi_url,headers=headers)

re_json = response.content

re_json = json.loads(re_json)

#读取网页并转为json字典格式

a = re_json['data']

for item in a:

# print(item)

#获取json中的网址

if 'share_url' in item:

base_url = item['share_url']

response = requests.get(base_url,headers=headers)

re_json = response.text

# with open('beiying.html','wb') as f:

# f.write(response.content)

pattern = r'gallery: JSON\.parse\((.*)\),'

k = re.search(pattern,re_json)

#利用正则切割出每一个街拍集

print(k.group(1))

if not os.path.exists('download'):

os.mkdir('download')

if k:

json_str = k.group(1)

json_str = json.loads(json_str)

json_dict = json.loads(json_str)

#两次转码读取街拍集网页转为dict格式

json_dict = json_dict['sub_images']

# print(json_dict)

for item in json_dict:

# print(item['url'])

image_url = item['url']

new_item = item['url'].split('/')[-1]

print(new_item)

file_name = 'download/' + new_item + '.jpg'

request.urlretrieve(image_url,file_name)