python爬取今日头条街拍_python小白爬取分析今日头条街拍美图中获取Json数据列表超出范围问题...

[Python] 纯文本查看 复制代码import json

import re

import requests

from requests.exceptions import RequestException

from urllib.parse import urlencode

from bs4 import BeautifulSoup

def get_page_index(offset,keyword):

data={

'offset': offset,

'format': 'json',

'keyword':keyword,

'autoload': 'true',

'count': '20',

'cur_tab':1,

}

url = 'https://www.toutiao.com/search_content/?'+ urlencode(data)

response = requests.get(url)

try:

if response.status_code ==200:

return response.text

return None

except RequestException:

print('请求索引出错')

return None

def parse_page_index(html):

data =json.loads(html)

if data and 'data' in data.keys():

for item in data.get('data'):

yield item.get('article_url')

def get_page_detail(url):

try:

response = requests.get(url)

if response.status_code ==200:

return response.text

return None

except RequestException:

print('请求详情页出错',url)

return None

def parse_page_detail(html):

soup = BeautifulSoup(html,'lxml')

title = soup.select('title')[0].get_text()

print(title)

images_pattern = re.compile('gallery=(.*?);',re.S)

result = re.search(images_pattern,html)

if result:

print(result.group(1))

def main():

html = get_page_index(0,'街拍')

# print(html)

for url in parse_page_index(html):

html = get_page_detail(url)

if html:

parse_page_detail(html)

# print(url)

if __name__ =='__main__':

main()