头条爬取

最新推荐文章于 2024-08-01 11:05:00 发布

有些故事只想说给陌生人听

最新推荐文章于 2024-08-01 11:05:00 发布

阅读量389

点赞数

本文链接：https://blog.csdn.net/a10090492/article/details/80702402

版权

# -*- coding:utf-8 -*-
import requests
from urllib import parse, request
import requests
import os


def get_page(offset):
    params = {
        'offset': offset,
        'format': 'json',
        'keyword': '街拍',
        'autoload': 'ture',
        'count:': '20',
        'cur_tab': '1',
        'from': 'search_tab'
    }
    data = parse.urlencode(params)
    url = 'https://www.toutiao.com/search_content/?' + data
    try:
        response = requests.get(url)
        if response.status_code == 200:
            json_dict = response.json()
            return json_dict
    except requests.ConnectionError:
        return None


def get_images(json_dict):
    if json_dict.get('data'):
        for item in json_dict.get('data'):
            if item.get('media_creator_id'):
                title = item.get('title')
                print(title)
                images = item.get('image_detail')
                path = 'images/' + title
                if not os.path.exists(path):
                    os.makedirs(path)
                for image in images:
                    image_url = image.get('url')
                    name = image_url.split("/")[-1]
                    request.urlretrieve(image_url, path + "/" + name + '.jpg')


start = 1
end = 20
page_list = list([x * 20 for x in range(start, end + 1)])
for offset in page_list:
    json_dict = get_page(offset)
    get_images(json_dict)