爬取今日头条街拍美女图片

爬取今日头条美女图片需要分析Ajax请求

首先打开网址：https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D

返回的数据以及每个data展开后为下图：

图中标出的是详情页的url，然后进入详情页：

详细代码：

import json
import os
import re
from urllib.parse import urlencode

from bs4 import BeautifulSoup
from requests.exceptions import RequestException

import requests


def get_page_index(offest, keyword):
    data = {
        'offset': offest,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'cur_tab': 1
    }
    url = 'https://www.toutiao.com/search_content/?' + urlencode(data)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print("请求索引页出错")
        return None


def parse_page_index(html):
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            yield item.get('article_url')


def get_page_detail(url):
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print("请求详情页出错", url)
        return None


def parse_page_detail(html, url):
    soup = BeautifulSoup(html, 'lxml')
    title = soup.select('title')[0].get_text()
    # print("hah"+title)
    images_pattern = re.compile('gallery: (.*?),\n', re.S)

    # print(images_pattern)

    result = re.search(images_pattern, html)
    if result:
        # print("haha"+result.group(1))
        data = json.loads(result.group(1))
        if data and 'sub_images' in data.keys():
            sub_images = data.get('sub_images')
            # print(sub_images)
            images = [item.get('url') for item in sub_images]
            if images:
                return {
                    'title': title,
                    'url': url,
                    'images': images
                }

    else:
        pass


def main():
    html = get_page_index(0, '街拍')
    for url in parse_page_index(html):
        html = get_page_detail(url)
        if html:
            result = parse_page_detail(html, url)
            if result is not None:
                for url in result.get('images'):
                    print(url)
                    pic=requests.get(url)
                    pic_cun='F:\images\\'+str(url)[-8:-1]+'.jpg'
                    fp = open(pic_cun, 'wb')  # 以二进制写入模式新建一个文件
                    fp.write(pic.content)  # 把图片写入文件
                    fp.close()



if __name__ == '__main__':
    main()