2020-10-22今日头条爬虫更新

今日头条爬取街拍小姐姐图片--20201022

今天发现今日头条有了新的反扒机制，导致之前的爬虫无法执行，现更新

## 导入需要的包
import json
import os
import time
from hashlib import md5
from urllib.parse import urlencode
import re
import requests
from bs4 import BeautifulSoup
from requests.exceptions import RequestException

自定义一个函数

def validatetile(title):
    """

    :param title: 传入的参数
    :return: 将无法作为标题的符号替换为_
    """
    rstr = '[/\:*?<>|@]'
    new_title = re.sub(rstr,'_',title)
    return new_title

获取起始页面信息

def get_page_index(offset):
    """

    :param offset: Ajax 网页每次的偏移值
    :return:输出每次刷新出来的网页
    """
    params = {
        'aid': '24',
        'app_name': 'web_search',
        'offset': offset,
        'format': 'json',
        'keyword': '街拍',
        'autoload': 'true',
        'count': '20',
        'cur_tab': 1
    }
    headers = {
        'user-agent' :'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36',
        'referer' : 'https://www.toutiao.com/search/?keyword=%E8%A1%97%E6%8B%8D'
    }
    url = 'https://www.toutiao.com/api/search/content/?' + urlencode(params)
    try:
        response = requests.get(url, headers = headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求索引页出错')
        return None

解析页面信息，获取标题和照片url

def parse_one_index(html):
    """

    :param html: 获取到的网页信息
    :return: 一个字典，包含标题和一个图片网址列表
    """
    data = json.loads(html)
    if data and 'data' in data.keys():
        for item in data.get('data'):
            title = item.get('title')
            if 'image_list' in item.keys() and item['image_list'] != []:
                images = item.get('image_list')
                imgs = []
                for image in  images:
                    imgs.append(image.get('url').replace('list/190x124', 'origin').replace('list', 'origin'))

                yield {
                    'title' : title,
                    'image_urls' : imgs
                }

保存图片到本地

def save_image(item):
    if not os.path.exists(r'D:/work/中移/告警事宜/脚本/spider/头条街拍/{}'.format(validatetile(item.get('title')))):
        os.mkdir(r'D:/work/中移/告警事宜/脚本/spider/头条街拍/{}'.format(validatetile(item.get('title'))))
        try:
            for image_url in item.get('image_urls'):
                response = requests.get(image_url)
                if response.status_code == 200:
                    file_path = r'D:/work/中移/告警事宜/脚本/spider/头条街拍/{0}/{1}.{2}'.format(validatetile(item.get('title')), md5(response.content).hexdigest(), 'jpg')
                    if not os.path.exists(file_path):
                        with open(file_path, 'wb') as f:
                            f.write(response.content)
                        print('Downloaded image path is:', file_path)
                    else:
                        print('Already Downloaded', file_path)
        except requests.ConnectionError:
            print('Failed to Save Image')

运行主程序

def main():
    for i in range(3):
        print(f'正在爬取offset为{i * 20}')
        html = get_page_index(i * 20)
        for item in parse_one_index(html):
            save_image(item)
        time.sleep(5)


if __name__ == '__main__':
    main()