抓取头条小姐姐照片并分类存储在image目录下

'''
AUTHOR: vforlove
CREATED: 2018-08-03
IDE:PyCharm
version:3.6.6
'''
import requests
import os
from hashlib import md5
from urllib.parse import urlencode
from multiprocessing.pool import Pool
#定义获取头条街拍页面函数
def get_page(offset):
    parameters = {
        'offset': offset,
        'format': 'json',
        'keyword':'街拍',
        'autoload': 'true',
        'count': '20',
        'cur_tab': '1',
        'from': 'search_tab'
    }
    url = 'https://www.toutiao.com/search_content/?' + urlencode(parameters)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            print('请求网页成功！')
            return response.json()
    except requests.ConnectionError as e:
        print('error:', e.args)
#定义解析网页函数
def parse_image(json):
    if json:
        items = json.get('data')
        for item in items:
            title = item.get('title')
            images = item.get('image_list')
            if images is None:
                continue
            for image in images:
                yield {
                    'image_url': image.get('url'),
                    'title': title
                }
#定义保存链接图片函数
def save_image(image_dict):
    image_path = './image/{0}'.format(image_dict.get('title'))
    if not os.path.exists(image_path):
        os.mkdir(image_path)
    try:
        image_url = 'http:{0}'.format(image_dict.get('image_url'))
        response = requests.get(image_url)
        if response.status_code == 200:
            #构建存储路径，标题作为路径，十六进制的MD5值作为图片名字
            file_path = './image/{0}/{1}{2}'.format(image_dict.get('title'), md5(response.content).hexdigest(), '.jpg')
            if not os.path.exists(file_path):
                with open(file_path, 'wb+') as f:
                    f.write(response.content)
            else:
                print('Already Downloaded')
    except requests.ConnectionError as e:
        print('download error:', e.args)
#定义main
def main(offset):
    image_dict = get_page(offset)
    items = parse_image(image_dict)
    for item in items:
        print(item)
        save_image(item)
#start
if __name__ == '__main__':
    if not os.path.exists('image'):  #创建存储目录
        os.mkdir('image')
    pool = Pool()  #创建进程池
    # 控制变量
    GROUP_START = 1
    GROUP_END = int(input('请输入你想抓取的图片页数：'))
    groups = ([x * 20 for x in range(GROUP_START, GROUP_END+1)])
    pool.map(main, groups)
    pool.close()  #关闭进程请求
    pool.join()  #等待进程结束