多线程爬取妹子图网图片 Python

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
"""
@File    : Crawl_meizitu.py.py
@Time    : 2019/7/25 13:24
@Author  : Sound_of_Silence
"""

import requests
import re
import time
import random
import os
import traceback
from multiprocessing.dummy import Pool
from bs4 import BeautifulSoup
from lxml import etree


def get_text(url, headers, code='utf-8'):
    try:
        r = requests.get(url, headers=headers, timeout=30)
        r.encoding = code
        r.raise_for_status()
        return r.text
    except BaseException:
        return ''


def lxml_get_detail_title_and_url(page_url, title_list, url_list):
    try:
        html = get_text(page_url, headers=hds)
        ehtml = etree.HTML(html)
        titles = ehtml.xpath('//*[@id="pins"]/li/span[1]/a/text()')
        hrefs = ehtml.xpath('//*[@id="pins"]/li/span[1]/a/@href')
        for title, href in zip(titles, hrefs):
            title_list.append(title)
            url_list.append(href)
    except BaseException:
        traceback.print_exc()
        pass


def sub_page_url_list(url, deph):
    deph = str(deph.encode('utf-8')).lstrip('b')
    url = str(url.encode('utf-8')).lstrip('b')
    ls = [eval(url)]
    ls.extend([eval(url) + '/{}'.format(i)
               for i in range(2, int(eval(deph)) + 1)])
    return ls


def crawl_pic(detail_pic_list):
    for j in range(len(detail_pic_list)):
        try:
            time.sleep(random.random())
            html = requests.get(detail_pic_list[j], headers=hds).text
            hl = etree.HTML(html)
            pic_url = hl.xpath(
                '/html/body/div[2]/div[1]/div[3]/p/a/img/@src')[0]  # 列表形式，需提取
            time.sleep(random.random())
            r = requests.get(pic_url, headers=hds)
            with open('{}.jpg'.format(j), 'wb') as file:
                file.write(r.content)
        except BaseException:
            print('Error while crawl pic')
            traceback.print_exc()
            continue


def crawl_pic_html(titles, urls, filepath):
    for i in range(len(titles)):
        os.chdir(filepath)
        if not os.path.exists(filepath + '\\mm_pics\\{}'.format(titles[i])):
            os.makedirs(filepath + '\\mm_pics\\{}'.format(titles[i]))
        os.chdir(filepath + '\\mm_pics\\{}'.format(titles[i]))
        try:
            html = get_text(urls[i], headers=hds)
            ehtml = etree.HTML(html)
            length = ehtml.xpath(
                '/html/body/div[2]/div[1]/div[4]/a/span/text()')[-2]
            detail_pic_list = sub_page_url_list(urls[i], length)
            crawl_pic(detail_pic_list)
        except BaseException:
            print('Error while get length!')
            traceback.print_exc()
            return ''


def do(depth):
    global hds
    hds = {
        'referer': 'https://www.mzitu.com/tag/youhuo/',
        'upgrade-insecure-requests': '1',
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
        'Chrome/75.0.3770.100 Safari/537.36'}

    page_url_list = ['https://www.mzitu.com/mm/']
    if depth > 1:
        page_url_list.extend(['https://www.mzitu.com/mm/page/{}'.format(i)
                              for i in range(2, depth + 1)])

    title_list, url_list = [], []
    fpath = 'I:\\'
    for page_url in page_url_list:
        lxml_get_detail_title_and_url(page_url, title_list, url_list)
        crawl_pic_html(title_list, url_list, fpath)


if __name__ == '__main__':
    start = time.ctime()
    p = Pool(8)
    p.apply_async(do, args=(12,))
    p.close()
    p.join()
    end = time.ctime()
    print("耗时{:.1f}min".format((end - start)/60))