多进程喜马拉雅音乐爬取

import requests
from lxml import etree
import os
import re
import time
import multiprocessing
from multiprocessing import Pool, Queue


def download(audio_url_title):
    (audio_url, title) = audio_url_title
    filename = 'guodegang/' + title + '.m4a'
    print(filename + '  downloading.....')
    audio = requests.get(audio_url).content
    with open(filename, 'wb') as f:
        f.write(audio)
    time.sleep(1)


def get_detail_url(li_ele, queue):
    title = li_ele.xpath('./@title')[0]
    uid = li_ele.xpath('./@href')[0].split('/')[-1]
    sub_url = 'https://www.ximalaya.com/revision/play/tracks?trackIds=%s' % uid
    audio_url = requests.get(sub_url, headers=headers).json()['data']['tracksForAudioPlay'][0]['src']
    # print(audio_url)
    queue.put((audio_url, title))
    # download(audio_url, title)


def ximalaya(queue):
    max_page = 0
    page = 1
    global headers
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3377.1 Safari/537.36',
    }
    while page != max_page:
        url = 'https://www.ximalaya.com/xiangsheng/9723091/p%d' % page
        print(url)
        response = requests.get(url, headers=headers)
        html_ele = etree.HTML(response.text)
        if max_page == 0:
            page_list = re.findall('xiangsheng/9723091/p(\d+)', response.text)
            for i in range(len(page_list)):
                page_list.append(int(page_list.pop(0)))
            max_page = max(page_list)
        li_list = html_ele.xpath('//ul[@class="dOi2"]/li/div[2]/a')
        if not os.path.exists('guodegang'):
            os.mkdir('guodegang')
        for li_ele in li_list:
            get_detail_url(li_ele, queue)
        page += 1


if __name__ == '__main__':
    q = Queue()
    p = multiprocessing.Process(target=ximalaya, args=(q,))
    p.start()
    pool = Pool(3)
    while True:
        try:
            url = q.get(timeout=5)
            # print(url)
            pool.apply_async(download, (url, ))
        except:
            break
    pool.close()
    pool.join()
    p.join()