import requests
from lxml import etree
import os
import re
import time
import multiprocessing
from multiprocessing import Pool, Queue
def download(audio_url_title):
(audio_url, title) = audio_url_title
filename = 'guodegang/' + title + '.m4a'
print(filename + ' downloading.....')
audio = requests.get(audio_url).content
with open(filename, 'wb') as f:
f.write(audio)
time.sleep(1)
def get_detail_url(li_ele, queue):
title = li_ele.xpath('./@title')[0]
uid = li_ele.xpath('./@href')[0].split('/')[-1]
sub_url = 'https://www.ximalaya.com/revision/play/tracks?trackIds=%s' % uid
audio_url = requests.get(sub_url, headers=headers).json()['data']['tracksForAudioPlay'][0]['src']
# print(audio_url)
queue.put((audio_url, title))
# download(audio_url, title)
def ximalaya(queue):
max_page = 0
page = 1
global headers
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3377.1 Safari/537.36',
}
while page != max_page:
url = 'https://www.ximalaya.com/xiangsheng/9723091/p%d' % page
print(url)
response = requests.get(url, headers=headers)
html_ele = etree.HTML(response.text)
if max_page == 0:
page_list = re.findall('xiangsheng/9723091/p(\d+)', response.text)
for i in range(len(page_list)):
page_list.append(int(page_list.pop(0)))
max_page = max(page_list)
li_list = html_ele.xpath('//ul[@class="dOi2"]/li/div[2]/a')
if not os.path.exists('guodegang'):
os.mkdir('guodegang')
for li_ele in li_list:
get_detail_url(li_ele, queue)
page += 1
if __name__ == '__main__':
q = Queue()
p = multiprocessing.Process(target=ximalaya, args=(q,))
p.start()
pool = Pool(3)
while True:
try:
url = q.get(timeout=5)
# print(url)
pool.apply_async(download, (url, ))
except:
break
pool.close()
pool.join()
p.join()