使用python编写的落网电台下载工具
落网是个非常不错的独立音乐介绍网站，但是上面提供的歌曲仅供试听，没有提供下载链接。相信不少同学都想过有没工具把歌曲下载下来呢？答案是：有。
我最早是使用遨游浏览器，它里面提供一个插件（资源探测器），只要点开，就能看到当前浏览器页面中有没有可下载的链接。当时觉得还挺方便的，就是
要一首一首地去下，还是有些麻烦。当然火狐浏览器里也有个插件（NetVideoHunter），功能上也是类似。
对于学习过python的同学，都知道它用来爬取网络资源上非常方便、强大。前段时间本人写过使用使用python抓取落网期刊图片，但觉得还是不够实用，
要是能够把任意一个（或者所有）期刊资源都下载到本地，那该多好！！于是，自己花了段时间去观察落网页面的HTML源码，发现期刊的曲目链接其实是隐藏
在audio标签内的（比如第一期第一首曲目的链接：http://mp3-cdn.luoo.net/low/luoo/radio1/1.mp3）。
# -*- coding:gbk -*-
'''
Created on 2017年2月26日
#
# 爬取落网期刊信息（包括标题，介绍，音频文件等）
#
@author: ysx
'''
from __future__ import unicode_literals
from Queue import Queue
import os
import re
import sys
import threading
import uuid
import bs4
from bs4.element import NavigableString
import requests
reload(sys)
sys.setdefaultencoding('gbk')
# 下载路径
BASE_DIR = r'F:\luonet'
# 下载资源超时时间
TIMEOUT = 10
# 当前最大期刊号
MAX_VOL_NO = 906
# 下载期刊使用最大线程数
MAX_THREAD_NUM = 10
def parse_vol_num(vol_url):
'''给定期刊Url,判定是第几期
'''
try:
return vol_url.split('/')[-1]
except:
raise RuntimeError('给定期刊Url[%s]不合法..')
def check_path_avail(path):
'''实际运行过程中，按曲目名去命名mp3文件，会出现写入报IOError（原因是歌曲名中包含?等非法字符）'''
check_result = True
if not os.path.exists(path):
try:
open(path, 'wb')
except IOError:
print '当前路径:[%s]无法写入..'%path
check_result = False
return check_result
def download_source(url, path):
print('下载资源:[%s]'%url)
resp = requests.get(url, timeout=TIMEOUT)
resp.raise_for_status() # 下载出现问题，立即抛出异常
with open(path, 'wb') as audio_handle:
for chunk in resp.iter_content(10000):
audio_handle.write(chunk)
class LuoVolInfo(object):
'''落网期刊类'''
def __init__(self, vol_url, vol_title, vol_desc, vol_image_url, basedir=None):
self.vol_num = parse_vol_num(vol_url)
self.vol_title = vol_title
self.vol_desc = vol_desc
self.vol_image_url = vol_image_url
self.vol_list = []
self._create_basedir(basedir)
def append_audio(self, luoAudioInfo):
'''追加曲目条目'''
self.vol_list.append(luoAudioInfo)
def _create_basedir(self, basedir):
if basedir is None:
basedir = BASE_DIR
self._dirname = 'Vol.%s %s'%(self.vol_num, self.vol_title)
basedir = os.path.join(basedir, self._dirname)
os.makedirs(basedir)
self.basedir = basedir
def download_vol_image(self):
'''下载专题封面'''
self.image_path = os.path.join(self.basedir, self._dirname+'.jpg')
if not check_path_avail(self.image_path):
# 重命名文件
self.image_path = os.path.join(self.basedir, '未命名-期刊封面.jpg')
download_source(self.vol_image_url, self.image_path)
def download_vol_desc(self):
'''下载专题描述'''
self.desc_path = os.path.join(self.basedir, '专题说明.txt')
with open(self.desc_path, 'w') as file_handle:
file_handle.write(self.vol_desc.encode('gbk'))
# 写入曲目列表
file_handle.write(os.linesep)
file_handle.write('>曲目列表')
file_handle.write(os.linesep)
if self.vol_list:
for vol in self.vol_list:
file_handle.write(vol.audio_name)
file_handle.write(os.linesep)
def download_all(self):
print '1.下载专题封面'
self.download_vol_image()
print '2.下载专题描述'
self.download_vol_desc()
# 下载曲目过程中，等待时间过长，尝试使用多线程加速下载
print '3.下载专题曲目'
# 生成任务队列
audio_dl_queue = Queue()
for vol in self.vol_list:
audio_dl_queue.put_nowait(vol)
# 定义线程目标函数
def target_vol_dl():
while True:
audio = audio_dl_queue.get()
if audio is None:
break
audio.download()
audio_dl_queue.task_done()
# 添加毒丸，并初始化线程池
thread_pool = []
for i in xrange(MAX_THREAD_NUM):
audio_dl_queue.put_nowait(None)
for i in xrange(MAX_THREAD_NUM):
t = threading.Thread(target=target_vol_dl)
t.setDaemon(True)
t.start()
thread_pool.append(t)
for t in thread_pool:
t.join()
del thread_pool[:]
class LuoAudioInfo(object):
'''落网电台曲目类
'''
# 曲目下载链接模板
audio_src_template = 'http://mp3-cdn.luoo.net/low/luoo/radio%d/%02d.mp3'
# 备用链接，曲目下载链接存在两种格式
audio_src_template2 = 'http://mp3-cdn.luoo.net/low/luoo/radio%d/%d.mp3'
def __init__(self, audio_index, audio_name, audio_artist, audio_image_url, vol_url, basedir=None):
self.audio_index = audio_index
self.audio_name = audio_name
self.audio_artist = audio_artist
self.audio_image_url = audio_image_url
self.vol_url = vol_url
self._create_audio_basedir(basedir)
self._parse_mp3_url()
def _parse_mp3_url(self):
'''给定期刊Url,判定是第几期
'''
try:
vol_num = parse_vol_num(self.vol_url)
except:
raise RuntimeError('给定期刊Url[%s]不合法..')
self.audio_mp3_url = self.audio_src_template%(int(vol_num.encode('gbk')), int(self.audio_index))
self.audio_mp3_url2 = self.audio_src_template2%(int(vol_num.encode('gbk')), int(self.audio_index))
print '解析后的mp3-Url:[%s]'%self.audio_mp3_url
def _create_audio_basedir(self, basedir):
'''创建曲目下载目录
'''
if basedir is None:
basedir = BASE_DIR
_dirname = self.audio_index
basedir = os.path.join(basedir, _dirname)
os.makedirs(basedir)
self.basedir = basedir
def download(self):
'''下载当前曲目的所有资源'''
self._download_image_or_audio()
self._download_image_or_audio(is_mp3=False)
def _download_image_or_audio(self, is_mp3=True):
'''下载图片或mp3'''
if is_mp3:
write_path = os.path.join(self.basedir, self.audio_name) + '.mp3'
request_url = self.audio_mp3_url
else:
write_path = os.path.join(self.basedir, self.audio_name) + '.jpg'
request_url = self.audio_image_url
# 检查待写入路径是否合法
if not check_path_avail(write_path):
# 重命名文件
new_name = uuid.uuid1()
write_path = os.path.join(self.basedir, '%s.%s'%(new_name, 'mp3' if is_mp3 else 'jpg'))
try:
download_source(request_url, write_path)
except:
if request_url==self.audio_mp3_url:
print '下载超时:%s'%(request_url)
print '尝试使用链接2：[%s]'%(self.audio_mp3_url2)
download_source(self.audio_mp3_url2, write_path)
def __repr__(self):
return '<落网期刊曲目> 序号:[%s], 曲目名:[%s], 曲目艺术家:[%s], 曲目封面Url:[%s], 曲目下载链接:[%s]'%(
self.audio_index,
self.audio_name,
self.audio_artist,
self.audio_image_url,
self.audio_mp3_url)
def download_vol_content(vol_url):
'''下载包括（1.期刊标题、图片；2.期刊介绍；3.期刊音频）
@param vol_url: 任意一个期刊Url，如果不存在（页面响应404错误），返回False
@param if_download_audio: 是否下载期刊下的音频（默认False）
@return: Boolean True 成功
'''
print('当前下载期刊链接：[%s]'%vol_url)
resp = requests.get(vol_url)
resp.raise_for_status() # 下载出现问题，立即抛出异常
resp.encoding = 'utf-8'
soup = bs4.BeautifulSoup(resp.text, 'lxml')
# 先判定是否返回404错误页面
error_msg_ele = soup.select('div.error-msg > h1')
if error_msg_ele and len(error_msg_ele):
print '页面返回404，也许该期刊不存在..'
return False
print
print '1.解析【期刊标题】'
vol_title_ele = soup.select('h1.vol-name > span.vol-title')
if vol_title_ele and len(vol_title_ele):
vol_title = vol_title_ele[0].string
print '本期标题：[%s]'%vol_title
print
print '2.解析【期刊说明】'
vol_desc_ele = soup.select('div.vol-desc')
if vol_desc_ele and len(vol_desc_ele):
vol_desc = ''.join(list(vol_desc_ele[0].strings))
print '本期说明：[%s]'%vol_desc
print
print '3.解析【期刊图片】'
vol_image_ele = soup.select('div.vol-cover-wrapper > img.vol-cover')
if vol_image_ele and len(vol_image_ele):
vol_image_url = vol_image_ele[0]['src']
print '图片链接：[%s]'%vol_image_url
luo_vol_obj = LuoVolInfo(vol_url, vol_title, vol_desc, vol_image_url, )
print '4.解析【曲目信息】'
vol_audio_ul_ele = soup.select('div.vol-tracklist > ul')
if vol_audio_ul_ele and len(vol_audio_ul_ele):
print vol_audio_ul_ele[0].children
for audio_li_ele in vol_audio_ul_ele[0].children:
if type(audio_li_ele) is NavigableString:
# 跳过字符串节点
continue
audio_li_soup = bs4.BeautifulSoup(audio_li_ele.prettify(), 'lxml')
audio_name = audio_li_soup.select('div.track-wrapper > a.trackname')[0].string.strip()
print '曲目名称：[%s]'%audio_name
# 获取当前曲目序号
pattern = re.compile(r'(\d+).*')
audio_index = pattern.match(audio_name).groups()[0]
audio_artist = audio_li_soup.select('div.track-wrapper > span.artist')[0].string.strip()
print '曲目作家：[%s]'%audio_artist
audio_image_url = audio_li_soup.select('div.track-wrapper > a.btn-action-share.icon-share')[0]['data-img'].strip()
print '曲目封面Url：[%s]'%audio_image_url
audio_obj = LuoAudioInfo(audio_index, audio_name, audio_artist, audio_image_url, vol_url, basedir=luo_vol_obj.basedir)
print repr(audio_obj)
print
# 追加至期刊曲目列表
luo_vol_obj.append_audio(audio_obj)
luo_vol_obj.download_all()
return True
def main():
def print_useage():
print '''
>使用说明：
1.下载所有期刊： luonet_download_tool.py all
2.下载特定期刊（比如下载最新的906）： luonet_download_tool.py 906
'''
sys.exit()
if len(sys.argv) == 1:
print_useage()
vol_url_template = 'http://www.luoo.net/music/%s'
if len(sys.argv) == 2:
dl_flag = sys.argv[1]
if dl_flag == 'all':
print '======================'
print '>开始下载所有期刊:[%s-%s]'%(1, MAX_VOL_NO)
print '======================'
for i in xrange(1, MAX_VOL_NO):
vol_url = vol_url_template%i
download_vol_content(vol_url)
elif dl_flag and dl_flag.isdigit():
print '======================'
print '>下载指定期刊号:[%s]'%int(dl_flag)
print '======================'
obj_vol = int(dl_flag)
download_vol_content(vol_url_template%obj_vol)
else:
print_useage()
if __name__ == '__main__':
main()
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
