#coding=utf-8 __author__ = 'Administrator' import os import sys from bs4 import BeautifulSoup as BSoup from wget import download import requests import mechanize import cookielib import json from pprint import pprint #http://service.5sing.kugou.com/song/getPermission?jsoncallback=jQuery1114929725&songId=14109513&songType=2 DOWNLOAD_FOLDER = r'G:\音乐' HOST_DOWNLOAD_URL= 'http://service.5sing.kugou.com/song/getPermission' HOST_URL = 'http://5sing.kugou.com' HOST_URL_LIST = ['http://5sing.kugou.com/sevengod/fc/1.html',] NODE_ATTR_HREF = 'href' NODE_ATTR_TITLE = 'title' SONG_ATTR_ID = 'data-songid' SONG_ATTR_TYPE = 'data-songtype' DEBUG = True COOKIES = {'自己的cookie',} def comm_utf82gbk(s): return s.decode(ISingSpider.SITE_ENCODING).encode('GB2312', errors='ignore') def comm_log2file(msg, fname): fname = comm_utf82gbk(fname) with open(fname, 'w') as f: f.write(msg) def utf82gbk(s): return s.decode('utf-8').encode('gb2312') class ISingSpider(object): SITE_ENCODING = 'utf-8' def __init__(self, host_song_url): self._host_song_url = host_song_url def url2bs(self,url): res = requests.get(url) if res.status_code != 200: self.log('获取链接失败:%s' % (url,)) return res.encoding = ISingSpider.SITE_ENCODING bs = BSoup(res.content) return bs def init(self): bs = self.url2bs(self._host_song_url) self._title = bs.head.title.text.encode(ISingSpider.SITE_ENCODING) self.log('歌手信息:%s' % (self._title,)) self._song_lists = [] return True def download_fc(self): self.get_song_lists(self._host_song_url) self.log('%s 总共%d 首歌曲。' % (self._title, len(self._song_lists))) for s in self._song_lists: self.download_song(s) def getattr(self, node, attrname): v = node[attrname] if type(v) == unicode: return v.encode(ISingSpider.SITE_ENCODING) return v def get_song_lists(self, url): if not url: return #开始从页面上爬取数据 songs_lists = [] bs = self.url2bs(url) songs = bs.find_all(name='div', attrs={'class':'song_name'}) for s in songs: lnk = s.find(name='a') if not lnk: continue href = self.getattr(lnk, NODE_ATTR_HREF) title = self.getattr(lnk, NODE_ATTR_TITLE) asong = {NODE_ATTR_HREF:href, NODE_ATTR_TITLE:title,} songs_lists.append(asong) for s in songs_lists: self.log('歌曲名称:%s 链接:%s' % (s[NODE_ATTR_TITLE], s[NODE_ATTR_HREF])) self._song_lists += songs_lists next_url = bs.find(name='a', attrs={'class':'page_next'}) if next_url: next_url = self.getattr(next_url, NODE_ATTR_HREF) next_url = HOST_URL + next_url self.get_song_lists(next_url) def dump_song(self, song): self.log('消息:%s 返回码:%d' % (song[u'message'], song[u'code'])) data = song[u'data'] self.log('songGd=%d 作者:%s 名称:%s 地址:%s' % (data[u'songGd'], data[u'authorName'], data[u'songName'], data[u'fileName'])) def get_song_info(self, song): song_name = song[u'data'][u'songName'] index = song_name.find(' ') song_name = song_name[0:index].encode('utf-8') return (song[u'data'][u'fileName'].encode('utf-8'), song[u'data'][u'authorName'].encode('utf-8'), song_name) def response2dict(self, res): res = res[res.find('(') + 1: res.rfind(')')].encode('utf-8') r = json.loads(res, encoding='utf-8') self.dump_song(r) return self.get_song_info(r) def download_song(self, asong): if type(asong) != dict: self.log('无效的歌曲信息') return False #继续提取信息进行下载 bs = self.url2bs(asong[NODE_ATTR_HREF]) if not bs: self.log('下载失败。') return False jcb = 'jQuery110' song_name = asong[NODE_ATTR_TITLE] self.log('正在下载:%s' % (song_name,)) #self.log2file(bs.prettify(), song_name) btn_down = bs.find(name='a', attrs={'id':'func_Down'}) if btn_down: song_id = self.getattr(btn_down, SONG_ATTR_ID) song_type = self.getattr(btn_down, SONG_ATTR_TYPE) self.log('song-id:%s song-type:%s' % (song_id, song_type)) #这里必须要有cookie才可以,否则就会加载失败 res = requests.get(HOST_DOWNLOAD_URL, params={'jsoncallback':jcb, 'songId':song_id, 'songType':song_type}, cookies=COOKIES) songi = self.response2dict(res.content) download_dir = os.path.join(DOWNLOAD_FOLDER, songi[1]) gbk_dir = utf82gbk(download_dir) if not os.path.exists(gbk_dir): self.log(download_dir) os.makedirs(gbk_dir) full_name = os.path.join(download_dir, songi[2] + '.mp3') download(songi[0], full_name.decode('utf-8')) #提取歌词 lrc = bs.find(name='div', attrs={'class':'lrc_info_clip lrc-tab-content'}) lrc_fullname = os.path.join(download_dir, songi[2] + '.lrc') lrc_content = lrc.text.strip() lrc_content = u'\r\n'.join(lrc_content.split(' ')) self.log2file(lrc_content, lrc_fullname) self.log('下载文件到:%s 下载歌词到:%s' % (full_name,lrc_fullname)) #self.log(res.url) #self.log(res.content) #sys.exit(0) #self.log(str(flashs)) #self.log(bs.text) def log(self, msg): if DEBUG: print(msg) def log2file(self, msg, fname): comm_log2file(msg, fname) def download_song(): song_url = 'http://5sing.kugou.com/fc/14109513.html' br = mechanize.Browser() #初始化浏览器的Cookie cj = cookielib.LWPCookieJar() br.set_cookiejar(cj) #打开基本上所有的浏览器功能 br.set_handle_equiv(True) br.set_handle_gzip(True) br.set_handle_redirect(True) br.set_handle_referer(True) br.set_handle_robots(True) br.set_handle_refresh(True) #br.set_handled_schemes(True) #打开Browser的所有调试功能 br.set_debug_http(True) br.set_debug_redirects(True) br.set_debug_responses(True) #浏览器的附加头信息,可以覆盖操作原来的 br.addheaders = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.101 Safari/537.36'),] br.open(song_url) comm_log2file(br.response().read(), '内容') def session_download_song(): sess = requests.Session() res = sess.get('http://5sing.kugou.com/fc/14217069.html') pprint(res.cookies.items()) #pprint(sess.cookies.items()) #print(str(sess.cookies)) def init_sys(): reload(sys) sys.setdefaultencoding('utf-8') if __name__ == '__main__': init_sys() session_download_song() #download_song() ''' for i in HOST_URL_LIST: spider = ISingSpider(i) if spider.init(): spider.download_fc() '''
抓取5sing上的歌曲
最新推荐文章于 2021-06-03 15:13:30 发布