最近写了个豆瓣个人电台自动下载加心歌曲的小程序,基本能够下载,但需要手动将"http://douban.fm/mine?type=liked"页面全都下载下来,有点蛋疼,- - !!。由于还没有实现程序登录豆瓣的功能,暂时先这样用吧。
#!/usr/bin/python2.7
# -*- coding:utf -*-
import urllib
import re
import socket
import cookielib
import urllib2
socket.setdefaulttimeout(1)
def getpag(url):
done = False
try:
response = urllib.urlopen(url)
re = response.read()
done = True
except Exception as e:
print "error in getpag({0})".format(url)
if done:
return re
else:
return ""
def removehtml(s):
p = re.compile(r'(<.*?>)|(&.*?;)', re.S)
return p.sub("", s)
def removeotherword(s):
p = re.compile(r'((([\(\[{])|(\xef\xbc\x88)).*?(([\)\]}])|(\xef\xbc\x89)))|(^\s+)|(\s+$)')
return p.sub("", s)
# login douban & sv cookie
# todo
def logindouban():
loginurl = "http://www.douban.com/accounts/login"
data = urllib.urlencode ({'source':'simple','form_email':'vodmaker@gmail.com','form_password':'xxx','remember':'on',})
print data
cj = cookielib.CookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
opener.addheaders = [("User-agent", "Mozilla/5.0 (X11; Linux i686; rv:2.0.1) Gecko/20100101 Firefox/4.0.1")]
req = urllib2.Request(loginurl, data)
print urllib2.urlopen(req).geturl()
#
# s1 match exactly to s2
def matchexact(s1, s2):
ps1 = re.compile(s1, re.I)
if ps1.search(s2) is not None:
return True
else:
return False
# s1 match to s2
def matchmost(s1, s2):
s1 = re.compile(r'\s').sub("\\s*", s1)
ps1 = re.compile(s1, re.I)
if ps1.search(s2) is not None:
return True
else:
return False
# [[name, artist], ...] per list_pag
def getmusiclist_perpag(list_pag):
ptable = re.compile(r'<table\s*class="olts"\s*width="100%">.*?</table>', re.S)
htable = ptable.search(list_pag).group()
ptbody = re.compile(r'<tbody>.*?</tbody>', re.S)
htbody = ptbody.search(htable).group()
ptr = re.compile(r'<tr>\s*<td>(.*?)</td>.*?<span>(.*?)</span>.*?</tr>', re.S)
musiclist = []
for m in ptr.finditer(htbody):
print "music:" + removeotherword(m.group(1)) + "artist:" + removeotherword(m.group(2))
musiclist.append([removeotherword(m.group(1)), removeotherword(m.group(2))])
return musiclist
# download music from mp3.yahoo.com
# parameter muscilist [[name, artist], ...]
def downloadfromyahoo(musiclist):
listurl = "http://music.yahoo.cn/s?q={0}&m=0"
for ma in musiclist:
music = ma[0]
artist = ma[1]
print "Music:\t" + music + "\tArtist:\t" + artist + "is Downloading..."
u = listurl.format(urllib.quote_plus(music))
listpag = getpag(u)
ptable = re.compile(r'<div class="yst-music">.*?</table>', re.S)
if ptable.search(listpag) is None:
print "No search result of {0} in yahoo.cn".format(music)
continue
htable = ptable.search(listpag).group()
ptr = re.compile(r'<tr>\s*<td class="m_song">\s*<a href=".*?url=(.*?)"'
r'.*?>(.*?)</a>'
r'.*?<td class="m_singer">.*?>(.*?)</a>'
r'.*?<td.*?<td>(.*?)</td>'
r'.*?<td>(.*?)[mM][bB]'
r'.*?</tr>'
, re.S);
find = False
for m in ptr.finditer(htable):
downurl = urllib.unquote(m.group(1))
music_t = removeotherword(removehtml(m.group(2)))
artist_t = removeotherword(removehtml(m.group(3)))
type_t = removeotherword(removehtml(m.group(4)))
size_t = removeotherword(removehtml(m.group(5)))
if matchexact(music_t, music) and matchexact(artist_t, artist) and float(size_t) > 2:
print "download from :" + downurl + ""
try:
music_stream = urllib.urlopen(downurl).read()
open("./down/"+music+"."+type_t, "wb").write(music_stream)
find = True
print "download success: music:{0}, artist:{1}".format(music, artist)
break
except Exception as e:
continue
print e
if not find:
for m in ptr.finditer(htable):
downurl = urllib.unquote(m.group(1))
music_t = removeotherword(removehtml(m.group(2)))
artist_t = removeotherword(removehtml(m.group(3)))
type_t = removeotherword(removehtml(m.group(4)))
size_t = removeotherword(removehtml(m.group(5)))
if matchmost(music_t, music) and matchmost(artist_t, artist) and float(size_t) > 1:
print "download from :" + downurl + ""
try:
music_stream = urllib.urlopen(downurl).read()
open("./down/"+music+"."+type_t, "wb").write(music_stream)
find = True
print "download success: music:{0}, artist:{1}".format(music, artist)
break
except Exception as e:
continue
print e
if not find:
print "download failed: music:{0}, artist:{1}".format(music, artist)
# end downloadfromyahoo func
musiclist = []
for i in range(17):
f = open("{0}.html".format(i), "r")
listp = f.read()
musiclist += getmusiclist_perpag(listp)
downloadfromyahoo(musiclist)
貌似豆瓣有屏蔽程序访问页面的措施,目前仍纠结于如何实现登录的部分,不能保证一定能解决登录豆瓣自动抓取加心页面的功能,程序更新期限未知。