使用HTMLParser解析html网页,顺便协程+多线程下载mp3文件
# coding=utf-8
import requests
import os
import chardet
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from enum import Enum
from html.parser import HTMLParser
gPool = ThreadPoolExecutor(30)
gLock = threading.Lock()
class EProcess(Enum):
idle = 0
start = 1
end = 2
class CParseHtml(HTMLParser):
def __init__(self, url):
HTMLParser.__init__(self)
hName = os.path.basename(url)
prefix = hName.strip(".html").strip("?")
self.mlog = "log" + prefix + ".txt"
if os.path.isfile(self.mlog):
os.remove(self.mlog)
self.mUrls = []
self.mProcess = EProcess.idle
self.mBasePrefix = "http://www.txxxxx.com"
def pinfo(self, text, flag=False):
if flag:
with open(self.mlog, "a") as f:
f.write(text + "\n")
print(text)
class CParsePageFirst(CParseHtml):
def __init__(self, url):
CParseHtml.__init__(self, url)
self.mUl = False
def handle_starttag(self, tag, attrs):
for kv in attrs:
if len(kv) == 2:
if kv[1] == "zaixianlianbo":
self.mProcess = EProcess.start
elif self.mProcess == EProcess.start and kv[0] == "href":
ul = self.mBasePrefix + kv[1]
self.mUrls.append(ul)
def handle_endtag(self, tag):
if self.mProcess == EProcess.start and tag == "ul":
self.mProcess = EProcess.end
class CParsePageDown(CParseHtml):
def __init__(self, url):
CParseHtml.__init__(self, url)
self.mUl = False
def handle_starttag(self, tag, attrs):
if tag == "script":
self.mProcess = EProcess.start
def handle_endtag(self, tag):
if tag == "script":
self.mProcess = EProcess.idle
def handle_data(self, data):
if (self.mProcess == EProcess.start):
# print("data = {}".format(data))
if data.find("getAspParas") != -1:
httpPrefix = "http://mp3"
httpSuffix = ".mp3"
idxStart = data.find(httpPrefix)
idxEnd = data.find(httpSuffix)
if idxStart != -1 and idxEnd != -1:
mp3Url = data[idxStart:idxEnd + len(httpSuffix)]
self.mUrls.append(mp3Url)
async def getHtml(url, loop):
global gPool
result = await loop.run_in_executor(gPool, requests.get, url)
if (result.status_code == 200):
en = chardet.detect(result.content)
# print("en = {}".format(en))
encode = en["encoding"]
if encode.lower() == "gb2312":
encode = "gb18030"
content = result.content.decode(encode)
# base = os.path.basename(url)
# with open(base.strip("?"), "w") as f:
# f.write(content)
return 200, content
return 404, ""
async def dealOne(url, loop):
code, content = await getHtml(url, loop)
if code == 200:
down = CParsePageDown(url)
down.feed(content)
if len(down.mUrls) != 1:
# down.pinfo("downOne:error len(murls) == {}".len(down.mUrls))
pass
else:
mp3Url = down.mUrls[0]
saveName = os.path.basename(mp3Url)
if not os.path.isfile(saveName):
global gPool
result = await loop.run_in_executor(gPool, requests.get, mp3Url)
if (result.status_code == 200):
with open(saveName, "wb") as f:
f.write(result.content)
global gLock
gLock.acquire()
print("down ok {}".format(saveName))
gLock.release()
return saveName, True
return url, False
async def dealAll(url, loop):
code, content = await getHtml(url, loop)
if code == 200:
parseFirst = CParsePageFirst(url)
parseFirst.feed(content)
ts = []
for ul in parseFirst.mUrls:
task = asyncio.ensure_future(dealOne(ul, loop))
ts.append(task)
rs = await asyncio.gather(*ts)
return rs
else:
print("error:dealAll")
return []
if __name__ == "__main__":
loop = asyncio.get_event_loop()
try:
url = "http://www.txxxxxxm/books/15051.html"
rs = loop.run_until_complete(dealAll(url, loop))
okCount = 0
failCount = 0
for name, success in rs:
if success:
okCount += 1
else:
failCount += 1
print("fail name = {}".format(name))
print("成功下载{}, 失败下载{}".format(okCount, failCount))
finally:
loop.close()
print("main:end")