python协程多线程HTMLParser下载mp3

使用HTMLParser解析html网页，顺便协程+多线程下载mp3文件

# coding=utf-8
import requests
import os
import chardet
import asyncio
import threading
from concurrent.futures import ThreadPoolExecutor
from enum import Enum
from html.parser import HTMLParser

gPool = ThreadPoolExecutor(30)
gLock = threading.Lock()


class EProcess(Enum):
    idle = 0
    start = 1
    end = 2


class CParseHtml(HTMLParser):

    def __init__(self, url):
        HTMLParser.__init__(self)
        hName = os.path.basename(url)
        prefix = hName.strip(".html").strip("?")
        self.mlog = "log" + prefix + ".txt"
        if os.path.isfile(self.mlog):
            os.remove(self.mlog)
        self.mUrls = []
        self.mProcess = EProcess.idle
        self.mBasePrefix = "http://www.txxxxx.com"

    def pinfo(self, text, flag=False):
        if flag:
            with open(self.mlog, "a") as f:
                f.write(text + "\n")
        print(text)


class CParsePageFirst(CParseHtml):

    def __init__(self, url):
        CParseHtml.__init__(self, url)
        self.mUl = False

    def handle_starttag(self, tag, attrs):
        for kv in attrs:
            if len(kv) == 2:
                if kv[1] == "zaixianlianbo":
                    self.mProcess = EProcess.start
                elif self.mProcess == EProcess.start and kv[0] == "href":
                    ul = self.mBasePrefix + kv[1]
                    self.mUrls.append(ul)

    def handle_endtag(self, tag):
        if self.mProcess == EProcess.start and tag == "ul":
            self.mProcess = EProcess.end


class CParsePageDown(CParseHtml):
    def __init__(self, url):
        CParseHtml.__init__(self, url)
        self.mUl = False

    def handle_starttag(self, tag, attrs):
        if tag == "script":
            self.mProcess = EProcess.start

    def handle_endtag(self, tag):
        if tag == "script":
            self.mProcess = EProcess.idle

    def handle_data(self, data):
        if (self.mProcess == EProcess.start):
            # print("data = {}".format(data))
            if data.find("getAspParas") != -1:
                httpPrefix = "http://mp3"
                httpSuffix = ".mp3"
                idxStart = data.find(httpPrefix)
                idxEnd = data.find(httpSuffix)
                if idxStart != -1 and idxEnd != -1:
                    mp3Url = data[idxStart:idxEnd + len(httpSuffix)]
                    self.mUrls.append(mp3Url)


async def getHtml(url, loop):
    global gPool
    result = await loop.run_in_executor(gPool, requests.get, url)
    if (result.status_code == 200):
        en = chardet.detect(result.content)
        # print("en = {}".format(en))
        encode = en["encoding"]
        if encode.lower() == "gb2312":
            encode = "gb18030"
        content = result.content.decode(encode)
        # base = os.path.basename(url)
        # with open(base.strip("?"), "w") as f:
        #     f.write(content)
        return 200, content
    return 404, ""


async def dealOne(url, loop):
    code, content = await getHtml(url, loop)
    if code == 200:
        down = CParsePageDown(url)
        down.feed(content)
        if len(down.mUrls) != 1:
            # down.pinfo("downOne:error len(murls) == {}".len(down.mUrls))
            pass
        else:
            mp3Url = down.mUrls[0]
            saveName = os.path.basename(mp3Url)
            if not os.path.isfile(saveName):
                global gPool
                result = await loop.run_in_executor(gPool, requests.get, mp3Url)
                if (result.status_code == 200):
                    with open(saveName, "wb") as f:
                        f.write(result.content)

                    global gLock
                    gLock.acquire()
                    print("down ok {}".format(saveName))
                    gLock.release()

            return saveName, True
    return url, False


async def dealAll(url, loop):
    code, content = await getHtml(url, loop)
    if code == 200:
        parseFirst = CParsePageFirst(url)
        parseFirst.feed(content)

        ts = []
        for ul in parseFirst.mUrls:
            task = asyncio.ensure_future(dealOne(ul, loop))
            ts.append(task)
        rs = await asyncio.gather(*ts)
        return rs
    else:
        print("error:dealAll")
    return []

if __name__ == "__main__":
    loop = asyncio.get_event_loop()
    try:
        url = "http://www.txxxxxxm/books/15051.html"
        rs = loop.run_until_complete(dealAll(url, loop))

        okCount = 0
        failCount = 0
        for name, success in rs:
            if success:
                okCount += 1
            else:
                failCount += 1
                print("fail name = {}".format(name))
        print("成功下载{}， 失败下载{}".format(okCount, failCount))
    finally:
        loop.close()
    print("main:end")