python测试实验

import requests
import json
import time
import os
import queue
import threading
from tqdm import tqdm


arr_map=[]

class Work(threading.Thread):
    def __init__(self, q, name):
        threading.Thread.__init__(self)
        self.q = q
        self.name = name

    def run(self):
        while True:
            if self.q.empty():
                break
            else:
                url = self.q.get()
                download(url)
                self.q.task_done()

def getPageContent():
    for i in range(1,10):
        url="https://live.freebuf.com/index/course/?type=1&page="+str(i)
        r=requests.get(url)
        j=json.loads(r.text)
        list=j['data']['list']
        list_len=len(list)
        for k in range(0,list_len):
            money=list[k]['money']
            title=list[k]['title']
            id=list[k]['url'][19:]
            print(title+"------"+money+'-----'+id)
            with open("result.txt","a+",encoding="utf-8") as f1:
                f1.write(title+"------"+money+'-----'+id+"\n")
            getCal(title,money,id)
    f1.close()


def getCal(title,money,id):
    url="https://live.freebuf.com/course/detail/?id="+str(id)
    r=requests.get(url)
    j=json.loads(r.text)
    try:
        list=j['data']['series_course']['list']
        other=j['data']['series_course']['other']
        list_len=len(list)
        other_len=len(other)
        for i in range(0,list_len):
            son=list[i]['son']
            son_len= len(son)
            for k in range(0,son_len):
                name=son[k]['name']
                video=son[k]['video']
                print("----------"+name+"------"+video)
                with open("result.txt","a+",encoding="utf-8") as f2:
                    f2.write("----------"+name+"------"+video+"\n")
                if video != "":
                    videoUrl=geturl(title,name,video)
                    print("----------"+name+"------"+videoUrl)
                    with open("videoUrl.txt","a+",encoding="utf-8") as f3:
                        f3.write(title+"------"+money+"\n")
                        f3.write("------------"+name+"------"+videoUrl+"\n")
                    with open("urls.txt","a+",encoding="utf-8") as f4:
                        f4.write(title+"------"+name+"------"+videoUrl+"\n")

        for m in range(0,other_len):
            name = other[m]['name']
            video=other[m]['video']
            print("----------"+name+"------"+video)
            with open("result.txt","a+",encoding="utf-8") as f5:
                f5.write("----------"+name+"------"+video+"\n")
            if video !="":
                videoUrl=geturl(title,name,video)
                print("----------"+name+"------"+videoUrl)
                with open("videoUrl.txt","a+",encoding="utf-8") as f6:
                    f6.write(title+"------"+money+"\n")
                    f6.write("----------"+name+"------"+videoUrl+"\n")
                with open("urls.txt","a+",encoding="utf-8") as f4:
                    f4.write(title+"------"+name+"------"+videoUrl+"\n")
    except:
        print(title+"---"+"暂未发现课程")
        with open("result.txt","a+",encoding="utf-8") as f:
                f.write(title+"---"+"暂未发现课程"+"\n")


def geturl(title,name,video):
    prefix="chrome-extension://jfedfbgedapdagkghmgibemcoggfppbb/download.html?url="
    url="https://playvideo.qcloud.com/getplayinfo/v2/1251001764/"+str(video)
    r=requests.get(url)
    j=json.loads(r.text)
    videoUrl=j['videoInfo']['sourceVideo']['url']
    suffix="&referer=https://live.freebuf.com/&filename="+str(title)+"/"+str(name)+".mp4"
    videoUrl = prefix + videoUrl + suffix
    return str(videoUrl)

def getProcess():
    txtFile = open(os.path.join("./", "urls.txt"),'rb')
    for line in txtFile.readlines():
        if line!="" :
            arr = str(line.strip(), encoding = "utf-8").split("------")
            info=[]
            title =arr[0]
            name=arr[1]
            url=arr[2]
            info.append(title)
            info.append(name)
            info.append(url)
            arr_map.append(info)
    title=""
    for arrs in arr_map:
        title=arrs[0]
        if os.path.exists('./out/'+title):
            pass
        else:
            os.makedirs("./out/"+title)

def download(url):
    title=""
    file_name = ""
    for arrs in arr_map:
        if arrs[2] == url:
            title=arrs[0]
            file_name =arrs[1]
    retry_times = 0
    while retry_times <= 5:
        try:
            header={
                'Accept': '*/*',
                'Accept-Encoding': 'gzip, deflate',
                'Accept-Language': 'zh-CN,zh;q=0.9',
                'Connection': 'keep-alive',
                'Host': '1251001764.vod2.myqcloud.com',
                'Referer': 'https://live.freebuf.com/',
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36',
            }
            video_content = requests.get(url, headers=header, stream=True)
            contentLength = int(video_content.headers['content-length'])/1024/1024
            print(file_name+"  开始下载  共" +"{:.2f}MB".format(contentLength)+"\n")
            process_bar=tqdm(colour='blue',total=contentLength,unit='MB',desc=file_name,initial=0)
            with open("./out/"+title+"/"+file_name+".mp4",'wb') as file:
                for chunk in video_content.iter_content(chunk_size=1024):
                    if chunk:
                        file.write(chunk)
                        process_bar.update(1024)
            file.close()
            print("\n"+file_name+"  下载结束\n")
            break
        except:
            pass
        retry_times += 1
    else:
        try:
            os.remove(file_name)
        except OSError:
            pass
        print("Failed to retrieve %s from %s.\n" % (url, file_name))



if __name__ == '__main__':
    getPageContent()
    #文本分割url
    getProcess()
    if os.path.exists('./out'):
        print("file exist!")
    else:
        os.makedirs("out")
    start_time = time.time()
    work_queue = queue.Queue(len(arr_map))
    threads = []
    for arrs in arr_map:
        work_queue.put(arrs[2])
    for i in range(3):
        thread = Work(work_queue, 'Thread-%s' % (i+1))
        thread.setDaemon(True)
        thread.start()
        threads.append(thread)

    if work_queue.join():
        for t in threads:
            t.join()
    end_time = time.time()
    print("用时：%s" % (end_time - start_time))