多进程requests

Python脚本

import concurrent.futures
import concurrent.futures
import requests
import json

def process_url(url, headers,data):
    try:
        # 拼接访问地址
        resp = requests.get(url + "/url地址", headers=headers,verify=False, timeout=5)
        # print(resp.text)
        # 判断返回关键词
        if "关键词" in resp.text:
            return url
        else:
            return None
    except requests.exceptions.Timeout:
        print(f"Timeout for URL: {url}")
        return None
    except Exception as e:
        print(e)



if __name__ == "__main__":
    # 结果文件
    resulttxt=open("result.txt", "a+")
    headers = {
        # 发送json数据时用
        # "Content-Type": "application/json;charset=utf-8",
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/119.0"
    }

    data = {""}
    # 发送json数据时用
    # data = json.dumps(data)

    # 请求url文件
    with open("req.txt", "r") as urllist:
        urls = [line.strip() for line in urllist]

    with concurrent.futures.ProcessPoolExecutor(max_workers=20) as executor:
        futures = [executor.submit(process_url, url,headers,data) for url in urls]

        for future in concurrent.futures.as_completed(futures):
            try:
                result = future.result()
                if result !=None:
                    print(result)
                    resulttxt.write(result+"\n")
                    resulttxt.flush()
            except Exception as e:
                pass
    resulttxt.close()