Python实现一个简单的代理池

以下代码构建了一个简单的代理池，我们把可用代理放在本地数据库，可以随机获取本地可用代理

import requests
import random
import threading
import time
import sqlite3

DBNAME = 'mydatabase.db'
TBNAME = 'mytable'

class SQLiteDB:
    def __init__(self, db_name):
        self.db_name = db_name
        self.connection = None
        self.cursor = None
        self.initDB()

    def initDB(self):
        self.connect()
        # 创建表
        self.cursor.execute(
            f'''CREATE TABLE IF NOT EXISTS {TBNAME} (
                id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
                proxy_url TEXT UNIQUE
            )''')
        self.connection.commit()

    def connect(self):
        self.connection = sqlite3.connect(self.db_name)
        self.cursor = self.connection.cursor()

    def disconnect(self):
        if self.connection:
            self.connection.close()

    def insert_data(self, table_name, column_values):
        placeholders2 = ", ".join(["?" for _ in column_values.keys()])
        placeholders1 = ", ".join([f"{column}" for column in column_values.keys()])
        query = f"INSERT INTO {table_name} ({placeholders1}) VALUES ({placeholders2})"
        try:
            self.cursor.execute(query, list(column_values.values()))
            self.connection.commit()
        except Exception as e:
            self.connection.rollback()

    def update_data(self, table_name, column_values, condition):
        placeholders = ", ".join([f"{column} = ?" for column in column_values.keys()])
        query = f"UPDATE {table_name} SET {placeholders} WHERE {condition}"
        self.cursor.execute(query, list(column_values.values()))
        self.connection.commit()

    def delete_data(self, table_name, condition):
        query = f"DELETE FROM {table_name} WHERE {condition}"
        self.cursor.execute(query)
        self.connection.commit()

    def fetch_data(self, table_name):
        query = f"SELECT * FROM {table_name}"
        self.cursor.execute(query)
        rows = self.cursor.fetchall()
        return rows

class ProxyPool(threading.Thread):
    def __init__(self):
        self.proxy_db = SQLiteDB(DBNAME)
        self.lock = threading.Lock()
    
    def has_proxy(self):
        return len(self.proxy_db.fetch_data(TBNAME))

    def add_proxy(self, proxy):
        with self.lock:
            self.proxy_db.insert_data(TBNAME, {"proxy_url": proxy})

    def remove_proxy(self, proxy):
        with self.lock:
            self.proxy_db.delete_data(TBNAME, f"proxy_url='{proxy}'")

    def get_random_proxy(self):
        with self.lock:
            if not self.has_proxy():
                return None
            return random.choice(self.proxy_db.fetch_data(TBNAME))[1]
    
    def run(self) -> None:
        while True:
            if self.has_proxy():
                proxy = self.get_random_proxy()
                if (proxy):
                    try:
                        headers = {
                            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
                        }
                        response = requests.get("http://www.ip111.cn/", headers=headers, proxies={'http': proxy, 'https': proxy}, timeout=5)
                        if response.status_code == 200:
                            print(f"代理：{proxy} 正常")
                        else:
                            print(f"代理：{proxy} 不正常，移出代理池")
                            self.remove_proxy(proxy)
                    except Exception as e:
                        print(f"代理：{proxy} 不正常，移出代理池。\n{e}")
                        self.remove_proxy(proxy)
            else:
                print("无可用代理，需要添加")
                self.add_proxy("http://localhost:8000")
                continue
            time.sleep(5)

if __name__ == "__main__":
    # 使用示例
    proxy_pool = ProxyPool()
    proxy_pool.run()

代码仅供学习使用