以下代码构建了一个简单的代理池,我们把可用代理放在本地数据库,可以随机获取本地可用代理
import requests
import random
import threading
import time
import sqlite3
DBNAME = 'mydatabase.db'
TBNAME = 'mytable'
class SQLiteDB:
def __init__(self, db_name):
self.db_name = db_name
self.connection = None
self.cursor = None
self.initDB()
def initDB(self):
self.connect()
# 创建表
self.cursor.execute(
f'''CREATE TABLE IF NOT EXISTS {TBNAME} (
id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
proxy_url TEXT UNIQUE
)''')
self.connection.commit()
def connect(self):
self.connection = sqlite3.connect(self.db_name)
self.cursor = self.connection.cursor()
def disconnect(self):
if self.connection:
self.connection.close()
def insert_data(self, table_name, column_values):
placeholders2 = ", ".join(["?" for _ in column_values.keys()])
placeholders1 = ", ".join([f"{column}" for column in column_values.keys()])
query = f"INSERT INTO {table_name} ({placeholders1}) VALUES ({placeholders2})"
try:
self.cursor.execute(query, list(column_values.values()))
self.connection.commit()
except Exception as e:
self.connection.rollback()
def update_data(self, table_name, column_values, condition):
placeholders = ", ".join([f"{column} = ?" for column in column_values.keys()])
query = f"UPDATE {table_name} SET {placeholders} WHERE {condition}"
self.cursor.execute(query, list(column_values.values()))
self.connection.commit()
def delete_data(self, table_name, condition):
query = f"DELETE FROM {table_name} WHERE {condition}"
self.cursor.execute(query)
self.connection.commit()
def fetch_data(self, table_name):
query = f"SELECT * FROM {table_name}"
self.cursor.execute(query)
rows = self.cursor.fetchall()
return rows
class ProxyPool(threading.Thread):
def __init__(self):
self.proxy_db = SQLiteDB(DBNAME)
self.lock = threading.Lock()
def has_proxy(self):
return len(self.proxy_db.fetch_data(TBNAME))
def add_proxy(self, proxy):
with self.lock:
self.proxy_db.insert_data(TBNAME, {"proxy_url": proxy})
def remove_proxy(self, proxy):
with self.lock:
self.proxy_db.delete_data(TBNAME, f"proxy_url='{proxy}'")
def get_random_proxy(self):
with self.lock:
if not self.has_proxy():
return None
return random.choice(self.proxy_db.fetch_data(TBNAME))[1]
def run(self) -> None:
while True:
if self.has_proxy():
proxy = self.get_random_proxy()
if (proxy):
try:
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:120.0) Gecko/20100101 Firefox/120.0"
}
response = requests.get("http://www.ip111.cn/", headers=headers, proxies={'http': proxy, 'https': proxy}, timeout=5)
if response.status_code == 200:
print(f"代理:{proxy} 正常")
else:
print(f"代理:{proxy} 不正常,移出代理池")
self.remove_proxy(proxy)
except Exception as e:
print(f"代理:{proxy} 不正常,移出代理池。\n{e}")
self.remove_proxy(proxy)
else:
print("无可用代理,需要添加")
self.add_proxy("http://localhost:8000")
continue
time.sleep(5)
if __name__ == "__main__":
# 使用示例
proxy_pool = ProxyPool()
proxy_pool.run()
代码仅供学习使用