import sys
from sqltool.mysql_client import MySqlClient
from queue import Queue, Empty
import os
import threading
base_dir = os.path.dirname(os.path.abspath('__file__'))
# 系统分隔符
os_sep = os.sep
sys.path.append(base_dir)
MYSQL_HOST = os.getenv("MYSQL_HOST", "localhost")
MYSQL_PORT = int(os.getenv("MYSQL_PORT", 3306))
MYSQL_USER = os.getenv("MYSQL_USER", "root")
MYSQL_PASSWORD = os.getenv("MYSQL_PASSWORD", "123")
MYSQL_SPIDER_DB = os.getenv("MYSQL_SPIDER_DB", "fosseye")
MYSQL_LJ_ID_DB = os.getenv("MYSQL_LJ_ID_DB", "lj_id")
MYSQL_OPEN_SOURCE_DB = os.getenv("MYSQL_OPEN_SOURCE_DB", "open_source_projects")
class MavenJar:
def __init__(self):
self.maven_cli = MySqlClient(
host=MYSQL_HOST,
port=MYSQL_PORT,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_SPIDER_DB
)
def get_maven_mysql(self):
sql = "SELECT * FROM `表名` WHERE jar_md5 IS NULL AND jar_name IS NOT NULL"
return self.maven_cli.query(sql)
def maven_jar(self, res):
pass
def run(self, que):
while True:
try:
k = que.get_nowait()
except Empty:
return
self.maven_jar(k)
if __name__ == '__main__':
maven = MavenJar()
que = Queue()
datas = maven.get_maven_mysql()
for data in datas:
que.put(data)
ths = [threading.Thread(target=maven.run, args=(que,)) for i in range(20)]
for th in ths:
th.start()
for th in ths:
th.join()
多线程+队列
于 2022-12-27 19:09:57 首次发布