假设有A表,包含图片地址,需要判断图片是否能正常打开,并返回错误代码。
import psycopg2
import psycopg2.extras
import requests
from config.load_config import get_connect_info
from functools import lru_cache
from concurrent.futures import ThreadPoolExecutor
class Link_Postgres():
def __init__(self):
conf1, conf2 = get_connect_info()
self.conn1 = psycopg2.connect(
dbname=conf1["dbname"],
user=conf1["user"],
password=conf1["password"],
host=conf1["host"],
port=conf1["port"]
)
self.cur1 = self.conn1.cursor()
def deal_head_picture_thread_pool(self):
self.cur1.execute("select asinid,goods from A表")
rows = self.cur1.fetchall()
executor = ThreadPoolExecutor(max_workers=10)
for row in rows:
id = row[0]
head_url = row[1]
executor.submit(self.deal_head_picture, id,head_url)
executor.shutdown(wait=True)
print('***读取完成***')
@lru_cache(maxsize=128)
def deal_head_picture(self, id,head_url):
if head_url != None:
print(head_url)
headers = {
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/537.36'
}
response = requests.get(url=head_url, headers=headers)
if response.status_code != 200 :
self.cur1.execute("INSERT INTO B表(asinid,goods,remark)VALUES(%s,%s,%s)",(id,head_url,response.status_code))
self.conn1.commit()
print(f'{head_url}----{response.status_code}图片打不开----')
lp = Link_Postgres()
lp.deal_head_picture_thread_pool()