import requests, json
import math, sys, logging, threading, time, queue
import pymongo, os, base64, datetime, traceback, pymysql
from pymysql.cursors import DictCursor
from apscheduler.schedulers.blocking import BlockingScheduler
from PIL import Image, ExifTags
import matplotlib.image as mpimg
from DBUtils.PooledDB import PooledDB
mutex = threading.Lock()
logger = logging.getLogger('log')
logger.setLevel(logging.DEBUG)
# 调用模块时,如果错误引用,比如多次调用,每次会添加Handler,造成重复日志,这边每次都移除掉所有的handler,后面在重新添加,可以解决这类问题
while logger.hasHandlers():
for i in logger.handlers:
logger.removeHandler(i)
# file log
formatter = logging.Formatter('%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s')
fh = logging.FileHandler(r'sync_znxk_brand_data_pymongo_try_2_optimizer.log', encoding='utf-8') # 本地测试
# fh = logging.FileHandler(r'/opt/service/log/apilog/sync_znxk_brand_data_pymongo_try_2_optimizer.log', encoding='utf-8') # TODO 线上配置
fh.setLevel(logging.DEBUG)
fh.setFormatter(formatter)
logger.addHandler(fh)
# console log
ch = logging.StreamHandler(sys.stdout)
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
DATABASE_CONFIG = {
"test": {
'user': 'XXX',
'passwd': '******',
'host': '127.0.0.1',
'database': 'XXX',
'port': 3306
},
"skynet_test": {
'user': 'XXX',
'passwd': '******',
'host': '127.0.0.1',
'database': 'XXX',
'port': 3306
}
}
class MultiThreadSyncBrandDatas(object):
def __init__(self, thread_num=10, database_flag='test', insert_flag=0):
self.qurl = queue.Queue()
self.thread_num = thread_num
self.database_flag = database_flag
self.insert_flag = insert_flag
self.myslq_pool = PooledDB(pymysql, thread_num,
host=DATABASE_CONFIG.get(database_flag).get("host"),
user=DATABASE_CONFIG.get(database_flag).get("user"),
passwd=DATABASE_CONFIG.get(database_flag).get("passwd"),
db=DATABASE_CONFIG.get(database_flag).get("database"),
port=DATABASE_CONFIG.get(database_flag).get("port"),
cursorclass=DictCursor) # thread_num为连接池里的最少连接数,与线程数保持一致
def execute_select_sql(self, cursor, select_sql):
"""执行myslq查询"""
try:
cursor.execute(select_sql)
result = cursor.fetchall()
return result
except:
raise Exception(traceback.format_exc())
def run(self):
try:
logger.info('****** start run at:{} ******'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
conn = self.myslq_pool.connection()
cursor = conn.cursor()
try:
cursor.execute(
'insert into season_datas(season, status, create_time, update_time) values("{}", "{}", "{}", "{}")'.format('春天'))
last_id = cursor.lastrowid
print('%%%%%%%%%%%%%%%%%%%%%%%%%% last_id = {}'.format(last_id))
cursor.execute('commit;')
except:
cursor.execute('rollback;')
traceback.print_exc()
except:
logger.error(traceback.format_exc())
raise Exception(traceback.format_exc())
def get_conn(self, host, user, passwd, database, port):
"""链接数据库,创建操作游标对象cursor并返回"""
db = pymysql.connect(host=host, user=user, passwd=passwd, db=database, port=int(port), cursorclass=DictCursor)
cursor = db.cursor()
return db, cursor
def main(thread_num, database_flag, insert_flag, once_handle_count):
""""""
s_t = time.time()
try:
my_multi_thread = MultiThreadSyncBrandDatas(thread_num=thread_num, database_flag=database_flag, insert_flag=insert_flag, once_handle_count=once_handle_count)
my_multi_thread.run()
except:
logger.error(traceback.format_exc())
logger.info('****** all task consum time = {}秒 约合 = {}分钟'.format(time.time()-s_t, math.ceil((time.time()-s_t)/60)))
if __name__ == "__main__":
#
logger.info('****** start ******* = {}'.format(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
thread_num = 10
once_handle_count = 1000
database_flag = 'skynet_test' # 配置数据库
insert_flag = 1 # 是否入库 0-不入库 1-入库
main(thread_num, database_flag, insert_flag, once_handle_count)