# 根据环境变量信息去选择配置信息
env = os.getenv('ENV', 'dev')
assert env in ['dev', 'st', 'uat', 'prd']
Config = locals().get(env.capitalize())
# MYSQL sqlalchemy
from retrying import retry
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from sqlalchemy.ext.declarative import declarative_baseBase = declarative_base()
# 配置
Mysql = {
'url': 'mysql+pymysql://cmbuser:Cmb%402018@99.13.211.225:3306/tianji_ai',
'pool_recycle': 180,
'pool_size': 10,
'pool_timeout': 200,
'echo_pool': True,
'pool_pre_ping': True,
'pool_use_lifo': True
}class Session:
def __init__(self):
"""延迟实例化"""
self.scoped_session = Nonedef init_config(self, *args, **config):
engine = create_engine(*args, **config)
session_factory = sessionmaker(bind=engine)
self.scoped_session = scoped_session(session_factory)def execute_sql(self, sql):
ss = self.scoped_session()
try:
return ss.execute(sql)
finally:
ss.close()def with_db_session(func):
def decorate(*args, **kwargs):
try:
ss = session.scoped_session()
return func(*args, **kwargs, ss=ss)
finally:
ss.close()
return decorate# 三次尝试 @retry
@retry(stop_max_attempt_number=3)
def ensure_execute(ss, sql):
try:
resp = ss.execute(sql)
ss.commit()
return resp
except Exception as e:
ss.rollback()
raise Exception('mysql 三次尝试 连接丢失')
session = Session() # dbORM-使用:
class Task(Base):
__tablename__='tablename'
id = Column(primary_key=True)原生sql:
@with_db_session
def func(ss=None):
#查询
data = ss.execute(sql).first()
#添加
encure_execute(ss,sql)
# redis 集群
# 配置
Redis = {'startup_nodes': [{'host': '0.0.0.0', 'port': '6379'},
{'host': '0.0.0.0', 'port': '6379'}],
'password': 'password',
'skip_full_coverage_check': True
}import rediscluster
class RedisClst:
def __init__(self):
self.client = Nonedef init_config(self, **config):
self.client = rediscluster.RedisCluster(**config)def acquire_lock(self, lock_key, lock_val, expire_time=100):
"""获取锁
"""
ret = self.client.set(lock_key, lock_val, ex=expire_time, nx=True)
if ret:
return ret
return Falsedef release_lock(self, lock_key):
"""释放锁
"""
self.client.delete(lock_key) # 直接释放锁# 可以使用多个实例来进行不同的配置
redis = RedisClst()
tj_redis = RedisClst()
# kafa 生产者配置
Kafka = {
'hosts': "",
'mechanism': 'PLAIN',
'protocol': 'SASL_PLAINTEXT',
'user': 'user',
'pwd': 'userw0rd',
'topic': 'msg-sink'
}from confluent_kafka import Producer as RawKfkProducer
class KfkProducer:def __init__(self):
self.topic = None
self.producer = Nonedef init_config(self, hosts, topic, user=None, pwd=None, **kwargs):
mechanism = kwargs.get('mechanism', 'PLAIN')
protocol = kwargs.get('protocol', 'SASL_PLAINTEXT')if not user:
conf = {
'bootstrap.servers': hosts
}
else:
conf = {
'bootstrap.servers': hosts,
'sasl.mechanism': mechanism,
'security.protocol': protocol,
'sasl.username': user,
'sasl.password': pwd
}
self.topic = topic
self.producer = RawKfkProducer(**conf)def send_msg(self, msg): # 生产者
if isinstance(msg, dict):
msg = json.dumps(msg)
self.producer.produce(self.topic, value=msg)
self.producer.flush(timeout=1)
kproducer = KfkProducer()# 消费者
from confluent_kafka import Consumer消费配置:
conf = {'group.id':topic_group,'auto.offset.reset':'earliest'}+普通配置
consumer = Consumer(**conf)
consumer.subscribe([topic_name])
msg = consumer.poll()
data_ = msg.value().decode()
# 基于pydantic 做数据格式校验
from pydantic import BaseModel, Fieldclass _SaveAnalysis(BaseModel):
task_id: str = Field(...)
company_list: List[str] = Field([], title='企业列表')
class SaveAnalysis(BaseModel):
user: User = Field(..., title='用户信息')
data: _SaveAnalysis = Field(..., title='模型参数')
def input(schema):
"""输入校验"""
def decorate(f):
def deco(*args, **kwargs):
try:
req_json = request.get_json()
req_json = schema(**req_json).dict(skip_defaults=False)
logger.info(f'request info: url {request.url}, json: {req_json}')
except Exception as e:
logger.error(f'参数解析异常:{e}')
raise AppException('参数解析异常, 请检查参数')return f(req_json, *args, **kwargs)
return deco
return decorate# 案例
@make_response
@input(analysis_schema.DrawAnalysis)
def draw_analysis_board(data):
task_id = data['data']['task_id']
user = data['user']
return result.draw_analysis_board(task_id, user)
# 统一响应
def make_response(func):
"""统一的响应请求"""
@wraps(func)
def decorate(*args, **kwargs):
code, msg, data = ResponseCode.Unknown.value, '服务异常, 请联系管理人员', {} # 默认返回信息
try:
data = func(*args, **kwargs)
code, msg = ResponseCode.Success.value, ''
except AppException as e:
logger.error(f'App内部错误: {e}')
logger.error(f'错误详情: {traceback.format_exc()}')
code, msg = ResponseCode.Fail.value, str(e)
except Exception as e:
logger.error(f'系统未知错误: {e}')
logger.error(f'错误详情: {traceback.format_exc()}')
finally:
logger.info(f'request info: url {request.url} code: {code}')
resp = {'code': code, 'msg': msg, 'data': data}
return resp
return decorate
# es查询封装,以及查询全部
from elasticsearch import Elasticsearch
class ElasticClient:def __init__(self):
self.client = Nonedef init_config(self, hosts, user=None, password=None):
if user and password:
self.client = Elasticsearch(hosts=hosts, http_auth=(user, password),timeout=500,
max_retries=3,retry_on_timeout=True)
else:
self.client = Elasticsearch(hosts=hosts)def search(self, index, body, params):
"""查询数据量小于1w 的"""
logger.debug(f'index: {index}, body: {body}, params: {params}')
return self.client.search(index=index, body=body, params=params)def scroll_search(self, index, body, params, size=10000, scroll='5m', timeout='2m', limit=100000):
"""查询全量数据 或 大于1w 条数据的"""# 添加默认参数
filter_path = ['hits.hits._source', '_scroll_id', 'hits.total']
_params = {'track_scores': 'false', 'search_type': 'query_then_fetch'}
params = {**_params, **params}def generate_resp(_scroll_id, _datas, _total, _limit):
self.client.clear_scroll(scroll_id=_scroll_id)
_resp = {'total': total, 'datas': _datas[:limit]}
return _resplogger.debug(f'index: {index}, body: {body}, params: {params}, size: {size}, limit: {limit}')
resp = self.client.search(index=index, body=body, params=params, scroll=scroll, timeout=timeout, size=size,
filter_path=filter_path)
scroll_id = resp['_scroll_id']
total = resp['hits']['total']
datas = [data['_source'] for data in resp.get('hits', {}).get('hits', [])]
if not datas:
logger.warning('no data to translate. exit')
return generate_resp(scroll_id, datas, total, limit)if len(datas) >= limit:
return generate_resp(scroll_id, datas, total, limit)while True:
res = self.client.scroll(scroll_id=scroll_id, scroll=scroll)
page_data = res.get('hits', {}).get('hits', [])
if not page_data:
break
datas.extend([item['_source'] for item in page_data])if len(datas) >= limit:
return generate_resp(scroll_id, datas, total, limit)
return generate_resp(scroll_id, datas, total, limit)def count(self, index, body):
logger.debug(f'index: {index}, body: {body}')
resp = self.client.count(index=index, body=body)
logger.debug(f'index: {index}, body: {body}, count: {resp["count"]}')
return resp['count']# 配置信息
conf = {'host':{'host':ip'','port':端口},'user':None,'password':None}
配置时需要**
# 分片返回
def slice_iter(data, size=5000):
times = len(data) // size + 1
for t in range(times):
item = data[t * size: (t + 1) * size]
if item:
yield item
# 动态导入
from importlib import import_model
def register_buleprint(app,blueprints):
"""动态蓝图注册"""
for blueprint_name in blueprints:
bp_module = import_model(blueprint_name)
app.register_blueprint(bp_module.bq)
func_name = sys.argv[1] def start_trial(): requests.post('http://192.168.30.85:5005/jobs/test') def finish_trial(): requests.put('http://192.168.30.85:5005/jobs/test') 执行函数 locals().get(func_name)()