import multiprocessing,sys
import pandas as pd
from loguru import logger
sys.path.append("..")
from QhSpiderTool import QhStarEndTime,QhJiSuanJinCheng,QhTimestamp
from QhCsvMode import QHDFDBJSON
@QhStarEndTime
def QhConsumerN(QhQueue):
"""
多进程消费者数据汇总 改进版 不输出日志文件,输出日志
作者:阙辉
"""
QhiM = 1
QhiD = 1
QhLogList = []
QhJieGuoDf = pd.DataFrame()
while True:
Qhitem = QhQueue.get() # 从队列中获取数据
if Qhitem is None: # 消息队列退出机制
logger.info("【消费消息队列】队列消费完毕!QueHui!")
QhQueue.task_done() # 标记任务完成
break
try:
QhKey,QhValue = Qhitem
except:
logger.error("【消费消息队列】消息格式错误!QueHui!")
continue
if QhKey == "QhMsg":
logger.info("【消费消息队列】消费队列第{}条消息日志!QueHui!".format(QhiM))
QhLogList.append(QhValue)
QhiM += 1
elif QhKey == "QhData":
logger.info("【消费消息队列】消费队列第{}条值数据!QueHui!".format(QhiD))
if QhiD == 1:
QhJieGuoDf = QhValue
else:
try: # 兼容旧版本处理
QhJieGuoDf = QhJieGuoDf._append(QhValue)
except:
QhJieGuoDf = QhJieGuoDf.append(QhValue)
QhiD += 1
QhQueue.task_done() # 标记任务完成
QhLogdf = pd.DataFrame(QhLogList,columns=QHDFDBJSON["QhErrorDf"]["QhFiled"])
return QhLogdf,QhJieGuoDf
@QhStarEndTime
def QhMultiProcessingMainN(QhSequence,QhFunction,QhCount=0,QhCpuCounts=1,*Qhargs,**Qhkwargs):
"""
作者:阙辉
功能:多进程生产者 改进传输参数
QhSequence: 序列
QhFunction: 执行方法
QhCount: 每组数量,大于0的话会自动计算线程数
QhCpuCounts: cpu的倍数,大于的话,线程数=CPU数量*QhCpuCounts
不做任何存储,和实际的逻辑,主要是负责分配进程和收集计算的结果
"""
QhManager = multiprocessing.Manager()
QhQueue = QhManager.Queue() # 创建消息队列
if isinstance(QhSequence,pd.DataFrame): # 计算序列的长度
QhSequenceCount = QhSequence.shape[0]
else:
QhSequenceCount = len(QhSequence)
QhTotalTimes,qh_group_count = QhJiSuanJinCheng(QhSequenceCount,QhCount=QhCount,QhCpuCounts=QhCpuCounts)
QhQueue.put(("QhMsg",["QhMultiProcessingMain",QhTimestamp(),
"1、读取所有股票数据共{}条,建{}条进程,分组数{}条*QueHui"
.format(QhSequenceCount,QhTotalTimes,qh_group_count),"","Success",""]))
QhProcessesList = []
QhStart = 0
for QhRow in range(1,QhTotalTimes+1):
QhGropData = QhSequence[QhStart:QhRow*qh_group_count]
QhStart = QhRow*qh_group_count
QhargsN = (QhGropData,QhQueue) + Qhargs # 固定参数+传入参数
QhProcesses = multiprocessing.Process(target=QhFunction, args=QhargsN, kwargs=Qhkwargs) # 创建进程
QhProcessesList.append(QhProcesses) # 添加进程到进程列表
QhProcesses.start() # 启动进程
for QhRow in QhProcessesList:
QhRow.join() # 等待进程结束
QhQueue.put(None) # 发送结束信号
QhJieGuoQ = QhConsumerN(QhQueue,QhFunction)
QhLogdf = QhJieGuoQ[0]
QhJieGuodf = QhJieGuoQ[1]
return QhLogdf,QhJieGuodf
python 多进程
于 2024-08-30 11:46:56 首次发布