python 多进程

import multiprocessing,sys
import pandas as pd
from loguru import logger
sys.path.append("..")
from QhSpiderTool import QhStarEndTime,QhJiSuanJinCheng,QhTimestamp
from QhCsvMode import QHDFDBJSON

@QhStarEndTime
def QhConsumerN(QhQueue):
    """
    多进程消费者数据汇总  改进版  不输出日志文件，输出日志
    作者：阙辉
    """
    QhiM = 1 
    QhiD = 1 
    QhLogList = []
    QhJieGuoDf = pd.DataFrame()
    while True:
        Qhitem = QhQueue.get()  # 从队列中获取数据
        if Qhitem is None:  # 消息队列退出机制
            logger.info("【消费消息队列】队列消费完毕！QueHui!")
            QhQueue.task_done()  # 标记任务完成
            break
        try:
            QhKey,QhValue = Qhitem
        except:
            logger.error("【消费消息队列】消息格式错误！QueHui!")
            continue
        
        if QhKey == "QhMsg":
            logger.info("【消费消息队列】消费队列第{}条消息日志！QueHui!".format(QhiM))
            QhLogList.append(QhValue)
            QhiM += 1
        elif QhKey == "QhData":
            logger.info("【消费消息队列】消费队列第{}条值数据！QueHui!".format(QhiD))
            if QhiD == 1:
                QhJieGuoDf = QhValue
            else:
                try:  # 兼容旧版本处理
                    QhJieGuoDf = QhJieGuoDf._append(QhValue)
                except:
                    QhJieGuoDf = QhJieGuoDf.append(QhValue)
            QhiD += 1
        QhQueue.task_done()  # 标记任务完成

    QhLogdf = pd.DataFrame(QhLogList,columns=QHDFDBJSON["QhErrorDf"]["QhFiled"])  

    return QhLogdf,QhJieGuoDf


@QhStarEndTime
def QhMultiProcessingMainN(QhSequence,QhFunction,QhCount=0,QhCpuCounts=1,*Qhargs,**Qhkwargs):
    """
    作者：阙辉
    功能：多进程生产者 改进传输参数
    QhSequence: 序列
    QhFunction: 执行方法
    QhCount: 每组数量，大于0的话会自动计算线程数
    QhCpuCounts: cpu的倍数，大于的话，线程数=CPU数量*QhCpuCounts
    不做任何存储，和实际的逻辑，主要是负责分配进程和收集计算的结果
    """
    QhManager = multiprocessing.Manager()
    QhQueue = QhManager.Queue()   # 创建消息队列
    if isinstance(QhSequence,pd.DataFrame):  # 计算序列的长度
        QhSequenceCount = QhSequence.shape[0]
    else:
        QhSequenceCount = len(QhSequence)
    QhTotalTimes,qh_group_count = QhJiSuanJinCheng(QhSequenceCount,QhCount=QhCount,QhCpuCounts=QhCpuCounts)
    QhQueue.put(("QhMsg",["QhMultiProcessingMain",QhTimestamp(),
                          "1、读取所有股票数据共{}条，建{}条进程，分组数{}条*QueHui"
                          .format(QhSequenceCount,QhTotalTimes,qh_group_count),"","Success",""]))
    QhProcessesList = []
    QhStart = 0
    for QhRow in range(1,QhTotalTimes+1):
        QhGropData = QhSequence[QhStart:QhRow*qh_group_count]
        QhStart = QhRow*qh_group_count 
        QhargsN = (QhGropData,QhQueue) + Qhargs # 固定参数+传入参数
        QhProcesses = multiprocessing.Process(target=QhFunction, args=QhargsN, kwargs=Qhkwargs)  # 创建进程
        QhProcessesList.append(QhProcesses)  # 添加进程到进程列表
        QhProcesses.start()  # 启动进程
    for QhRow in QhProcessesList:
        QhRow.join()  # 等待进程结束

    QhQueue.put(None)  # 发送结束信号
    QhJieGuoQ = QhConsumerN(QhQueue,QhFunction)
    QhLogdf = QhJieGuoQ[0]
    QhJieGuodf = QhJieGuoQ[1]

    return QhLogdf,QhJieGuodf