multiprocessing快速入门和总结

最新推荐文章于 2024-05-18 16:42:38 发布

2401_84182373

最新推荐文章于 2024-05-18 16:42:38 发布

阅读量889

点赞数 6

分类专栏： 2024年程序员学习文章标签： java python 算法

本文链接：https://blog.csdn.net/2401_84182373/article/details/137658634

版权

2024年程序员学习专栏收录该内容

63 篇文章 0 订阅

订阅专栏

return x\*x

if name == ‘__main__’:
# start 4 worker processes
with Pool(processes=4) as pool:

    # print "[0, 1, 4,..., 81]"
    print(pool.map(f, range(10)))


### 8.示例：多线程翻译


创建多个线程同时调用gpt4翻译，（gpt4调用代码没有贴出来，请自行封装）核心是多线程代码。

import logging.config
import multiprocessing
import os
import time
from datetime import datetime

import openpyxl
import yaml

from shenmo.TUnit import TUnit
from shenmo.app_utils import identify_language
from shenmo.os_utils import get_current_file_names, get_tree_file_names
from shenmo.pat_api_utils import call_strategy_api

定义常量：最大线程数

MAX_THREAD_COUNT = 10
MAX_SAVE_COUNT = 100
SYSTEM_MSG = 'You are a professional game translator and your task is to translate the text in the game. Chinese ’
‘translated into {}. The output includes only the translation.’

def setup_logging(default_path=“logging.yaml”, default_level=logging.INFO, env_key=“LOG_CFG”):
path = default_path
value = os.getenv(env_key, None)
if value:
path = value
if os.path.exists(path):
with open(path, “r”) as f:
config = yaml.load(f, Loader=yaml.FullLoader)
logging.config.dictConfig(config)
else:
logging.basicConfig(level=default_level)

def worker(in_queue, out_queue, language_type):
while True:
task = in_queue.get()
if task is None: # sentinel value indicating to exit
break
# 翻译内容
t1 = time.time()
system_msg = SYSTEM_MSG.format(language_type)
translated_value = call_strategy_api(task.get_original_text(), system_msg)
t2 = time.time()
# 设置耗时
task.set_cost_ms(t2 - t1)

    if translated_value.startswith("Error:") or translated_value.startswith("Request failed:"):
        task.set_error_msg(translated_value)
    else:
        task.set_translated_text(translated_value)
    out_queue.put(task)

def save_file_then_rename(origin_file_name, wb):
tmp_file_name = origin_file_name + “.tmp”
# 保存文件
wb.save(tmp_file_name)
# 删除原文件
os.remove(origin_file_name)
# 重命名临时文件
os.rename(tmp_file_name, origin_file_name)

def print_statistics(error_count, processed_count, task_count, t1):
t2 = time.time()
logging.info(“-” * 50)
logging.info(
f"已处理{processed_count}条，，异常{error_count}条，总共{task_count}条，进度：{(processed_count + error_count) / task_count:.2%}，已耗时：{t2 - t1:.2f}秒。保存文件中…“)
logging.info(”-" * 50)

class Translator:
def __init__(self, file_name):
self.file_name = file_name

def translate(self):
    in_queue = multiprocessing.Queue()
    out_queue = multiprocessing.Queue()

    # 打开Excel文件
    wb = openpyxl.load_workbook(self.file_name)

    # 选择sheet页“多语言翻译”
    sheet = wb.active

    # 循环遍历第2列，过滤前2行
    for row in sheet.iter_rows(min_row=2, max_row=sheet.max_row):
        # 第一列不为空，第二列为空
        if (row[0].value is not None and row[0].value.strip() != "") and (
                row[1].value is None or row[1].value.strip() == ""):
            row_num = row[1].row
            org_value = row[0].value
            tUnit = TUnit(row_num, org_value)
            # 将tUnit放入队列
            in_queue.put(tUnit)

    # in\_queue长度
    task_count = in_queue.qsize()
    identify_language_type = identify_language(self.file_name)
    logging.info(
        f"文件[{self.file\_name}]装载完成，语言类型：{identify\_language\_type}，共{task\_count}条翻译任务，创建{MAX\_THREAD\_COUNT}个线程，任务即将开始...")

    t1 = time.time()

    # Create and start worker processes
    processes = []
    for _ in range(MAX_THREAD_COUNT):
        process = multiprocessing.Process(target=worker, args=(in_queue, out_queue, identify_language_type))
        process.start()
        processes.append(process)

    # 已处理的任务数量
    processed_count = 0
    error_count = 0
    # out\_queue存在数据时，不断取出数据
    for _ in range(task_count):
        out = out_queue.get(block=True)
        if out.get_error_msg() is not None:
            error_count += 1
            logging.error(
                f"[{processed\_count}/{task\_count}]第{out.get\_row\_number()}行，翻译失败：{out.get\_original\_text()} -> {out.get\_error\_msg()}，耗时：{out.get\_cost\_ms():.2f}秒。")
        else:
            processed_count += 1
            logging.info(
                f"[{processed\_count}/{task\_count}]第{out.get\_row\_number()}行，翻译成功：{out.get\_original\_text()} -> {out.get\_translated\_text()}，耗时：{out.get\_cost\_ms():.2f}秒。")
            translate_text = out.get_translated_text()
            sheet.cell(row=out.get_row_number(), column=2).value = translate_text
            # 每处理100条，保存一次文件
            if processed_count % MAX_SAVE_COUNT == 0:
                print_statistics(error_count, processed_count, task_count, t1)
                save_file_then_rename(self.file_name, wb)

    if processed_count % MAX_SAVE_COUNT != 0:
        print_statistics(error_count, processed_count, task_count, t1)
        save_file_then_rename(self.file_name, wb)

    wb.close()
    logging.info(f"所有翻译任务处理完毕。")

    # 处理完毕，退出
    # Add sentinel values to signal the worker processes to exit
    for _ in range(MAX_THREAD_COUNT):
        in_queue.put(None)

    logging.info("已发送线程退出信号，等待线程退出...")

    # Wait for all processes to finish
    for process in processes:
        process.join()

    logging.info(f"所有线程已退出。")

if name == “__main__”:
# 创建文件夹：/export/logs/gpt/cache/
if not os.path.exists(“logs/”):
os.makedirs(“logs/”)

setup_logging()

# 允许用户输入线程数量，默认10
MAX_THREAD_COUNT = int(input("请输入线程数量（默认10）：") or "10")
logging.info(f"线程数量：{MAX\_THREAD\_COUNT}")

directory_path = "."
file_names_list = get_tree_file_names(directory_path, ".xlsx")
logging.info(f"文件序号\t文件名")
for file_name in file_names_list:
    # 打印：数组下标，文件名，语言类型
    logging.info(f"{file\_names\_list.index(file\_name)}\t{file\_name}")

index = -1
try:
    index = int(input("请输入要翻译的文件序号："))
except ValueError:
    index = -2
# 检测用户输入的序号合法，执行下面代码
if 0 <= index < len(file_names_list):
    selected_file_name = file_names_list[index]
    logging.info(f"你选择的文件是：[{index}]{selected\_file\_name}")

    translator = Translator(selected_file_name)
    translator.translate()
else:
    logging.error(f"输入的序号[{index}]不合法，程序即将退出。")

# 程序即将退出
logging.info("程序即将退出。")
# 暂停3秒
time.sleep(3)


### TUnit.py

class TUnit:
# 类变量
_id_alloc = 0

def \_\_init\_\_(self, row_number, original_text):
    TUnit._id_alloc += 1
    self._id = TUnit._id_alloc
    self.row_number = row_number
    self.original_text = original_text
    self.translated_text = None
    self.cost_ms = 0
    self.error_msg = None

# get id
def get\_id(self):
    return self._id

# get
def get\_row\_number(self):
    return self.row_number

def get\_original\_text(self):
    return self.original_text

def get\_translated\_text(self):
    return self.translated_text

# set translated\_text
def set\_translated\_text(self, translated_text):
    self.translated_text = translated_text
    return self.translated_text

# set cost\_ms
def set\_cost\_ms(self, cost_ms):
    self.cost_ms = cost_ms
    return self.cost_ms

def get\_cost\_ms(self):
    return self.cost_ms

# set error\_msg
def set\_error\_msg(self, error_msg):
    self.error_msg = error_msg
    return self.error_msg

def get\_error\_msg(self):
    return self.error_msg

def \_\_str\_\_(self):
    return f"row\_number:{self.row\_number}, original\_text:{self.original\_text}, translated\_text:{self.translated\_text}, cost\_ms:{self.cost\_ms}"


### os\_utils.py




**自我介绍一下，小编13年上海交大毕业，曾经在小公司待过，也去过华为、OPPO等大厂，18年进入阿里一直到现在。**

**深知大多数大数据工程师，想要提升技能，往往是自己摸索成长或者是报班学习，但对于培训机构动则几千的学费，着实压力不小。自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！**

**因此收集整理了一份《2024年大数据全套学习资料》，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友。**
![img](https://img-blog.csdnimg.cn/img_convert/19f2392215024a0aab1c141c49d510fa.png)
![img](https://img-blog.csdnimg.cn/img_convert/c986c5538e38a19a55f98c1ac85bd288.png)
![img](https://img-blog.csdnimg.cn/img_convert/7fea600ad02798fe27c99405a13e36f2.png)
![img](https://img-blog.csdnimg.cn/img_convert/0af6c59866d27b79a7d382b807bdd7e0.png)
![img](https://img-blog.csdnimg.cn/img_convert/d87f662f921d6fa8fa80ed24694d9634.png)

**既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，基本涵盖了95%以上大数据开发知识点，真正体系化！**

**由于文件比较大，这里只是将部分目录大纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新**

**如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）**
![img](https://img-blog.csdnimg.cn/img_convert/ed53a76114383c880032f940cb06cc51.png)

2853149504)]
[外链图片转存中...(img-xxhVhCsF-1712853149504)]

**既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，基本涵盖了95%以上大数据开发知识点，真正体系化！**

**由于文件比较大，这里只是将部分目录大纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新**

**如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）**
[外链图片转存中...(img-Iz4lYIeM-1712853149505)]

2401_84182373

关注

6
点赞
踩
10

收藏

觉得还不错? 一键收藏
0
评论
multiprocessing快速入门和总结

if value:else:break# 翻译内容# 设置耗时else:# 保存文件# 删除原文件# 重命名临时文件f"已处理{processed_count}条，，异常{error_count}条，总共{task_count}条，进度：{(processed_count + error_count) / task_count:.2%}，已耗时：{t2 - t1:.2f}秒。保存文件中…“)
复制链接

扫一扫