multiprocessing快速入门和总结，2024年最新面试阿里

• Pipes

5.Synchronization between processes

• Lock

6.Sharing state between processes

• Shared memory: Value or Array
• Server process: Manager

7.Using a pool of workers

from multiprocessing import Pool, TimeoutError
import time
import os

def f(x):
return x*x

if name == ‘__main__’:

start 4 worker processes

with Pool(processes=4) as pool:

print “[0, 1, 4,…, 81]”

print(pool.map(f, range(10)))

8.示例：多线程翻译

创建多个线程同时调用gpt4翻译，（gpt4调用代码没有贴出来，请自行封装）核心是多线程代码。

import logging.config
import multiprocessing
import os
import time
from datetime import datetime

import openpyxl
import yaml

from shenmo.TUnit import TUnit
from shenmo.app_utils import identify_language
from shenmo.os_utils import get_current_file_names, get_tree_file_names
from shenmo.pat_api_utils import call_strategy_api

定义常量：最大线程数

MAX_THREAD_COUNT = 10
MAX_SAVE_COUNT = 100
SYSTEM_MSG = 'You are a professional game translator and your task is to translate the text in the game. Chinese ’
‘translated into {}. The output includes only the translation.’

def setup_logging(default_path=“logging.yaml”, default_level=logging.INFO, env_key=“LOG_CFG”):
path = default_path
value = os.getenv(env_key, None)
if value:
path = value
if os.path.exists(path):
with open(path, “r”) as f:
config = yaml.load(f, Loader=yaml.FullLoader)
logging.config.dictConfig(config)
else:
logging.basicConfig(level=default_level)

def worker(in_queue, out_queue, language_type):
while True:
task = in_queue.get()
if task is None: # sentinel value indicating to exit
break

翻译内容

t1 = time.time()
system_msg = SYSTEM_MSG.format(language_type)
translated_value = call_strategy_api(task.get_original_text(), system_msg)
t2 = time.time()

设置耗时

task.set_cost_ms(t2 - t1)

if translated_value.startswith(“Error:”) or translated_value.startswith(“Request failed:”):
task.set_error_msg(translated_value)
else:
task.set_translated_text(translated_value)
out_queue.put(task)

def save_file_then_rename(origin_file_name, wb):
tmp_file_name = origin_file_name + “.tmp”

保存文件

wb.save(tmp_file_name)

删除原文件

os.remove(origin_file_name)

重命名临时文件

os.rename(tmp_file_name, origin_file_name)

def print_statistics(error_count, processed_count, task_count, t1):
t2 = time.time()
logging.info(“-” * 50)
logging.info(
f"已处理{processed_count}条，，异常{error_count}条，总共{task_count}条，进度：{(processed_count + error_count) / task_count:.2%}，已耗时：{t2 - t1:.2f}秒。保存文件中…“)
logging.info(”-" * 50)

class Translator:
def __init__(self, file_name):
self.file_name = file_name

def translate(self):
in_queue = multiprocessing.Queue()
out_queue = multiprocessing.Queue()

打开Excel文件

wb = openpyxl.load_workbook(self.file_name)

选择sheet页“多语言翻译”

sheet = wb.active

循环遍历第2列，过滤前2行

for row in sheet.iter_rows(min_row=2, max_row=sheet.max_row):

第一列不为空，第二列为空

if (row[0].value is not None and row[0].value.strip() != “”) and (
row[1].value is None or row[1].value.strip() == “”):
row_num = row[1].row
org_value = row[0].value
tUnit = TUnit(row_num, org_value)

将tUnit放入队列

in_queue.put(tUnit)

in_queue长度

task_count = in_queue.qsize()
identify_language_type = identify_language(self.file_name)
logging.info(
f"文件[{self.file_name}]装载完成，语言类型：{identify_language_type}，共{task_count}条翻译任务，创建{MAX_THREAD_COUNT}个线程，任务即将开始…")

t1 = time.time()

Create and start worker processes

processes = []
for _ in range(MAX_THREAD_COUNT):
process = multiprocessing.Process(target=worker, args=(in_queue, out_queue, identify_language_type))
process.start()
processes.append(process)

已处理的任务数量

processed_count = 0
error_count = 0

out_queue存在数据时，不断取出数据

for _ in range(task_count):
out = out_queue.get(block=True)
if out.get_error_msg() is not None:
error_count += 1
logging.error(
f"[{processed_count}/{task_count}]第{out.get_row_number()}行，翻译失败：{out.get_original_text()} -> {out.get_error_msg()}，耗时：{out.get_cost_ms():.2f}秒。“)
else:
processed_count += 1
logging.info(
f”[{processed_count}/{task_count}]第{out.get_row_number()}行，翻译成功：{out.get_original_text()} -> {out.get_translated_text()}，耗时：{out.get_cost_ms():.2f}秒。")
translate_text = out.get_translated_text()
sheet.cell(row=out.get_row_number(), column=2).value = translate_text

每处理100条，保存一次文件

if processed_count % MAX_SAVE_COUNT == 0:
print_statistics(error_count, processed_count, task_count, t1)
save_file_then_rename(self.file_name, wb)

if processed_count % MAX_SAVE_COUNT != 0:
print_statistics(error_count, processed_count, task_count, t1)
save_file_then_rename(self.file_name, wb)

wb.close()
logging.info(f"所有翻译任务处理完毕。")

处理完毕，退出

Add sentinel values to signal the worker processes to exit

for _ in range(MAX_THREAD_COUNT):
in_queue.put(None)

logging.info(“已发送线程退出信号，等待线程退出…”)

Wait for all processes to finish

for process in processes:
process.join()

logging.info(f"所有线程已退出。")

自我介绍一下，小编13年上海交大毕业，曾经在小公司待过，也去过华为、OPPO等大厂，18年进入阿里一直到现在。

深知大多数大数据工程师，想要提升技能，往往是自己摸索成长或者是报班学习，但对于培训机构动则几千的学费，着实压力不小。自己不成体系的自学效果低效又漫长，而且极易碰到天花板技术停滞不前！

因此收集整理了一份《2024年大数据全套学习资料》，初衷也很简单，就是希望能够帮助到想自学提升又不知道该从何学起的朋友。

既有适合小白学习的零基础资料，也有适合3年以上经验的小伙伴深入学习提升的进阶课程，基本涵盖了95%以上大数据开发知识点，真正体系化！

由于文件比较大，这里只是将部分目录大纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新

如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）

合3年以上经验的小伙伴深入学习提升的进阶课程，基本涵盖了95%以上大数据开发知识点，真正体系化！**

由于文件比较大，这里只是将部分目录大纲截图出来，每个节点里面都包含大厂面经、学习笔记、源码讲义、实战项目、讲解视频，并且后续会持续更新

如果你觉得这些内容对你有帮助，可以添加VX：vip204888 （备注大数据获取）
[外链图片转存中…(img-aJfI6AA5-1712528782261)]