小黑受到了封校的恐惧，秋招结束该何去何从的日常积累：进程初步

内容来源

武老师b站:https://www.bilibili.com/video/BV1bK411A7tV/?spm_id_from=333.999.0.0

进程使用实例(name未被修改)

import multiprocessing
import time


def task(name):
    name.append(123)

if __name__ == '__main__':
    name = []
    p1 = multiprocessing.Process(target=task,args=(name,))
    p1.start()

    time.sleep(2)
    print(name)

[]

进程不能拷贝文件对象及线程锁等

import multiprocessing
import time


def task(name,file_object):
    print(name)
    print(file_object)

if __name__ == '__main__':
    name = []
    file_object = open('x1.txt',mode='a+',encoding='utf-8')
    p1 = multiprocessing.Process(target=task,args=(name,file_object))
    p1.start()

Traceback (most recent call last):
File “D:\python_file\notebook\python线程进程与携程\day1\1.1多线程.py”, line 13, in
p1.start()
File “D:\Anaconda\lib\multiprocessing\process.py”, line 121, in start
self._popen = self._Popen(self)
File “D:\Anaconda\lib\multiprocessing\context.py”, line 224, in _Popen
return _default_context.get_context().Process._Popen(process_obj)
File “D:\Anaconda\lib\multiprocessing\context.py”, line 327, in _Popen
return Popen(process_obj)
File “D:\Anaconda\lib\multiprocessing\popen_spawn_win32.py”, line 93, in init
reduction.dump(process_obj, to_child)
File “D:\Anaconda\lib\multiprocessing\reduction.py”, line 60, in dump
ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle ‘_io.TextIOWrapper’ object

案例1

import multiprocessing


def task():
    print(name)
    file_object.write("alex\n")
    file_object.flush()


if __name__ == '__main__':
    multiprocessing.set_start_method("fork")

    name = []
    file_object = open('x1.txt', mode='a+', encoding='utf-8')
    file_object.write("武沛齐\n")

    p1 = multiprocessing.Process(target=task)
    p1.start()

武沛齐
alex
武沛齐

案例2

import multiprocessing


def task():
    print(name)
    file_object.write("alex\n")
    file_object.flush()


if __name__ == '__main__':
    multiprocessing.set_start_method("fork")

    name = []
    file_object = open('x1.txt', mode='a+', encoding='utf-8')
    file_object.write("武沛齐\n")
    file_object.flush()

    p1 = multiprocessing.Process(target=task)
    p1.start()

武沛齐
alex

案例3

import multiprocessing
import threading
import time

def func():
    print('小黑来了')
    with lock:
        print(666)
        time.sleep(1)

def task():
    # 拷贝的锁是申请走的状态
    # 被谁申请走了?被子进程中的主线程申请走了
    for i in range(10):
        t = threading.Thread(target=func)
        t.start()
    time.sleep(2)
    lock.release()

if __name__ == '__main__':
    multiprocessing.set_start_method('fork')
    name = []
    lock = threading.RLock()
    lock.acquire()

    p1 = multiprocessing.Process(target=task)
    p1.start()

小黑来了
小黑来了
小黑来了小黑来了

小黑来了
小黑来了
小黑来了
小黑来了
小黑来了
小黑来了
666
666
666
666
666
666
666
666
666
666

常见功能

p.start()，当前进程准备就绪，等待被CPU调度（工作单元其实是进程中的线程）。

p.join()，等待当前进程的任务执行完毕后再向下继续执行。

实例1

import time
from multiprocessing import Process
import multiprocessing
def task(arg):
    time.sleep(2)
    print('执行中...')

if __name__ == '__main__':
    multiprocessing.set_start_method('spawn')
    p = Process(target=task,args=('xxx',))
    p.start()
    p.join()
    print('继续执行....')

执行中…
继续执行…

守护进程

p.daemon = 布尔值，守护进程（必须放在start之前）

p.daemon =True，设置为守护进程，主进程执行完毕后，子进程也自动关闭。
p.daemon =False，设置为非守护进程，主进程等待子进程，子进程执行完毕后，主进程才结束。

实例1

import time
from multiprocessing import Process
import multiprocessing
def task(arg):
    time.sleep(2)
    print('执行中...')

if __name__ == '__main__':
    multiprocessing.set_start_method('spawn')
    p = Process(target=task,args=('xxx',))
    p.daemon = True
    p.start()
    print('继续执行...')

继续执行…

Process finished with exit code 0

进程的名称的设置和获取

import os
import time
import threading
import multiprocessing

def func():
    time.sleep(3)

def task(arg):
    for i in range(10):
        t = threading.Thread(target=func)
        t.start()
    print(os.getpid(),os.getppid())
    print('线程个数:',len(threading.enumerate()))
    time.sleep(2)
    print('当前进程的名称:',multiprocessing.current_process().name)

if __name__ == '__main__':
    print(os.getpid())
    multiprocessing.set_start_method('spawn')
    p = multiprocessing.Process(target=task,args=('xxx',))
    p.name = '小黑无敌'
    a = multiprocessing.Process(target=task,args=('xxx',))
    a.name = '大黑驴'
    p.start()
    a.start()

    print('继续执行...')

22087
继续执行…
22154 22087
线程个数: 11
22155 22087
线程个数: 11
当前进程的名称: 小黑无敌
当前进程的名称: 大黑驴

Process finished with exit code 0

自定义进程类，直接将线程需要做的事写到run方法中。

import multiprocessing

class MyProcess(multiprocessing.Process):
    def run(self):
        print('执行此进程:',self._args)

if __name__ == '__main__':
    multiprocessing.set_start_method('spawn')
    p = MyProcess(args=('xxx',))
    p.start()
    print('继续执行....')

继续执行…
执行此进程: (‘xxx’,)

Process finished with exit code 0

import multiprocessing
multiprocessing.cpu_count()
print(multiprocessing.cpu_count())
import time
def task(args):
    time.sleep(1)
if __name__ == '__main__':
    count = multiprocessing.cpu_count()
    for i in range(count-1):
        p = multiprocessing.Process(target=task,args=('xxx',))
        p.start()

48

进程间的共享

进程是资源分配的最小单元，每个进程中都维护自己独立的数据，不共享。

import multiprocessing

def task(data):
    data.append(666)

if __name__ == '__main__':
    data_list = []
    p = multiprocessing.Process(target=task,args=(data_list,))
    p.start()
    p.join()
    print('主进程:',data_list)

主进程: []

Process finished with exit code 0

共享

Shared memory

Data can be stored in a shared memory map using Value or Array. For example, the following code

'c': ctypes.c_char,  'u': ctypes.c_wchar,
'b': ctypes.c_byte,  'B': ctypes.c_ubyte, 
'h': ctypes.c_short, 'H': ctypes.c_ushort,
'i': ctypes.c_int,   'I': ctypes.c_uint,  （其u表示无符号）
'l': ctypes.c_long,  'L': ctypes.c_ulong, 
'f': ctypes.c_float, 'd': ctypes.c_double

from multiprocessing import Process,Value,Array

def func(n,m1,m2):
    n.value = 888
    m1.value = 'a'.encode('utf-8')
    m2.value = '黑'

if __name__ == '__main__':
    num = Value('i',666)
    v1 = Value('c')
    v2 = Value('u')

    p = Process(target=func,args=(num,v1,v2))
    p.start()
    p.join()

    print(num.value)
    print(v1.value)
    print(v2.value)

888
b’a’
黑

from multiprocessing import Process,Value,Array

def f(data_array):
    data_array[0] = 666

if __name__ == '__main__':
    arr = Array('i',[11,22,33,44])
    p = Process(target=f,args=(arr,))
    p.start()
    p.join()
    print(arr[:])

[666, 22, 33, 44]

Process finished with exit code 0

Server process

A manager object returned by Manager() controls a server process which holds Python objects and allows other processes to manipulate them using proxies.

from multiprocessing import Process,Manager

def f(d,l):
    d[1] = 'l'
    d['2'] = 2
    d[0.25] = None
    l.append(666)

if __name__ == '__main__':
    with Manager() as manager:
        d = manager.dict()
        l = manager.list()

        p = Process(target=f,args=(d,l))
        p.start()
        p.join()

        print(d)
        print(l)

{1: ‘l’, ‘2’: 2, 0.25: None}
[666]

Process finished with exit code 0

2.2 交换

multiprocessing supports two types of communication channel between processes

Queues

import multiprocessing

def task(q):
    for i in range(10):
        q.put(i)

if __name__ == '__main__':
    queue = multiprocessing.Queue()

    p = multiprocessing.Process(target=task,args=(queue,))
    p.start()
    p.join()

    for i in range(5):
        print(queue.get())

0
1
2
3
4

Pipes

import time
import multiprocessing

def task(conn):
    time.sleep(1)
    conn.send([111,22,33,44])
    data = conn.recv()   # 阻塞
    print('子进程接收',data)
    time.sleep(2)

if __name__ == '__main__':
    parent_conn,child_conn = multiprocessing.Pipe()

    p = multiprocessing.Process(target=task,args=(child_conn,))
    p.start()

    info = parent_conn.recv()    # 阻塞
    print('主进程接收:',info)
    parent_conn.send(666)

主进程接收: [111, 22, 33, 44]
子进程接收 666

Process finished with exit code 0

进程锁

案例

import time
import multiprocessing

def task():
    # 假设文件中保存的内容是一个值:10
    with open('f1.txt',mode='r',encoding='utf-8') as f:
        current_num = int(f.read())
    print('排队抢票了')
    time.sleep(1)
    current_num -= 1
    with open('f1.txt',mode='w',encoding='utf-8') as f:
        f.write(str(current_num))

if __name__ == '__main__':
    for i in range(20):
        p = multiprocessing.Process(target=task)
        p.start()

排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了

import time
import multiprocessing

def task(lock):
    print('开始')
    lock.acquire()
    # 假设文件中保存的内容是一个值:4
    with open('f1.txt',mode='r',encoding='utf-8') as f:
        current_num = int(f.read())
    print('排队抢票了')
    time.sleep(1)
    current_num -= 1
    with open('f1.txt',mode='w',encoding='utf-8') as f:
        f.write(str(current_num))
    lock.release()
if __name__ == '__main__':
    multiprocessing.set_start_method('spawn')
    lock = multiprocessing.RLock()

    for i in range(20):
        p = multiprocessing.Process(target=task,args=(lock,))
        p.start()
    time.sleep(7)

开始
排队抢票了
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
开始
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了

Process finished with exit code 0

import time
import multiprocessing
import os

def task(lock):
    print('开始')
    lock.acquire()
    # 假设文件中保存的内容就是一个值:10
    with open('f1.txt',mode='r',encoding='utf-8') as f:
        current_num = int(f.read())
    print(os.getpid(),'排队抢票了')
    time.sleep(0.5)
    current_num -= 1
    with open('f1.txt', mode='w', encoding='utf-8') as f:
        f.write(str(current_num))
    lock.release()

if __name__ == '__main__':
    multiprocessing.set_start_method('spawn')
    lock = multiprocessing.RLock()
    process_list = []

    for i in range(10):
        p = multiprocessing.Process(target=task,args=(lock,))
        p.start()
        process_list.append(p)
    # spawn模式,需要特殊处理
    for item in process_list:
        item.join()

开始
46822 排队抢票了
开始开始

开始
开始
开始
开始
开始
开始
开始
46824 排队抢票了
46821 排队抢票了
46826 排队抢票了
46828 排队抢票了
46820 排队抢票了
46823 排队抢票了
46829 排队抢票了
46825 排队抢票了
46827 排队抢票了

Process finished with exit code 0

进程池

import time
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor

def task(num):
    print('执行',num)
    time.sleep(2)

if __name__ == '__main__':
    # 修改模式
    pool = ProcessPoolExecutor(4)
    for i in range(10):
        pool.submit(task,i)
    print('主进程结束!!!')

主进程结束!!!
执行 0
执行 1
执行 2
执行 3
执行 4
执行 5
执行 6
执行 7
执行 8
执行 9

Process finished with exit code 0

import time
from concurrent.futures import ProcessPoolExecutor,ThreadPoolExecutor

def task(num):
    print('执行',num)
    time.sleep(2)

if __name__ == '__main__':
    # 修改模式
    pool = ProcessPoolExecutor(4)
    for i in range(10):
        pool.submit(task,i)
    pool.shutdown(True)
    print('主进程结束!!!')

执行 0
执行 1
执行 2
执行 3
执行 4
执行 5
执行 6
执行 7
执行 8
执行 9
主进程结束!!!

import time
from concurrent.futures import ProcessPoolExecutor
import multiprocessing

def task(num):
    print('执行',num)
    time.sleep(2)
    return num

def done(res):
    # 回调函数由主进程执行，而线程池是由子线程
    print(multiprocessing.current_process())
    time.sleep(1)
    print(res.result())
    time.sleep(1)

if __name__ == '__main__':
    pool = ProcessPoolExecutor(4)
    for i in range(5):
        fur = pool.submit(task,i)
        fur.add_done_callback(done)
    print(multiprocessing.current_process())
    pool.shutdown(True)

<_MainProcess name=‘MainProcess’ parent=None started>
执行 0
执行 1
执行 2
执行 3
执行 4
<_MainProcess name=‘MainProcess’ parent=None started>
0
<_MainProcess name=‘MainProcess’ parent=None started>
2
<_MainProcess name=‘MainProcess’ parent=None started>
1
<_MainProcess name=‘MainProcess’ parent=None started>
3
<_MainProcess name=‘MainProcess’ parent=None started>
4

Process finished with exit code 0

注意：如果在进程池中要使用进程锁，则需要基于Manager中的Lock和RLock来实现。

import time
import multiprocessing
from concurrent.futures.process import ProcessPoolExecutor

def task(lock):
    print('开始')

    with lock:
        with open('f1.txt', mode='r', encoding='utf-8') as f:
            current_num = int(f.read())

        print('排队抢票了')
        time.sleep(1)
        current_num -= 1

        with open('f1.txt', mode='w', encoding='utf-8') as f:
            f.write(str(current_num))

if __name__ == '__main__':
    pool = ProcessPoolExecutor()
    manager = multiprocessing.Manager()
    lock_object = manager.RLock()

    for i in range(10):
        pool.submit(task,lock_object)

开始
开始
排队抢票了
开始
开始
开始
开始
开始
开始
开始
开始
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了
排队抢票了

Process finished with exit code 0

小黑生活

补发一下昨晚的喜茶