python多线程

低级接口

基本使用

join()方法用来阻塞主线程。可以注释掉、看看打印顺序。

import threading
import time

def test(n):
    time.sleep(1)
    print(n)

t = threading.Thread(target=test, args=(7,))
t.start()
t.join()   # 等待子线程执行完毕，才继续执行主线程。

print('主线程执行完毕')

Event

当Event被激活时候，所有等待Event的线程将全部激活！
event.set() 用来激活，event.wait() 等待事件发生。

import threading
import time

def produce(n, ee):
    time.sleep(1)    
    print(n)    
    print('go !!!')    
    ee.set()

def consumer(n, ee):    
    print('wait')    
    ee.wait()    
    print(f'{n} running......')

ee = threading.Event()
m = threading.Thread(target=produce, args=(0, ee))
t = threading.Thread(target=consumer, args=(1, ee))
t.start()
# m.start()

Semaphore

当信号量被释放，其他线程才能竞争这个信号量。
只有当线程通过竞争 获取到这个信号量，才能继续执行程序。

from threading import Semaphore, Thread
from time import sleep

def worker(n, sema):    
    sema.acquire()    
    sleep(1)    
    print(n)    
    # breakpoint()    
    sema.release()

sema = Semaphore(5)
for i in range(10):
   t = Thread(target=worker, args=(i, sema))    
   t.start()

线程通信

使用queue即可。用put()方法入队，get()方法获取队列元素。

from queue import Queue
import threading
import time


def producer(q):
    while True:
        time.sleep(5)
        q.put('apple')


def consumer(q):
    while True:
        print(f'eat {q.get()}')


if __name__ == '__main__':
    q = Queue(maxsize=5)
    for i in range(10):
        t1 = threading.Thread(target=producer, args=(q,))
        t1.start()
    t2 = threading.Thread(target=consumer, args=(q,))
    t2.start()

高级接口

请参考concurrent.futures模块

参考写法一（submit()）:

from concurrent.futures import ThreadPoolExecutor,ProcessPoolExecutor,as_completed
import time,random


def mytimeit(f):
    def ret():
        start_time = time.time()
        f()
        end_time = time.time()
        print('timeit : ',end_time-start_time)
    return ret


def ppow(a,b):
    time.sleep(1)
    return pow(a,b)

@mytimeit
def run():
    with ThreadPoolExecutor(max_workers=1000) as executor:
        all_tasks = {executor.submit(ppow,2,j):j for j in range(10000)}
        for task in as_completed(all_tasks):
            print(task.result(),all_tasks[task])


if __name__ == "__main__":
    run()

参考写法二(map())：

# python3.7 
# 更新pip安装的模块(多进程)
# 想用多线程，只需要把文件里的ProcessPoolExecutor全部替换成ThreadPoolExecutor即可
import subprocess
from concurrent.futures import ProcessPoolExecutor

res = subprocess.run('pip list -o',capture_output=True,text=True)

# 获取需要更新的模块
need_update=[item.split(' ')[0] for item in res.stdout.split('\n')[2:]]

with ProcessPoolExecutor(max_workers = 100) as exe:
    exe.map(subprocess.run,[f'pip install -U {name}' for name in need_update if name])

上面这个可能无法运行

import subprocess
from concurrent.futures import ProcessPoolExecutor

res = subprocess.run('pip list -o -i https://mirrors.aliyun.com/pypi/simple'.split(),stdout=subprocess.PIPE)

# 获取需要更新的模块
need_update=[item.split(' ')[0] for item in str(res).split('\\n')[2:]]

print(need_update)

with ProcessPoolExecutor(max_workers = 100) as exe:
    exe.map(subprocess.run,['pip install -i https://mirrors.aliyun.com/pypi/simple -U {}'.format(name).split() for name in need_update if name])