python 多线程、多进程和数据队列

最新推荐文章于 2024-07-10 09:15:55 发布

江凌

最新推荐文章于 2024-07-10 09:15:55 发布

阅读量477

点赞数

分类专栏：线程文章标签： python 开发语言

本文链接：https://blog.csdn.net/xdkb159/article/details/128616736

版权

线程专栏收录该内容

2 篇文章 0 订阅

订阅专栏

一、多线程

pytohn多线程因为全局GIL锁的存在，不会使用多核运行，而是多个线程去竞争cpu一个核的锁，所以对于计算密集型任务，多线程并不能提高效率。

#!/usr/bin/python
# encoding:utf-8
import threading
import queue
import time
from concurrent.futures import ThreadPoolExecutor
from log.log import get_logger


def counter(n=10200000):
    # 10200000累计求和耗时接近1秒
    t = 0
    for i in range(n):
        t += i
    return t


def run(idx, num):
    log.info('线程【{0}】开始工作'.format(idx))
    while num > 0:
        num -= 1
        # time.sleep(1)  # sleep不占用cpu
        start = time.time()
        counter()
        log.info('线程【{0}】耗时：{1}'.format(idx, time.time()-start))

    log.info('线程【{0}】工作完成'.format(idx))


def start_sync_thread():
    log.info('主进程启动')
    max_worker_num = 4
    pool = ThreadPoolExecutor(max_workers=max_worker_num)
    num = 60
    future_list = []
    for i in range(max_worker_num):
        param = [i, num / max_worker_num]
        future = pool.submit(lambda p: run(*p), param)
        future_list.append(future)
    while True:
        flag = True
        for future in future_list:
            flag = future.done()
            if not flag:
                time.sleep(1)
                break
        if flag:
            break
    log.info('主进程结束')


if __name__ == '__main__':
    start_sync_thread()

执行结果如下，多个线程竞争单核的计算时间片，导致单个线程的计算任务耗时变长，多线程最终总任务的计算时间并没有缩短（对于计算密集型任务，如果run方法里不是计算counter函数，而是用sleep，多线程是可以提升效率的）。

"C:\Program Files\Python\Python37\python.exe" G:/workspace/syncdata/sync_data_thread_pool.py
2023-02-07 10:40:35,915 - G:/workspace/syncdata/sync_data_thread_pool.py[line:129] - INFO - 71236: 主进程启动
2023-02-07 10:40:35,916 - G:/workspace/syncdata/sync_data_thread_pool.py[line:117] - INFO - 18492: 进程【0】开始工作
2023-02-07 10:40:35,952 - G:/workspace/syncdata/sync_data_thread_pool.py[line:117] - INFO - 73104: 进程【1】开始工作
2023-02-07 10:40:36,110 - G:/workspace/syncdata/sync_data_thread_pool.py[line:117] - INFO - 46796: 进程【2】开始工作
2023-02-07 10:40:36,315 - G:/workspace/syncdata/sync_data_thread_pool.py[line:117] - INFO - 44980: 进程【3】开始工作
2023-02-07 10:40:38,814 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：2.846147298812866
2023-02-07 10:40:39,257 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.340761423110962
2023-02-07 10:40:40,489 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：4.173534631729126
2023-02-07 10:40:40,505 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.39494252204895
2023-02-07 10:40:41,998 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.1385769844055176
2023-02-07 10:40:42,294 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.035914659500122
2023-02-07 10:40:44,045 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.450774669647217
2023-02-07 10:40:44,561 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：4.044219255447388
2023-02-07 10:40:45,953 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.872706413269043
2023-02-07 10:40:46,534 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：4.238697528839111
2023-02-07 10:40:47,650 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.5256357192993164
2023-02-07 10:40:48,587 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：4.025278568267822
2023-02-07 10:40:48,934 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：2.981029748916626
2023-02-07 10:40:50,427 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.688110589981079
2023-02-07 10:40:51,699 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.833750009536743
2023-02-07 10:40:51,961 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.37398099899292
2023-02-07 10:40:52,426 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.490663528442383
2023-02-07 10:40:54,933 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.2343533039093018
2023-02-07 10:40:54,948 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：4.411207437515259
2023-02-07 10:40:55,571 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.6103456020355225
2023-02-07 10:40:57,470 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：5.044482469558716
2023-02-07 10:40:58,600 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.6133410930633545
2023-02-07 10:40:58,992 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.044187068939209
2023-02-07 10:40:59,525 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.817793846130371
2023-02-07 10:41:01,119 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.605329990386963
2023-02-07 10:41:03,868 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：5.194113731384277
2023-02-07 10:41:03,874 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.881947755813599
2023-02-07 10:41:04,059 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：4.071114540100098
2023-02-07 10:41:05,686 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：4.533877849578857
2023-02-07 10:41:07,341 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.466731309890747
2023-02-07 10:41:07,991 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.664203643798828
2023-02-07 10:41:08,665 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.363331317901611
2023-02-07 10:41:09,689 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.8756377696990967
2023-02-07 10:41:11,546 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.4617433547973633
2023-02-07 10:41:12,358 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.6681923866271973
2023-02-07 10:41:12,772 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：5.3267576694488525
2023-02-07 10:41:13,251 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.5006394386291504
2023-02-07 10:41:14,900 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.2692601680755615
2023-02-07 10:41:16,319 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.9255049228668213
2023-02-07 10:41:17,312 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：4.5069499015808105
2023-02-07 10:41:17,711 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：4.34039568901062
2023-02-07 10:41:19,079 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：4.135910511016846
2023-02-07 10:41:20,611 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.1076910495758057
2023-02-07 10:41:20,647 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.262605667114258
2023-02-07 10:41:22,526 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.446765899658203
2023-02-07 10:41:22,903 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：5.124280691146851
2023-02-07 10:41:23,917 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.304145097732544
2023-02-07 10:41:25,259 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：4.6116485595703125
2023-02-07 10:41:25,883 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.269259452819824
2023-02-07 10:41:27,555 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：4.456085681915283
2023-02-07 10:41:28,862 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：2.914208173751831
2023-02-07 10:41:28,899 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.5864102840423584
2023-02-07 10:41:28,931 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：4.921841621398926
2023-02-07 10:41:31,947 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：4.0761024951934814
2023-02-07 10:41:32,521 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 44980: 线程【3】耗时：3.5205881595611572
2023-02-07 10:41:32,763 - G:/workspace/syncdata/sync_data_thread_pool.py[line:125] - INFO - 44980: 进程【3】工作完成
2023-02-07 10:41:32,870 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：3.8078203201293945
2023-02-07 10:41:33,063 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：3.969385862350464
2023-02-07 10:41:35,265 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 73104: 线程【1】耗时：3.093729257583618
2023-02-07 10:41:35,295 - G:/workspace/syncdata/sync_data_thread_pool.py[line:125] - INFO - 73104: 进程【1】工作完成
2023-02-07 10:41:35,347 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 46796: 线程【2】耗时：2.4464612007141113
2023-02-07 10:41:35,360 - G:/workspace/syncdata/sync_data_thread_pool.py[line:125] - INFO - 46796: 进程【2】工作完成
2023-02-07 10:41:35,548 - G:/workspace/syncdata/sync_data_thread_pool.py[line:123] - INFO - 18492: 线程【0】耗时：2.378640651702881
2023-02-07 10:41:35,548 - G:/workspace/syncdata/sync_data_thread_pool.py[line:125] - INFO - 18492: 进程【0】工作完成
2023-02-07 10:41:36,548 - G:/workspace/syncdata/sync_data_thread_pool.py[line:147] - INFO - 71236: 主进程结束

Process finished with exit code 0

二、多进程

对于计算密集任务，多线程因为GIL锁并不能提高计算效率，这时就需要用多进程，通过引入from multiprocessing import Pool, Manager，具体代码如下：

#!/usr/bin/python
# encoding:utf-8
import threading
import queue
import time
from multiprocessing import Pool, Manager
from log.log import get_logger


def counter(n=10200000):
    # 10200000累计求和耗时接近1秒
    t = 0
    for i in range(n):
        t += i
    return t


def run(idx, num):
    log.info('进程【{0}】开始工作'.format(idx))
    while num > 0:
        num -= 1
        # time.sleep(1)  # sleep不占用cpu
        start = time.time()
        counter()
        log.info('进程【{0}】耗时：{1}'.format(idx, time.time()-start))

    log.info('进程【{0}】工作完成'.format(idx))


def start_sync_process():
    log.info('主进程启动')
    max_worker_num = 4
    pool = Pool(processes=max_worker_num)
    num = 60
    for i in range(max_worker_num):
        pool.apply_async(run, (i, num / max_worker_num))
    pool.close()
    pool.join()
    log.info('主进程结束')


if __name__ == '__main__':
    start_sync_process()

由于多进程会使用cpu的多个核进行计算，单个任务的计算时间不会缩短，可以提升效率，执行结果如下：

"C:\Program Files\Python\Python37\python.exe" G:/workspace/syncdata/sync_data_thread_pool.py
2023-02-07 11:00:32,879 - G:/workspace/syncdata/sync_data_thread_pool.py[line:98] - INFO - 11000: 主进程启动
2023-02-07 11:00:33,387 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 69748: 线程【0】开始工作
2023-02-07 11:00:33,433 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 20136: 线程【1】开始工作
2023-02-07 11:00:33,435 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 49604: 线程【2】开始工作
2023-02-07 11:00:33,439 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 58180: 线程【3】开始工作
2023-02-07 11:00:34,430 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：0.9933438301086426
2023-02-07 11:00:34,464 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0761215686798096
2023-02-07 11:00:34,464 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0242605209350586
2023-02-07 11:00:34,489 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0551788806915283
2023-02-07 11:00:35,612 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.148928165435791
2023-02-07 11:00:35,612 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.14793062210083
2023-02-07 11:00:35,621 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.1918127536773682
2023-02-07 11:00:35,713 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.2247247695922852
2023-02-07 11:00:36,814 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.200788974761963
2023-02-07 11:00:36,822 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.2007901668548584
2023-02-07 11:00:36,869 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.2566404342651367
2023-02-07 11:00:36,935 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.221733570098877
2023-02-07 11:00:37,789 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：0.9743955135345459
2023-02-07 11:00:37,826 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0023205280303955
2023-02-07 11:00:37,856 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：0.9863626956939697
2023-02-07 11:00:37,963 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0282509326934814
2023-02-07 11:00:38,829 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0402181148529053
2023-02-07 11:00:38,906 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0791139602661133
2023-02-07 11:00:38,947 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0910828113555908
2023-02-07 11:00:39,066 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.103050708770752
2023-02-07 11:00:39,863 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0332379341125488
2023-02-07 11:00:39,970 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0222671031951904
2023-02-07 11:00:39,974 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0681443214416504
2023-02-07 11:00:40,051 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：0.9843685626983643
2023-02-07 11:00:40,888 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.025259017944336
2023-02-07 11:00:41,031 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0611631870269775
2023-02-07 11:00:41,044 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.070138692855835
2023-02-07 11:00:41,143 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0910825729370117
2023-02-07 11:00:41,929 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0392205715179443
2023-02-07 11:00:42,060 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0162830352783203
2023-02-07 11:00:42,060 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.029247760772705
2023-02-07 11:00:42,170 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0252583026885986
2023-02-07 11:00:42,942 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0122933387756348
2023-02-07 11:00:43,044 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：0.9833710193634033
2023-02-07 11:00:43,053 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：0.9913492202758789
2023-02-07 11:00:43,175 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0043141841888428
2023-02-07 11:00:43,960 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.017279863357544
2023-02-07 11:00:44,045 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0013227462768555
2023-02-07 11:00:44,070 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.013291597366333
2023-02-07 11:00:44,187 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0112967491149902
2023-02-07 11:00:44,968 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0083048343658447
2023-02-07 11:00:45,042 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：0.9963362216949463
2023-02-07 11:00:45,073 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0033166408538818
2023-02-07 11:00:45,193 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.005312442779541
2023-02-07 11:00:46,118 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.1499247550964355
2023-02-07 11:00:46,156 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.1130239963531494
2023-02-07 11:00:46,166 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0930776596069336
2023-02-07 11:00:46,310 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.1170129776000977
2023-02-07 11:00:47,154 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0352323055267334
2023-02-07 11:00:47,160 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0023202896118164
2023-02-07 11:00:47,186 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0192749500274658
2023-02-07 11:00:47,308 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：0.99833083152771
2023-02-07 11:00:48,168 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0132908821105957
2023-02-07 11:00:48,176 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：1.0152854919433594
2023-02-07 11:00:48,201 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：1.0152854919433594
2023-02-07 11:00:48,311 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.0023205280303955
2023-02-07 11:00:49,166 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 49604: 线程【2】耗时：0.9893548488616943
2023-02-07 11:00:49,166 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 49604: 线程【2】工作完成
2023-02-07 11:00:49,186 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 69748: 线程【0】耗时：0.9853663444519043
2023-02-07 11:00:49,186 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 69748: 线程【0】工作完成
2023-02-07 11:00:49,208 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 58180: 线程【3】耗时：1.0402188301086426
2023-02-07 11:00:49,208 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 58180: 线程【3】工作完成
2023-02-07 11:00:49,328 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 20136: 线程【1】耗时：1.017279863357544
2023-02-07 11:00:49,328 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 20136: 线程【1】工作完成
2023-02-07 11:00:49,435 - G:/workspace/syncdata/sync_data_thread_pool.py[line:106] - INFO - 11000: 主进程结束

Process finished with exit code 0

需要注意是，使用多进程时，设置的最大进程数Processes最好和CPU的核心数一致，如果大于CPU的核心数，同和和多线程一样，存在多个进程去竞争CPU核心计算时间片，导致单个进程计算任务的时间变长，并不能提高任务效率，比如CPU是4核，设置Processes为8，结果如下：

"C:\Program Files\Python\Python37\python.exe" G:/workspace/syncdata/sync_data_thread_pool.py
2023-02-07 11:04:45,948 - G:/workspace/syncdata/sync_data_thread_pool.py[line:98] - INFO - 73636: 主进程启动
2023-02-07 11:04:47,180 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 21664: 线程【0】开始工作
2023-02-07 11:04:47,187 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 66980: 线程【1】开始工作
2023-02-07 11:04:47,196 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 70684: 线程【2】开始工作
2023-02-07 11:04:47,238 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 65760: 线程【3】开始工作
2023-02-07 11:04:47,365 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 57832: 线程【4】开始工作
2023-02-07 11:04:47,672 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 2852: 线程【5】开始工作
2023-02-07 11:04:47,798 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 3204: 线程【6】开始工作
2023-02-07 11:04:48,238 - G:\workspace\syncdata\sync_data_thread_pool.py[line:118] - INFO - 72604: 线程【7】开始工作
2023-02-07 11:04:49,082 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：1.8939359188079834
2023-02-07 11:04:49,097 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：1.9168736934661865
2023-02-07 11:04:49,102 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：1.9049062728881836
2023-02-07 11:04:49,246 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.0086300373077393
2023-02-07 11:04:49,469 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.1023786067962646
2023-02-07 11:04:49,864 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.191141366958618
2023-02-07 11:04:50,038 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.2400102615356445
2023-02-07 11:04:50,449 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.2100906372070312
2023-02-07 11:04:51,195 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.0973925590515137
2023-02-07 11:04:51,231 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.1492536067962646
2023-02-07 11:04:51,255 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.153243064880371
2023-02-07 11:04:51,374 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.127311944961548
2023-02-07 11:04:51,537 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.0674710273742676
2023-02-07 11:04:51,947 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.0834295749664307
2023-02-07 11:04:52,113 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.073456287384033
2023-02-07 11:04:52,569 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.1203315258026123
2023-02-07 11:04:53,341 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.146261692047119
2023-02-07 11:04:53,373 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.1183364391326904
2023-02-07 11:04:53,380 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.1492538452148438
2023-02-07 11:04:53,587 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.213083505630493
2023-02-07 11:04:53,712 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.1741867065429688
2023-02-07 11:04:54,129 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.1821658611297607
2023-02-07 11:04:54,320 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.2070987224578857
2023-02-07 11:04:54,674 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.1043734550476074
2023-02-07 11:04:55,695 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.3537075519561768
2023-02-07 11:04:55,734 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.3606889247894287
2023-02-07 11:04:55,768 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.3876163959503174
2023-02-07 11:04:56,055 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.468400478363037
2023-02-07 11:04:56,218 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.4404752254486084
2023-02-07 11:04:56,622 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.4923365116119385
2023-02-07 11:04:56,918 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.5980539321899414
2023-02-07 11:04:57,396 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.7217230796813965
2023-02-07 11:04:58,356 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.6608853340148926
2023-02-07 11:04:58,423 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.688811779022217
2023-02-07 11:04:58,459 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.6908059120178223
2023-02-07 11:04:58,651 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.595061779022217
2023-02-07 11:04:58,780 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.562150478363037
2023-02-07 11:04:59,373 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.750645875930786
2023-02-07 11:04:59,577 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.658891201019287
2023-02-07 11:04:59,972 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.575115442276001
2023-02-07 11:05:00,968 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.5451958179473877
2023-02-07 11:05:01,041 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.684821844100952
2023-02-07 11:05:01,077 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.6180014610290527
2023-02-07 11:05:01,275 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.623983860015869
2023-02-07 11:05:01,327 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.546192169189453
2023-02-07 11:05:01,815 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：2.4414730072021484
2023-02-07 11:05:02,001 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.4245176315307617
2023-02-07 11:05:02,323 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：2.351712226867676
2023-02-07 11:05:03,075 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.1073639392852783
2023-02-07 11:05:03,120 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.078441619873047
2023-02-07 11:05:03,135 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.058495283126831
2023-02-07 11:05:03,278 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.00264573097229
2023-02-07 11:05:03,303 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：1.9747202396392822
2023-02-07 11:05:03,782 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：1.9667420387268066
2023-02-07 11:05:04,006 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：2.0036427974700928
2023-02-07 11:05:04,316 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：1.9926722049713135
2023-02-07 11:05:05,476 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 70684: 线程【2】耗时：2.1283087730407715
2023-02-07 11:05:05,476 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 70684: 线程【2】工作完成
2023-02-07 11:05:05,503 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 57832: 线程【4】耗时：2.1203298568725586
2023-02-07 11:05:05,504 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 57832: 线程【4】工作完成
2023-02-07 11:05:05,526 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 66980: 线程【1】耗时：2.1522457599639893
2023-02-07 11:05:05,526 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 66980: 线程【1】工作完成
2023-02-07 11:05:05,534 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 65760: 线程【3】耗时：2.155237913131714
2023-02-07 11:05:05,534 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 65760: 线程【3】工作完成
2023-02-07 11:05:05,546 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 21664: 线程【0】耗时：2.1362879276275635
2023-02-07 11:05:05,546 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 21664: 线程【0】工作完成
2023-02-07 11:05:05,719 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 2852: 线程【5】耗时：1.9378187656402588
2023-02-07 11:05:05,720 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 2852: 线程【5】工作完成
2023-02-07 11:05:05,828 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 3204: 线程【6】耗时：1.822160243988037
2023-02-07 11:05:05,828 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 3204: 线程【6】工作完成
2023-02-07 11:05:05,957 - G:\workspace\syncdata\sync_data_thread_pool.py[line:124] - INFO - 72604: 线程【7】耗时：1.64064621925354
2023-02-07 11:05:05,957 - G:\workspace\syncdata\sync_data_thread_pool.py[line:126] - INFO - 72604: 线程【7】工作完成
2023-02-07 11:05:06,143 - G:/workspace/syncdata/sync_data_thread_pool.py[line:106] - INFO - 73636: 主进程结束

Process finished with exit code 0

三、数据队列

#!/usr/bin/python
# encoding:utf-8
import threading
import queue
import time
from log.log import get_logger

log = get_logger()
exitFlag = False
total = 10  # 生产者生产产品总数，固定不变
product_total = 10  # 生产者生产产品总数，线程共享变量
producerThreadNum = 3  # 生产者线程数
consumerThreadNum = 3  # 消费者线程数
queueLock = threading.Lock()  # 互斥锁
workQueue = queue.Queue(10)  # 先入先出队列
nameList = ["One", "Two", "Three", "Four", "Five"]
# 填充队列
queueLock.acquire()
for word in nameList:
    workQueue.put(word)
queueLock.release()


class syncThread(threading.Thread):
    def __init__(self, thread_id, thread_name, thread_type, queue):
        threading.Thread.__init__(self)
        self.thread_id = thread_id
        self.thread_name = thread_name
        self.thread_type = thread_type
        self.queue = queue

    def run(self):
        log.debug('开启线程：' + self.thread_name)
        if self.thread_type == 1:
            # 生产者处理过程
            producer_process(self.thread_name, self.queue)
        else:
            # 消费者处理过程
            consumer_process(self.thread_name, self.queue)
        log.debug('线程处理完成：' + self.thread_name)


def producer_process(thread_name, queue):
    global product_total
    while True:
        queueLock.acquire()
        if product_total > 0:
            queue.put('product-' + str(total - product_total + 1))
            log.debug('生产者[{0}]生产一个产品：product-{1}'.format(thread_name, total - product_total + 1))
            product_total -= 1
            queueLock.release()
            time.sleep(10)
        else:
            queueLock.release()
            break


def consumer_process(thread_name, queue):
    while True:
        queueLock.acquire()
        if not workQueue.empty():
            data = queue.get()
            queueLock.release()
            log.debug("消费者[{0}]消费一个产品：product-{1}".format(thread_name, data))
        elif exitFlag:
            queueLock.release()  # 生产者生产完成
            break
        else:
            queueLock.release()  # 生产者还在生产，等待消费
            log.debug('消费者[{0}]等待消费'.format(thread_name))
        time.sleep(1)


def start_sync():
    # 创建新线程
    threads = []
    for i in range(1, producerThreadNum + 1):
        producer = syncThread(i, 'Producer-' + str(i), 1, workQueue)
        producer.start()
        threads.append(producer)
    for i in range(1, consumerThreadNum + 1):
        consumer = syncThread(i, 'Consumer-' + str(i), 0, workQueue)
        consumer.start()
        threads.append(consumer)
    # 等待队列清空
    while not workQueue.empty() or product_total > 0:
        pass
    global exitFlag  # 修改全局变量需要先用global申明
    exitFlag = True  # 线程退出循环
    # 等待所有线程完成
    for t in threads:
        t.join()
    log.info("退出主线程")


if __name__ == '__main__':
    start_sync()

江凌

关注

0
点赞
踩
1

收藏

觉得还不错? 一键收藏
0
评论
python 多线程、多进程和数据队列

多个线程竞争单核的计算时间片，导致单个线程的计算任务耗时变长，多线程最终总任务的计算时间并没有缩短（对于计算密集型任务，如果run方法里不是计算counter函数，而是用sleep，多线程是可以提升效率的）。pytohn多线程因为全局GIL锁的存在，不会使用多核运行，而是多个线程去竞争cpu一个核的锁，所以对于计算密集型任务，多线程并不能提高效率。
复制链接

扫一扫