python threading库_python中threading和queue库实现多线程编程

摘要

本文主要介绍了利用python的 threading和queue库实现多线程编程，并封装为一个类，方便读者嵌入自己的业务逻辑。最后以机器学习的一个超参数选择为例进行演示。

多线程实现逻辑封装

实例化该类后，在.object_func函数中加入自己的业务逻辑，再调用.run方法即可。

# -*- coding: utf-8 -*-

# @Time : 2021/2/4 14:36

# @Author : CyrusMay WJ

# @FileName: run.py

# @Software: PyCharm

# @Blog ：https://blog.csdn.net/Cyrus_May

import queue

import threading

class CyrusThread(object):

def __init__(self,num_thread = 10,logger=None):

"""

:param num_thread: 线程数

:param logger: 日志对象

"""

self.num_thread = num_thread

self.logger = logger

def object_func(self,args_queue,max_q):

while 1:

try:

arg = args_queue.get_nowait()

step = args_queue.qsize()

self.logger.info("progress:{}\{}".format(max_q,step))

except:

self.logger.info("no more arg for args_queue!")

break

"""

此处加入自己的业务逻辑代码

"""

def run(self,args):

args_queue = queue.Queue()

for value in args:

args_queue.put(value)

threads = []

for i in range(self.num_thread):

threads.append(threading.Thread(target=self.object_func,args = args_queue))

for t in threads:

t.start()

for t in threads:

t.join()

模型参数选择实例

# -*- coding: utf-8 -*-

# @Time : 2021/2/4 14:36

# @Author : CyrusMay WJ

# @FileName: run.py

# @Software: PyCharm

# @Blog ：https://blog.csdn.net/Cyrus_May

import queue

import threading

import numpy as np

from sklearn.datasets import load_boston

from sklearn.svm import SVR

import logging

import sys

class CyrusThread(object):

def __init__(self,num_thread = 10,logger=None):

"""

:param num_thread: 线程数

:param logger: 日志对象

"""

self.num_thread = num_thread

self.logger = logger

def object_func(self,args_queue,max_q):

while 1:

try:

arg = args_queue.get_nowait()

step = args_queue.qsize()

self.logger.info("progress:{}\{}".format(max_q,max_q-step))

except:

self.logger.info("no more arg for args_queue!")

break

# 业务代码

C, epsilon, gamma = arg[0], arg[1], arg[2]

svr_model = SVR(C=C, epsilon=epsilon, gamma=gamma)

x, y = load_boston()["data"], load_boston()["target"]

svr_model.fit(x, y)

self.logger.info("score:{}".format(svr_model.score(x,y)))

def run(self,args):

args_queue = queue.Queue()

max_q = 0

for value in args:

args_queue.put(value)

max_q += 1

threads = []

for i in range(self.num_thread):

threads.append(threading.Thread(target=self.object_func,args = (args_queue,max_q)))

for t in threads:

t.start()

for t in threads:

t.join()

# 创建日志对象

logger = logging.getLogger()

logger.setLevel(logging.INFO)

screen_handler = logging.StreamHandler(sys.stdout)

screen_handler.setLevel(logging.INFO)

formatter = logging.Formatter('%(asctime)s - %(module)s.%(funcName)s:%(lineno)d - %(levelname)s - %(message)s')

screen_handler.setFormatter(formatter)

logger.addHandler(screen_handler)

# 创建需要调整参数的集合

args = []

for C in [i for i in np.arange(0.01,1,0.01)]:

for epsilon in [i for i in np.arange(0.001,1,0.01)] + [i for i in range(1,10,1)]:

for gamma in [i for i in np.arange(0.001,1,0.01)] + [i for i in range(1,10,1)]:

args.append([C,epsilon,gamma])

# 创建多线程工具

threading_tool = CyrusThread(num_thread=20,logger=logger)

threading_tool.run(args)

运行结果

2021-02-04 20:52:22,824 - run.object_func:31 - INFO - progress:1176219\1

2021-02-04 20:52:22,824 - run.object_func:31 - INFO - progress:1176219\2

2021-02-04 20:52:22,826 - run.object_func:31 - INFO - progress:1176219\3

2021-02-04 20:52:22,833 - run.object_func:31 - INFO - progress:1176219\4

2021-02-04 20:52:22,837 - run.object_func:31 - INFO - progress:1176219\5

2021-02-04 20:52:22,838 - run.object_func:31 - INFO - progress:1176219\6

2021-02-04 20:52:22,841 - run.object_func:31 - INFO - progress:1176219\7

2021-02-04 20:52:22,862 - run.object_func:31 - INFO - progress:1176219\8

2021-02-04 20:52:22,873 - run.object_func:31 - INFO - progress:1176219\9

2021-02-04 20:52:22,884 - run.object_func:31 - INFO - progress:1176219\10

2021-02-04 20:52:22,885 - run.object_func:31 - INFO - progress:1176219\11

2021-02-04 20:52:22,897 - run.object_func:31 - INFO - progress:1176219\12

2021-02-04 20:52:22,900 - run.object_func:31 - INFO - progress:1176219\13

2021-02-04 20:52:22,904 - run.object_func:31 - INFO - progress:1176219\14

2021-02-04 20:52:22,912 - run.object_func:31 - INFO - progress:1176219\15

2021-02-04 20:52:22,920 - run.object_func:31 - INFO - progress:1176219\16

2021-02-04 20:52:22,920 - run.object_func:39 - INFO - score:-0.01674283914287855

2021-02-04 20:52:22,929 - run.object_func:31 - INFO - progress:1176219\17

2021-02-04 20:52:22,932 - run.object_func:39 - INFO - score:-0.007992354170952565

2021-02-04 20:52:22,932 - run.object_func:31 - INFO - progress:1176219\18

2021-02-04 20:52:22,945 - run.object_func:31 - INFO - progress:1176219\19

2021-02-04 20:52:22,954 - run.object_func:31 - INFO - progress:1176219\20

2021-02-04 20:52:22,978 - run.object_func:31 - INFO - progress:1176219\21

2021-02-04 20:52:22,984 - run.object_func:39 - INFO - score:-0.018769934807246536

2021-02-04 20:52:22,985 - run.object_func:31 - INFO - progress:1176219\22

到此这篇关于python中threading和queue库实现多线程编程的文章就介绍到这了,更多相关python 多线程编程内容请搜索聚米学院以前的文章或继续浏览下面的相关文章希望大家以后多多支持聚米学院！