python实现线程池并可自动拓展和减小线程数

实现代码：

#!/tool/python/3.6.12/bin/python3

"""
@File: threading_control.py
@Author: 
@Date: 20211122
@Contact:
@Desc: define common threading function
"""


from queue import Queue, Empty
import threading
import time
import datetime
import base


log = base.Log()


class ThreadControl:

    def __init__(self, thread_num=-1, night_thread_num=None, night_mode_hour=19, day_mode_hour=8, delay=0,
                 specify_begin_hour=None):
        """
        set thread manager information
        :param thread_num: create the specified num threading tool, if set to -1, no limit mode.
        :param night_thread_num: change the thread num to the specified num when night.
        :param night_mode_hour: the time to change the day thread num to night thread num
        :param day_mode_hour: the time to change the night thread num to thread num
        :param delay: unit is second, if set, wait the delay time to begin to execute.
        :param specify_begin_hour: type must be int (0-24), delay the specified hours.
        """
        self.thread_num = thread_num
        log.logger.info("You set thread num: %s" % self.thread_num)
        self.night_thread_num = night_thread_num
        if self.night_thread_num is not None:
            self.night_mode_hour = night_mode_hour
            self.day_mode_hour = day_mode_hour
            if not isinstance(self.night_thread_num, int) or not isinstance(self.night_mode_hour, int) or not isinstance(self.day_mode_hour, int):
                log.logger.error("Night thread num, night mode hour, day mode hour must be int type!")
                sys.exit()
            else:
                log.logger.info("You set night thread num: %s" % self.night_thread_num)
                log.logger.info("You set day mode time: %s" % self.day_mode_hour)
                log.logger.info("You set night mode time: %s" % self.night_mode_hour)
        self.delay = delay
        self.specify_begin_hour = specify_begin_hour
        if self.specify_begin_hour is not None:
            self.delay = self.get_interval_seconds(self.specify_begin_hour)
        log.logger.info("You set delay: %s" % self.delay)

    @staticmethod
    def get_interval_seconds(hour):
        now = datetime.datetime.now()
        if now.hour < hour:
            today_begin = datetime.datetime(now.year, now.month, now.day, hour, 0, 0)
            interval_seconds = (today_begin - now).seconds
        else:
            interval_seconds = 0
        return interval_seconds


class ThreadPoolManager():

    def __init__(self, thread_control=None, cmd_handler=ShellProcess):
        if thread_control is None:
            self.thread_control = ThreadControl()
        else:
            self.thread_control = thread_control
        self.cmd_handler = cmd_handler
        self.work_queue = Queue()
        self.thread_pool = []
        self.no_limit_mode = False
        self.delay = self.thread_control.delay
        self.night_thread_num = self.thread_control.night_thread_num
        self.day_thread_num = self.thread_control.thread_num
        if self.day_thread_num == -1:
            self.no_limit_mode = True
            return
        self.start_threading = threading.Thread(target=self.__delay_before_execute)
        self.start_threading.setDaemon(True)
        self.start_threading.start()

    def __delay_before_execute(self):
        time.sleep(self.delay)
        self.thread_num = self.get_init_thread_num()
        self.__init_thread_pool(self.thread_num)
        if self.night_thread_num is not None:
            self.__auto_change_thread_num()

    def get_init_thread_num(self):
        if self.night_thread_num is None:
            thread_num = self.thread_control.thread_num
        else:
            now = datetime.datetime.now()
            if now.hour >= self.thread_control.night_mode_hour:
                thread_num = self.thread_control.night_thread_num
            else:
                thread_num = self.thread_control.thread_num
        return thread_num

    def __init_thread_pool(self, thread_num):
        """ initial threading pool """
        for i in range(thread_num):
            log.logger.debug("Start thread %s" % i)
            thread = ThreadFactory(self.work_queue, self.cmd_handler)
            self.thread_pool.append(thread)

    def add_job(self, func, *args, **kwargs):
        """
        put duties to queue, wait for threading pool execute
        func can be str("pwd", "pwd&&ls") or executable function
        """
        if self.no_limit_mode:
            job_thread = RunJobThread(self.cmd_handler, self.delay, func, *args, **kwargs)
            job_thread.setDaemon(True)
            job_thread.start()
            self.thread_pool.append(job_thread)
        else:
            self.work_queue.put((func, args, kwargs))
        time.sleep(1)       # add delay for bsub get best queue

    def wait_completion(self):
        """ wait for completion of all the jobs in the queue """
        if self.no_limit_mode:
            for thread in self.thread_pool:
                thread.join()
        else:
            self.work_queue.join()

    def _close_all_threads(self):
        """ Signal all threads to exit and lose the references to them """
        if not self.no_limit_mode:
            for thread in self.thread_pool:
                thread.signal_exit()
            self.thread_pool = []

    def __extend_thread_pool(self, num):
        self.__init_thread_pool(num)
        self.thread_num = self.thread_num + num
        log.logger.info("Threading num change to %s" % self.thread_num)

    def __reduce_thread_pool(self, num):
        if num < len(self.thread_pool):
            for i in range(num):
                self.thread_pool[-1].signal_exit()
                del self.thread_pool[-1]
            self.thread_num = self.thread_num - num
            log.logger.info("Threading num change to %s" % self.thread_num)
        else:
            log.logger.error("Reduce threading number > current threading number")

    def adjust_thread_pool(self, num):
        if num > len(self.thread_pool):
            self.__extend_thread_pool(num - len(self.thread_pool))
        elif num < len(self.thread_pool):
            self.__reduce_thread_pool(len(self.thread_pool) - num)
        else:
            pass

    def __auto_change_thread_num(self):
        while True:
            time.sleep(3600)
            now = datetime.datetime.now()
            if now.hour == self.thread_control.night_mode_hour:
                self.adjust_thread_pool(self.thread_control.night_thread_num)
            elif now.hour == self.thread_control.day_mode_hour:
                self.adjust_thread_pool(self.thread_control.thread_num)
            else:
                pass

    def __del__(self):
        self._close_all_threads()


class ThreadFactory(threading.Thread):
    """ Thread executing tasks from a work queue. Thread can exit by signal """
    _TIMEOUT = 2

    def __init__(self, work_queue, cmd_handler):
        threading.Thread.__init__(self)
        self.work_queue = work_queue
        self.cmd_handler = cmd_handler
        self.daemon = True
        self.done = threading.Event()
        self.start()

    def run(self):
        while not self.done.is_set():
            try:
                target, args, kwargs = self.work_queue.get(block=True,
                                                   timeout=self._TIMEOUT)
                try:
                    self.__run_target(target, *args, **kwargs)
                except Exception as e:
                    log.logger.error(e)
                finally:
                    self.work_queue.task_done()
            except Empty as e:
                pass
        return

    def __run_target(self, target=None, *args, **kwargs):
        if isinstance(target, str):
            p = self.cmd_handler(target, *args, **kwargs)
            p.run()
            p.wait_completion()
        elif isfunction(target) or ismethod(target):
            try:
                target(*args, **kwargs)
            except Exception:
                log.logger.error(traceback.format_exc())
        else:
            log.logger.error("Your target is not str or function, please check.")

    def signal_exit(self):
        """ Signal to thread to exit """
        self.done.set()


class RunJobThread(threading.Thread):
    def __init__(self, cmd_handler, delay, target, *args, **kwargs):
        super(RunJobThread, self).__init__()
        self.cmd_handler = cmd_handler
        self.delay = delay
        self.target = target
        self.args = args
        self.kwargs = kwargs

    def run(self):
        time.sleep(self.delay)
        try:
            if isinstance(self.target, str):
                p = self.cmd_handler(self.target, *self.args, **self.kwargs)
                p.run()
                p.wait_completion()
            elif isfunction(self.target) or ismethod(self.target):
                try:
                    self.target(*self.args, **self.kwargs)
                except Exception:
                    log.logger.error(traceback.format_exc())
            else:
                log.logger.error("Your target is not str or function, please check.")
        except Exception as e:
            log.logger.error(e)


class Watcher:

    def __init__(self):
        self.child = os.fork()
        if self.child == 0:
            return
        else:
            self.watch()

    def watch(self):
        try:
            os.wait()
        except KeyboardInterrupt:
            print('\r' + ' ' * 128 + '\n\rKeyBoardInterrupt')
            self.kill()
        sys.exit()

    def kill(self):
        try:
            os.kill(self.child, signal.SIGKILL)
        except OSError:
            pass

base 代码：

#!/tool/python/3.6.12/bin/python3

"""
@File: base.py
@Author: 
@Date: 20211116
@Contact: 
@Desc: define some base function
"""


import logging


def read_file_to_lines(file_name):
    """
    read the specified file and return a list of lines
    :param file_name:
    """
    fp = open(file_name, "r")
    lines = fp.readlines()
    fp.close()
    return lines


class Log:

    def __init__(self):
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        self.formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
        ch = logging.StreamHandler()
        ch.setLevel(logging.DEBUG)
        ch.setFormatter(self.formatter)
        self.logger.addHandler(ch)

    def set_log_name(self, log_name):
        fh = logging.FileHandler(log_name, mode='w')
        fh.setLevel(logging.DEBUG)
        fh.setFormatter(self.formatter)
        self.logger.addHandler(fh)

测试代码：

def run(n):
    # new_lock.lock.acquire()
    task = 'task ' + str(n)
    print(task)
    time.sleep(10)
    print('%s 2s' % task)
    time.sleep(1)
    print('%s 1s' % task)
    time.sleep(1)
    print('%s 0s' % task)
    time.sleep(1)
    print('%s exit' % task)


# 创建一个有4个线程的线程池
thread_control=ThreadControl(4, 10)
thread_pool = ThreadPoolManger(thread_control)

print("threading count: ", threading.activeCount())
for i in range(25):
    thread_pool.add_job(run, i)
time.sleep(2)
print("threading count: ", threading.activeCount())
# thread_pool.change_to_specified_thread_num(15)
print("threading count: ", threading.activeCount())
thread_pool.wait_completion()