RabbitMQ多线程消费者（含线程安全）

前言

这几年一直在it行业里摸爬滚打，一路走来，不少总结了一些python行业里的高频面试，看到大部分初入行的新鲜血液，还在为各样的面试题答案或收录有各种困难问题

于是乎，我自己开发了一款面试宝典，希望能帮到大家，也希望有更多的Python新人真正加入从事到这个行业里，让python火不只是停留在广告上。

微信小程序搜索：Python面试宝典

或可关注原创个人博客：https://lienze.tech

也可关注微信公众号，不定时发送各类有趣猎奇的技术文章：Python编程学习

线程消费者

线程安全

可能某些时候，在手动开启确认消费者的情况下，并且还处于多线程模型中，如何线程安全的对ack确认进行处理

pika提供了add_callback_threadsafe方法来异步通知RabbitMQ消息处理成功

该方法是pika库中唯一线程安全的, 可被多个消费者(线程)同时调用

conn.add_callback_threadsafe(partial(channel.basic_ack, delivery_tag=...))

一万的发布，双线程单任务0.2s耗时，不需要额外加锁

消费者

import functools
import time
from multiprocessing.pool import ThreadPool
from multiprocessing import Process
from multiprocessing import current_process
from threading import Thread, current_thread

import pika


class CPUTest:
    def __init__(self):
        self.total = 0
        self.user = "guest"
        self.pwd = "guest"
        self.ip = "192.168.17.3"
        self.port = 5671  # 代理端口
        self.tp = None
        self.rt = []

    def __call__(self):
        p1 = Process(target=self.consumer)
        # p2 = Process(target=self.consumer)
        p1.start()
        print('p1 start')
        # p2.start()
        # print('p2 start')
        p1.join()
        # p2.join()

    def work(self, total, ch, method, connection):
        print('[%s][%s] start... [%s]' % (current_process().name, current_thread().name, total))
        print('tatol: ', total)
        print('[%s][%s] over... [消息数:%s]' % (current_process().name, current_thread().name, total))
        print('---------')
        cb = functools.partial(ch.basic_ack, delivery_tag=method.delivery_tag)
        connection.add_callback_threadsafe(cb)
        return 1

    def callback(self, ch, method, properties, body, args):
        connection = args[0]
        self.total += 1
        rt = self.tp.apply_async(func=self.work, args=(self.total, ch, method, connection))
        self.rt.append(rt)
        # ch.basic_ack(delivery_tag=method.delivery_tag)  # 非线程安全确认

    def consumer(self):
        self.tp = ThreadPool(16)
        credentials = pika.PlainCredentials(self.user, self.pwd)
        connection = pika.BlockingConnection(
            pika.ConnectionParameters(self.ip, self.port, '/', credentials))
        channel = connection.channel()
        channel.basic_qos(prefetch_count=16)
        # channel.queue_declare(queue='queue', durable=True)
        # 便函数绑定默认的参数 是连接对象 在任务完成时进行消费确认
        callback = functools.partial(self.callback, args=(connection,))
        channel.basic_consume(
            queue='order',  # 队列balance
            on_message_callback=callback,
        )
        channel.start_consuming()  # 长时间测试 阻塞
        self.tp.close()
        self.tp.join()
        channel.cancel()  # 清除消费者
        channel.close()
        connection.close()


def main():
    c = CPUTest()
    c()


if __name__ == '__main__':
    print('start ...')
    # import sys sys.stdout.flush()
    main()

生产者

import time
import pika
from pika.exceptions import UnroutableError, NackError
from multiprocessing.pool import Pool


class CPUTest:
    def __init__(self):
        self.user = "guest"
        self.pwd = "guest"
        self.ip = "192.168.17.3"
        self.port = 5671

    def __call__(self):
        tp = Pool(2)
        rt = []
        for var in range(2):
            rt.append(tp.apply_async(func=self.producer))
        for job in rt:
            job.get()
        tp.close()
        tp.join()

    def producer(self):
        credentials = pika.PlainCredentials(self.user, self.pwd)
        connection = pika.BlockingConnection(
            pika.ConnectionParameters(self.ip, self.port, '/', credentials,))
        channel = connection.channel()
        channel.queue_declare(queue='queue', durable=True)
        # channel.confirm_delivery()
        for num in range(10000):
            # if num % 1000 == 0:
            #     time.sleep(0.2)
            try:
                channel.basic_publish(
                    exchange='',
                    routing_key='order',
                    body="【{}】".format(num),
                    properties=pika.BasicProperties(delivery_mode=2),
                    mandatory=True,
                )
            except UnroutableError:
                print('[UnroutableError] 消息不可达')
            except NackError:
                print("[NackError] 消息不可达")
        channel.close()
        connection.close()  # 连接关闭


def main():
    time.sleep(2)
    c = CPUTest()
    c()
    print('...... over')


if __name__ == '__main__':
    main()