python实现RabbitMQ同步跟异步消费模型

最新推荐文章于 2023-07-13 21:37:49 发布

weixin_30644369

最新推荐文章于 2023-07-13 21:37:49 发布

阅读量481

点赞数

文章标签： python 大数据 git

原文链接：http://www.cnblogs.com/zzqit/p/10165069.html

版权

1，消息推送类

 1 import pika
 2 
 3 
 4 # 同步消息推送类
 5 class RabbitPublisher(object):
 6 
 7     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
 8     def __init__(self, host, user, password):
 9         self.host = host
10         self.user = user
11         self.password = password
12         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
13         self.channel = self.connection.channel()
14 
15     # 发送消息在队列中
16     def send(self, queue_name, body):
17         self.channel.queue_declare(queue=queue_name, durable=True)  # 声明一个持久化队列
18         self.channel.basic_publish(exchange='',
19                                    routing_key=queue_name,  # 队列名字
20                                    body=body,  # 消息内容
21                                    properties=pika.BasicProperties(
22                                        delivery_mode=2,  # 消息持久化
23                                    ))
24 
25     # 清除指定队列的所有的消息
26     def purge(self, queue_name):
27         self.channel.queue_purge(queue_name)
28 
29     # 删除指定队列
30     def delete(self, queue_name, if_unused=False, if_empty=False):
31         self.channel.queue_delete(queue_name, if_unused=if_unused, if_empty=if_empty)
32 
33     # 断开连接
34     def stop(self):
35         self.connection.close()

View Code

2.消息消费类

（1）同步消息消费

在同步消息消费的时候可能会出现pika库断开的情况，原因是因为pika客户端没有及时发送心跳，连接就被server端断开了。解决方案就是做一个心跳线程来维护连接。

心跳线程类

 1 class Heartbeat(threading.Thread):
 2 
 3     def __init__(self, connection):
 4         super(Heartbeat, self).__init__()
 5         self.lock = threading.Lock()  # 线程锁
 6         self.connection = connection  # rabbit连接
 7         self.quitflag = False  # 退出标志
 8         self.stopflag = True  # 暂停标志
 9         self.setDaemon(True)  # 设置为守护线程，当消息处理完，自动清除
10 
11     # 间隔10s发送心跳
12     def run(self):
13         while not self.quitflag:
14             time.sleep(10)  # 睡10s发一次心跳
15             self.lock.acquire()  # 加线程锁
16             if self.stopflag:
17                 self.lock.release()
18                 continue
19             try:
20                 self.connection.process_data_events()  # 一直等待服务段发来的消息
21             except Exception as e:
22                 print "Error format: %s" % (str(e))
23                 self.lock.release()
24                 return
25             self.lock.release()
26 
27     # 开启心跳保护
28     def startheartbeat(self):
29         self.lock.acquire()
30         if self.quitflag:
31             self.lock.release()
32             return
33         self.stopflag = False
34         self.lock.release()

View Code

消息消费类

 1 # 同步消息消费类
 2 class RabbitConsumer(object):
 3 
 4     # 传入RabbitMQ的ip，用户名，密码，实例化一个管道
 5     def __init__(self, host, user, password):
 6         self.host = host
 7         self.user = user
 8         self.password = password
 9         self.connection = pika.BlockingConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)))
10         self.channel = self.connection.channel()
11 
12     # 进行消费
13     def receive(self, queue_name, callback_worker, prefetch_count=1):  # callback_worker为消费的回调函数
14         self.channel.queue_declare(queue=queue_name, durable=True)
15         self.channel.basic_qos(prefetch_count=prefetch_count)  # 设置预取的数量，如果为0则不预取，消费者处理越快，可以将这个这设置的越高
16         self.channel.basic_consume(callback_worker, queue=queue_name)  # callback_worker为消费的回调函数
17         heartbeat = Heartbeat(self.connection)  # 实例化一个心跳类
18         heartbeat.start()  # 开启一个心跳线程，不传target的值默认运行run函数
19         heartbeat.startheartbeat()  # 开启心跳保护
20         self.channel.start_consuming()  # 开始消费

View Code

调用方法

# 消费回调函数
def callback(ch, method, properties, body):
    print(" [x] Received %r" % body)
    ch.basic_ack(delivery_tag=method.delivery_tag)  # 告诉生产者处理完成

consumer = RabbitConsumer(host="12.12.12.12", user="test", password="123")
consumer.receive(queue_name="queue1", callback_worker=callback)

（2）异步消息消费（推荐）

pika提供了支持异步发送模式的selectconnection方法支持异步发送接收（通过回调的方式）

connectioon建立时回调建立channel， channel建立时一次回调各种declare方法，declare建立时依次回调publish。

同使用blockconnection方法相比，通过wireshark抓包来看，使用异步的方式会对发包进行一些优化，会将几个包合并成一个大包，然后做一次ack应答从而提高效率，与之相反使用blockconnection时将会做至少两次ack，head一次content一次等

因此再试用异步的方式时会获得一定的优化

异步消息消费类

  1 # 异步消息消费类
  2 class RabbitConsumerAsync(object):
  3     EXCHANGE = 'amq.direct'
  4     EXCHANGE_TYPE = 'direct'
  5 
  6     def __init__(self, host, user, password, queue_name="fish_test", callback_worker=None, prefetch_count=1):
  7         self.host = host
  8         self.user = user
  9         self.password = password
 10         self._connection = None
 11         self._channel = None
 12         self._closing = False
 13         self._consumer_tag = None
 14         self.QUEUE = queue_name
 15         self.callbackworker = callback_worker
 16         self.prefetch_count = prefetch_count
 17 
 18     def connect(self):
 19         return pika.SelectConnection(pika.ConnectionParameters(host=self.host, credentials=pika.PlainCredentials(self.user, self.password)), self.on_connection_open,
 20                                      stop_ioloop_on_close=False)
 21 
 22     def on_connection_open(self, unused_connection):
 23         self.add_on_connection_close_callback()
 24         self.open_channel()
 25 
 26     def add_on_connection_close_callback(self):
 27         self._connection.add_on_close_callback(self.on_connection_closed)
 28 
 29     def on_connection_closed(self, connection, reply_code, reply_text):
 30         self._channel = None
 31         if self._closing:
 32             self._connection.ioloop.stop()
 33         else:
 34             self._connection.add_timeout(5, self.reconnect)
 35 
 36     def reconnect(self):
 37         self._connection.ioloop.stop()
 38         if not self._closing:
 39             self._connection = self.connect()
 40             self._connection.ioloop.start()
 41 
 42     def open_channel(self):
 43         self._connection.channel(on_open_callback=self.on_channel_open)
 44 
 45     def on_channel_open(self, channel):
 46         self._channel = channel
 47         self._channel.basic_qos(prefetch_count=self.prefetch_count)
 48         self.add_on_channel_close_callback()
 49         self.setup_exchange(self.EXCHANGE)
 50 
 51     def add_on_channel_close_callback(self):
 52         self._channel.add_on_close_callback(self.on_channel_closed)
 53 
 54     def on_channel_closed(self, channel, reply_code, reply_text):
 55         print reply_text
 56         self._connection.close()
 57 
 58     def setup_exchange(self, exchange_name):
 59         self._channel.exchange_declare(self.on_exchange_declareok, exchange_name, self.EXCHANGE_TYPE, durable=True)
 60 
 61     def on_exchange_declareok(self, unused_frame):
 62         self.setup_queue()
 63 
 64     def setup_queue(self):
 65         self._channel.queue_declare(self.on_queue_declareok, self.QUEUE, durable=True)
 66 
 67     def on_queue_declareok(self, method_frame):
 68         self._channel.queue_bind(self.on_bindok, self.QUEUE, self.EXCHANGE, self.QUEUE)
 69 
 70     def on_bindok(self, unused_frame):
 71         self.start_consuming()
 72 
 73     def start_consuming(self):
 74         self.add_on_cancel_callback()
 75         self._consumer_tag = self._channel.basic_consume(self.on_message, self.QUEUE)
 76 
 77     def add_on_cancel_callback(self):
 78         self._channel.add_on_cancel_callback(self.on_consumer_cancelled)
 79 
 80     def on_consumer_cancelled(self, method_frame):
 81         if self._channel:
 82             self._channel.close()
 83 
 84     def on_message(self, unused_channel, basic_deliver, properties, body):
 85         self.callbackworker(body)
 86         self.acknowledge_message(basic_deliver.delivery_tag)
 87 
 88     def acknowledge_message(self, delivery_tag):
 89         self._channel.basic_ack(delivery_tag)
 90 
 91     def stop_consuming(self):
 92         if self._channel:
 93             self._channel.basic_cancel(self.on_cancelok, self._consumer_tag)
 94 
 95     def on_cancelok(self, unused_frame):
 96         self.close_channel()
 97 
 98     def close_channel(self):
 99         self._channel.close()
100 
101     def run(self):
102         self._connection = self.connect()
103         self._connection.ioloop.start()
104 
105     def stop(self):
106         self._closing = True
107         self.stop_consuming()
108         self._connection.ioloop.start()
109 
110     def close_connection(self):
111         self._connection.close()

View Code

调用方法

# 消费回调函数
def callback(body):
    print(" [x] Received %r" % body)

consumer = RabbitConsumerAsync(host="12.12.12.12", user="test", password="123", queue_name="fish_test", callback_worker=callback, prefetch_count=2)
consumer.run()