python：kafka的消费者（待完善）

本篇是kafka消费者，目前功能还不太完善，还需深入学习

import concurrent.futures
import time
import asyncio
from kafka import KafkaConsumer, TopicPartition
#


class Consumer():
    def __init__(self,*args,group_id=0):
        self.consumer = KafkaConsumer(bootstrap_servers=args,fetch_min_bytes=5,fetch_max_bytes=5,fetch_max_wait_ms=5000,group_id=str(group_id))  # 参数为接收主题和kafka服务器地址
        self.topic=None


    #设置监听消息
    def listener(self,*args,**kwargs):
        print("监听的消息：{},{}".format(args,kwargs))

    #设置订阅的topic
    def set_topic(self,*args):
        self.topic=args
        self.consumer.subscribe(topics=args,listener=self.listener(args),)  # 订阅要消费的主题


    #获取topic的偏移量
    def position(self,topic=None,partition=0):
        try:
            if topic is None and self.topic is None:
                raise Exception("topic is None!")
            elif topic !=None:
                offset=self.consumer.position(TopicPartition(topic=u'{}'.format(topic), partition=partition))  # 获取当前主题的最新偏移量
                return {topic:offset}
            else:
                offset_dict={}
                for i in self.topic:
                    offset=self.consumer.position(TopicPartition(topic=u'{}'.format(i), partition=partition))  # 获取当前主题的最新偏移量
                    offset_dict.setdefault(i,offset)
                return offset_dict
        except Exception as error:
            return "{}".format(error)

    #接受数据
    def recv(self):
        num = 0
        #获取用户有权查看的所有主题
        self.consumer.topics()
        # self.consumer.pause(TopicPartition(topic=u'test', partition=0)) 这个挂起的操作
        while True:
            print("Currently suspended consumers：{}".format(self.consumer.paused()))  # 获取当前挂起的消费者
            msg = self.consumer.poll(timeout_ms=5,max_records=10)
            print("当前获取的消息:{}".format(msg))
            time.sleep(1)
            num = num + 1
            # if num == 2:
            #     self.consumer.resume(TopicPartition(topic=u'test', partition=0)) #这个停止挂起
            #     print("resume......")
    async def main(self):
        loop=asyncio.get_event_loop()
        with concurrent.futures.ThreadPoolExecutor() as pool:
            await loop.run_in_executor(pool,self.recv)



#运行代码段
consumer=Consumer('127.0.0.1:9092')
consumer.set_topic("test") #订阅topic
consumer.position("test") #获取topic的偏移量
# consumer.recv()
asyncio.run(consumer.main())#进行消费，功能待完善




#调试代码段
# asyncio.run(consumer.main())

# consumer = KafkaConsumer('test', bootstrap_servers=['127.0.0.1:9092'])  # 参数为接收主题和kafka服务器地址
# #
# # 这是一个永久堵塞的过程，生产者消息会缓存在消息队列中,并且不删除,所以每个消息在消息队列中都有偏移
# for message in consumer:  # consumer是一个消息队列，当后台有消息时，这个消息队列就会自动增加．所以遍历也总是会有数据，当消息队列中没有数据时，就会堵塞等待消息带来
#     print("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition, message.offset, message.key, message.value))
#     consumer.poll()

# consumer = KafkaConsumer(bootstrap_servers=['127.0.0.1:9092'])
# consumer.subscribe(topics=('test'))
# consumer.topics()
# consumer.pause(TopicPartition(topic=u'test', partition=0))
# num = 0
# while True:
#     print(consumer.paused())   #获取当前挂起的消费者
#     msg = consumer.poll(timeout_ms=5)
#     print(msg)
#     time.sleep(2)
#     num = num + 1
#     if num == 10:
#         consumer.resume(TopicPartition(topic=u'test', partition=0))
#         print("resume......")