python发送zookeeper的kafka集群消息

python发kafka消息

1. 目前技术一般采用基于zk的kafka集群，传统直连方式不可取。因此采用pykafka三方类库来进行消息的发送和接收，本文实现了同步sync和异步async的消息发送produce，接收消息consume部分topic查询不到消息，欢迎技术同学帮我补充（本文也是参考网上文章，基于python3.7）
2. 直接上代码（实践可用）

rom pykafka import KafkaClient
from pykafka.simpleconsumer import SimpleConsumer
import time
# import logging
# logging.basicConfig(level=logging.DEBUG)


# 创建类
class kafkaConfig:

    # 类中函数 连接
    def __init__(self):
        host_brokers = '172.13.34.168:9081,172.13.34.169:9081,172.13.34.170:9081'
        # 如果有chroot path 就放到端口号后面/kfk1
        zookeeper_hosts = '172.13.42.165:2181,172.16.42.170:2181,172.16.42.171:2181/kfk1'
        try:
            self.client = KafkaClient(hosts=host_brokers,zookeeper_hosts=zookeeper_hosts)
            print('[conn success!]')
        except Exception as e:
            print('[exception conn]:%s,[msg]:%s'%(e,self.client))

    # obtains all topics
    def topic_lists(self):
        try:
            topic_lists = self.client.topics;
            print('[topic lists]:%s'%topic_lists)
        except Exception as e:
            print('[exception topic lists]:%s'%(e,topic_lists))
        return topic_lists

    # produce 同步消息send 单条消息 需要等待
    def msg_sync_send(self,topic,content):
        try:
            topic = self.client.topics[topic]
            with topic.get_sync_producer() as produce:
                produce.produce(bytes(content,encoding='utf-8'), timestamp=round(time.time()*1000))
            print('[send sync success!]')
            return "send sync success!"
        except Exception as e:
            print('[exception sync send]:%s'%e)

    # produce 异步消息send 单条消息  但生产环境 高吞吐量
    def msg_async_send(self,topic,content):
        try:
            topic = self.client.topics[topic]
            with topic.get_producer() as produce:
                produce.produce(bytes(content,encoding='utf-8'),timestamp=round(time.time()*1000))
            print('[send async success!]')
        except Exception as e:
            print('[exception async send]:%s'%e)

    # consumer 消息receive  simple适用于需要消费指定分区且不需要自动的重分配(自定义)
    # 在consumer_timeout_ms内没有任何信息返回，则中断接受消息
    def msg_simple_receive(self,topic,param):
        try:
            topic = self.client.topics[topic.encode()]
            partitions = topic.partitions
            last_offset = topic.latest_available_offsets()
            print("info msg:%s,%s"%(partitions,last_offset))  # 查看所有分区
            message = topic.get_simple_consumer(consumer_timeout_ms=1000,consumer_group=None)
            print('[message type]:%s,%s'%(type(message),SimpleConsumer.consume(message)))
            msg_list=[]
            for msg in message:
                # if param in msg.value:
                print('[msg,offset,content]:%s,%s,%s'%(type(msg), msg.offset, msg.value))
                msg_list.append(msg)
            print('[receive simple success!]%s,%s'%(message,type(message)))
            return msg_list
        except Exception as e:
            print('[exception simple receive]:%s'%e)
            return {"code": 300, "msg": "[exception]:%s"%e}

3. 关于集群host_brokers以及zookeeper_hosts地址 需要更换对应的地址，上述仅用例参考。
4. 上述msg_async_send以及msg_sync_send可以完成正常的消息生产发送，但是msg_simple_receive方法查询获取消息还有点问题，具体定位在get_simple_consumer方法，目前测试消息体可以获取到消息列表，但是业务流程中一些消息topic获取不到，会有报错INFO:pykafka.simpleconsumer:Continuing in response to UnknownError报错。希望有技术同学能帮助我。
5. 此外日志打印方式：

import logging
logging.basicConfig(level=logging.DEBUG)