python消费kafka性能_python之kafka消费

1 #-*- coding: utf-8 -*-

2

3 importuuid4 importjson5 from kafka importKafkaConsumer6 from xxxxxx importMessageToDict7 from xxx importObjectInfo8

9 importsys10 importcodecs11

12 sys.stdout = codecs.getwriter("utf-8")(sys.stdout.detach())13

14

15 classReadKafkaContent(object):16 @staticmethod17 defdeserialize(msg):18 """

19 反序列化20 :param msg:21 :return:22 """

23 pb_obj =ObjectInfo()24 pb_obj.Clear()25 pb_obj.ParseFromString(msg.value)26 return MessageToDict(pb_obj, including_default_value_fields=True, preserving_proto_field_name=True)27

28 defconsume_msg(self, consumer_obj):29 """

30 逐条消费，返回反序列化后的内容31 :param consumer_obj:32 :return:33 """

34 try:35 whileTrue:36 msg =next(consumer_obj, None)37 if notmsg:38 continue

39 content =self.deserialize(msg)40 returncontent41 exceptException as ex:42 print(u"消费kafka错误,退出测试")43 returnNone44

45 def entry(self, topic, ip, count=10, log="log_read_kafka_content.json"):46 """

47

48 :param topic:topic49 :param ip:ip50 :param count:查询kafka数据数量,默认10条51 :param log:内容保存地址,默认52 :return:53 """

54 print(u"开始......")55 try:56 #创建kafka消费对象

57 print(u"创建kafka消费对象...")58 consumer = KafkaConsumer(topic, group_id=uuid.uuid4().hex,59 bootstrap_servers=[ip],60 auto_offset_reset="latest", consumer_timeout_ms=3 * 1000)61 exceptException as ex:62 print(u"连接kafka失败!")63 returnFalse64 print(u"kafka消费对象创建成功.")65

66 with open(log, "w") as f:67 for i inrange(count):68 print(u"开始消费第%s条数据..." % str(i + 1))69 content =self.consume_msg(consumer)70 if notcontent:71 returnFalse72

73 #dict转json保存数据内容

74 content_json = json.dumps(content, ensure_ascii=False, indent=4)75 f.write(content_json)76 f.write("\n\n")77 print(u"第%s条数据写入完成." % str(i + 1))78

79 print(u"完成.")