python消费kafka性能_Python消费kafka数据

1.# -*- coding: utf-8 -*-

2.importjson

3.

4.fromkafkaimportKafkaConsumer, TopicPartition

5.fromapps.log_databus.exceptionsimportKafkaPartitionException, KafkaConnectException

6.

7.

8.classKafkaConsumerHandle(object):

9.

10.def__init__(self, server, port, topic, username=None, password=None):

11.self.server = server

12.self.port = int(port)

13.self.topic = topic

14.self.kafka_server = server +':'+ str(port)

15.try:

16.ifusername:

17.self.consumer = KafkaConsumer(self.topic, bootstrap_servers=self.kafka_server,

18.security_protocol='SASL_PLAINTEXT',

19.sasl_mechanism='PLAIN', sasl_plain_username=username,

20.sasl_plain_password=password, request_timeout_ms=1000,

21.consumer_timeout_ms=1000)

22.else:

23.self.consumer = KafkaConsumer(self.topic, bootstrap_servers=self.kafka_server,

24.request_timeout_ms=1000, consumer_timeout_ms=1000)

25.exceptException as e:

26.raiseKafkaConnectException(KafkaConnectException.MESSAGE.format(error=e))

27.

28.defget_latest_log(self):

29."""

30.读取kafka的数据

31.:return:

32."""

33.message_count = 10

34.self.consumer.poll(10)

35.

36.#获取topic分区信息

37.topic_partitions = self.consumer.partitions_for_topic(self.topic)

38.ifnottopic_partitions:

39.raiseKafkaPartitionException()

40.

41.log_content = []

42.for_partitionintopic_partitions:

43.

44.#获取该分区最大偏移量

45.tp = TopicPartition(topic=self.topic, partition=_partition)

46.end_offset = self.consumer.end_offsets([tp])[tp]

47.ifnotend_offset:

48.continue

49.

50.#设置消息消费偏移量

51.ifend_offset >= message_count:

52.self.consumer.seek(tp, end_offset - message_count)

53.else:

54.self.consumer.seek_to_beginning()

55.

56.#消费消息

57.for_msginself.consumer:

58.try:

59.log_content.insert(0, json.loads(_msg.value.decode()))

60.exceptException:

61.pass

62.iflen(log_content) == message_count:

63.self.consumer.close()

64.returnlog_content

65.if_msg.offset == end_offset - 1:

66.break

67.

68.self.consumer.close()

69.returnlog_content