python消费kafka性能_python生产消费Kafka

from kafka importKafkaProducer, KafkaConsumerclassPythonKafka(object):def __init__(self, topic=None, username=None, password=None):

self.topic=topic

self.username=username

self.password=password

self.count=0

self.error_count=0def on_send_success(self, *args, **kwargs):

self.count= self.count + 1

def on_send_error(self, *args, **kwargs):

self.error_count= self.error_count + 1

defget_kafka_producer(self):

kafka_producer= KafkaProducer(bootstrap_servers=['ip1:port', 'ip2:port', 'ip3:port'],

security_protocol='SASL_PLAINTEXT',

sasl_mechanism='PLAIN',

sasl_plain_username='username',

sasl_plain_password='password',

request_timeout_ms=40000,

api_version=(0, 10, 1))returnkafka_producerdefget_kafka_consumer(self):#指定超时时间，不然会一直阻塞

consumer_params ={'security_protocol': 'SASL_PLAINTEXT','sasl_mechanism': 'PLAIN','group_id': 'test_group2','sasl_plain_username': 'username','sasl_plain_password': 'password','auto_offset_reset': 'earliest','api_version': (0, 10),'consumer_timeout_ms': 10000}returnKafkaConsumer(self.topic,

bootstrap_servers=['ip1:port', 'ip2:port', 'ip3:port'],**consumer_params)#异步发送

defsend_data(self, json_str):

json_bytes=str.encode(json_str)

producer=self.get_kafka_producer()

producer.send(self.topic, value=json_bytes).add_callback(self.on_send_success).add_errback(

self.on_send_error)

producer.flush()

producer.close()#接收数据

defreceive_data(self):

consumer=self.get_kafka_consumer()for msg inconsumer:

msg_str= str(msg.value, encoding='utf8')print(msg_str)

consumer.close()