python连接kafka消费者_python连接kafka生产者，消费者脚本

#-*- coding: utf-8 -*-

‘‘‘‘‘

使用kafka-Python 1.3.3模块

# pip install kafka==1.3.5

# pip install kafka-python==1.3.5‘‘‘

importsysimporttimeimportjsonfrom kafka importKafkaProducerfrom kafka importKafkaConsumerfrom kafka.errors importKafkaError

KAFAKA_HOST= "101.236.51.235"KAFAKA_PORT= 9092KAFAKA_TOPIC= "test"

classKafka_producer():‘‘‘‘‘

生产模块：根据不同的key，区分消息‘‘‘

def __init__(self, kafkahost,kafkaport, kafkatopic, key):

self.kafkaHost=kafkahost

self.kafkaPort=kafkaport

self.kafkatopic=kafkatopic

self.key=keyprint("producer:h,p,t,k",kafkahost,kafkaport,kafkatopic,key)

bootstrap_servers= ‘{kafka_host}:{kafka_port}‘.format(

kafka_host=self.kafkaHost,

kafka_port=self.kafkaPort

)print("boot svr:",bootstrap_servers)

self.producer= KafkaProducer(bootstrap_servers =bootstrap_servers

)defsendjsondata(self, params):try:

parmas_message= json.dumps(params,ensure_ascii=False)

producer=self.producerprint(parmas_message)

v= parmas_message.encode(‘utf-8‘)

k= key.encode(‘utf-8‘)print("send msg:(k,v)",k,v)

producer.send(self.kafkatopic, key=k, value=v)

producer.flush()exceptKafkaError as e:print(e)classKafka_consumer():‘‘‘‘‘

消费模块: 通过不同groupid消费topic里面的消息‘‘‘

def __init__(self, kafkahost, kafkaport, kafkatopic, groupid):

self.kafkaHost=kafkahost

self.kafkaPort=kafkaport

self.kafkatopic=kafkatopic

self.groupid=groupid

self.key=key

self.consumer= KafkaConsumer(self.kafkatopic, group_id =self.groupid,

bootstrap_servers= ‘{kafka_host}:{kafka_port}‘.format(

kafka_host=self.kafkaHost,

kafka_port=self.kafkaPort )

)defconsume_data(self):try:for message inself.consumer:yieldmessageexceptKeyboardInterrupt as e:print(e)defmain(xtype, group, key):‘‘‘‘‘

测试consumer和producer‘‘‘

if xtype == "p":#生产模块

producer =Kafka_producer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, key)print ("===========> producer:", producer)for _id in range(100):

params= ‘{"msg" : "%s"}‘ %str(_id)

params=[{"msg0" :_id},{"msg1":_id}]

producer.sendjsondata(params)

time.sleep(1)if xtype == ‘c‘:#消费模块

consumer =Kafka_consumer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, group)print ("===========> consumer:", consumer)

message=consumer.consume_data()for msg inmessage:print (‘msg---------------->k,v‘, msg.key,msg.value)print (‘offset---------------->‘, msg.offset)if __name__ == ‘__main__‘:

xtype= sys.argv[1]

group= sys.argv[2]

key= sys.argv[3]

main(xtype, group, key)