#-*- coding: utf-8 -*-
‘‘‘‘‘
使用kafka-Python 1.3.3模块
# pip install kafka==1.3.5
# pip install kafka-python==1.3.5‘‘‘
importsysimporttimeimportjsonfrom kafka importKafkaProducerfrom kafka importKafkaConsumerfrom kafka.errors importKafkaError
KAFAKA_HOST= "101.236.51.235"KAFAKA_PORT= 9092KAFAKA_TOPIC= "test"
classKafka_producer():‘‘‘‘‘
生产模块:根据不同的key,区分消息‘‘‘
def __init__(self, kafkahost,kafkaport, kafkatopic, key):
self.kafkaHost=kafkahost
self.kafkaPort=kafkaport
self.kafkatopic=kafkatopic
self.key=keyprint("producer:h,p,t,k",kafkahost,kafkaport,kafkatopic,key)
bootstrap_servers= ‘{kafka_host}:{kafka_port}‘.format(
kafka_host=self.kafkaHost,
kafka_port=self.kafkaPort
)print("boot svr:",bootstrap_servers)
self.producer= KafkaProducer(bootstrap_servers =bootstrap_servers
)defsendjsondata(self, params):try:
parmas_message= json.dumps(params,ensure_ascii=False)
producer=self.producerprint(parmas_message)
v= parmas_message.encode(‘utf-8‘)
k= key.encode(‘utf-8‘)print("send msg:(k,v)",k,v)
producer.send(self.kafkatopic, key=k, value=v)
producer.flush()exceptKafkaError as e:print(e)classKafka_consumer():‘‘‘‘‘
消费模块: 通过不同groupid消费topic里面的消息‘‘‘
def __init__(self, kafkahost, kafkaport, kafkatopic, groupid):
self.kafkaHost=kafkahost
self.kafkaPort=kafkaport
self.kafkatopic=kafkatopic
self.groupid=groupid
self.key=key
self.consumer= KafkaConsumer(self.kafkatopic, group_id =self.groupid,
bootstrap_servers= ‘{kafka_host}:{kafka_port}‘.format(
kafka_host=self.kafkaHost,
kafka_port=self.kafkaPort )
)defconsume_data(self):try:for message inself.consumer:yieldmessageexceptKeyboardInterrupt as e:print(e)defmain(xtype, group, key):‘‘‘‘‘
测试consumer和producer‘‘‘
if xtype == "p":#生产模块
producer =Kafka_producer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, key)print ("===========> producer:", producer)for _id in range(100):
params= ‘{"msg" : "%s"}‘ %str(_id)
params=[{"msg0" :_id},{"msg1":_id}]
producer.sendjsondata(params)
time.sleep(1)if xtype == ‘c‘:#消费模块
consumer =Kafka_consumer(KAFAKA_HOST, KAFAKA_PORT, KAFAKA_TOPIC, group)print ("===========> consumer:", consumer)
message=consumer.consume_data()for msg inmessage:print (‘msg---------------->k,v‘, msg.key,msg.value)print (‘offset---------------->‘, msg.offset)if __name__ == ‘__main__‘:
xtype= sys.argv[1]
group= sys.argv[2]
key= sys.argv[3]
main(xtype, group, key)