python使用kafka-kafka基本命令，python-kafka使用

1.启动

nohup kafka-server-start /usr/local/etc/kafka/server-1.properties &

2.查看消费者

kafka-console-consumer --bootstrap-server localhost:9092 --topic test --from-beginning

3.描述

kafka-topics --describe --bootstrap-server localhost:9092 --topic my-replicated-topic

生产者:

class Kafkaproducer():

"""

生产模块：根据不同的key,区分消息

"""

def __init__(self, kafka_host,kafka_port,kafka_topic, key):

self.kafka_host = kafka_host

self.kafka_port = kafka_port

self.kafka_topic = kafka_topic

self.key = key

bootstrap_servers = "{kafka_host}:{kafka_port}".format(

kafka_host=self.kafka_host,kafka_port=self.kafka_port

)

self.producer = KafkaProducer(bootstrap_servers = [bootstrap_servers])

def send_message(self,params):

try:

params_message = json.dumps(params).encode('utf-8')

producer = self.producer

#producer.send(self.kafka_topic, key=self.key, value=params_message)

producer.send(self.kafka_topic,key=self.key,value=params_message) #key,value必须是字节

producer.flush()

except KafkaError as e:

print(e)

消费者:

class Kafkaconsumer():

"""

消费者

"""

def init(self,kafka_host,kafka_port, kafka_topic, groupid):

self.kafka_host = kafka_host

self.kafka_port = kafka_port

self.kafka_topic = kafka_topic

self.groupid = groupid

self.consumer = KafkaConsumer(self.kafka_topic,group_id=self.groupid,

bootstrap_servers = "{kafka_host}:{kafka_port}".format(

kafka_host=self.kafka_host,

kafka_port=self.kafka_port

))

def consume_message(self):

try:

for message in self.consumer:

yield message

except KeyboardInterrupt as e:

print(e)