之前用的kafka的依赖是confluent_kafka,该依赖并不是异步的,找到了一款异步kafka依赖aiokafka,并发大量kafka消息代码如下,QPS可达1000。
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import asyncio
import logging
import os
import random
import time
from aiokafka import AIOKafkaProducer
from nio_messages.nextev_msg import gen_nextev_message
formatter = "[%(asctime)s] %(name)s {%(filename)s:%(lineno)d} %(levelname)s - %(message)s"
logging.basicConfig(level=logging.INFO, format=formatter)
base_dir = os.path.dirname(os.path.realpath(__file__))
kafka_conf = {
"bootstrap_servers": ["xxx1.xxx.com:9092",
"xxx2.xxx.com:9092",
"xxx3.xxx.com:9092"],
"security_protocol": "SASL_PLAINTEXT",
"sasl_mechanism": "PLAIN",
"sasl_plain_username": "xxx",
"sasl_plain_password": "xxx"}
async def send_msg(producer, topic, account_id):
for i in range(10000):
status = random.choice(["OFFLINE", "ONLINE"])
msg = gen_nextev_message(sub_type="connection_status_event",
data={
"status": status
},
publish_ts=round(time.time() * 1000),
msg_type=4,
account_id=account_id)
if i == 9999:
logging.info(f"{account_id} : {status}")
try:
await producer.send_and_wait(topic, msg)
except Exception:
logging.info(f"send {account_id} error")
async def send_concurrently():
account_ids = ["7c62cebc0f834e6f866d62394a9304f6", "34d0b67de87147839edaa30d9f009132",
"b6f7892697c84a8cbbbfd3bd85a0217e", "94187e6e55804baebf8bcec2fd4cee40",
"fedeb10e43ee496bb7718ad02a676a53", "2f095231467c4f35a76cbad9edb0d3e3",
"e61e91f2e35d4d3f9d963047f309e222", "7048c7996e3e41858604905474936055",
"7dba29e78f9d4f97940215b04a562426", "c188f22642f54432a4d90706b73e59c2"]
topic = "my_topic"
producer = AIOKafkaProducer(**kafka_conf)
await producer.start()
loop = asyncio.get_event_loop()
try:
tasks = [loop.create_task(send_msg(producer, topic, account_id)) for account_id in account_ids]
await asyncio.wait(tasks)
logging.info('All Done')
finally:
await producer.stop()
if __name__ == '__main__':
start_time = time.time()
asyncio.run(send_concurrently())
logging.info(f"time cost: {time.time() - start_time}")
注:需要python3.7以上,因为希望kafka实例为单例模式,以避免频繁创建和销毁kafka实例带来资源消耗,所以将kafka实例的创建放到了async类型的send_concurrently函数中,main中采用了asyncio.run()方式,这样可在多线程中使用同一个kafka单实例。但aiokafka中对于kafka实例的创建并不是单例模式,所以在高并发情况下,往往并不能保证只创建一个实例,这里需要做优化。