利用python asyncio 异步publish kafka消息

之前用的kafka的依赖是confluent_kafka，该依赖并不是异步的，找到了一款异步kafka依赖aiokafka，并发大量kafka消息代码如下，QPS可达1000。

#!/usr/bin/env python  
# -*- coding:utf-8 -*- 
import asyncio
import logging
import os
import random
import time

from aiokafka import AIOKafkaProducer
from nio_messages.nextev_msg import gen_nextev_message

formatter = "[%(asctime)s] %(name)s {%(filename)s:%(lineno)d} %(levelname)s - %(message)s"
logging.basicConfig(level=logging.INFO, format=formatter)

base_dir = os.path.dirname(os.path.realpath(__file__))

kafka_conf = {
    "bootstrap_servers": ["xxx1.xxx.com:9092",
                          "xxx2.xxx.com:9092",
                          "xxx3.xxx.com:9092"],
    "security_protocol": "SASL_PLAINTEXT",
    "sasl_mechanism": "PLAIN",
    "sasl_plain_username": "xxx",
    "sasl_plain_password": "xxx"}


async def send_msg(producer, topic, account_id):
    for i in range(10000):
        status = random.choice(["OFFLINE", "ONLINE"])
        msg = gen_nextev_message(sub_type="connection_status_event",
                                 data={
                                     "status": status
                                 },
                                 publish_ts=round(time.time() * 1000),
                                 msg_type=4,
                                 account_id=account_id)
        if i == 9999:
            logging.info(f"{account_id} : {status}")
        try:
            await producer.send_and_wait(topic, msg)
        except Exception:
            logging.info(f"send {account_id} error")


async def send_concurrently():
    account_ids = ["7c62cebc0f834e6f866d62394a9304f6", "34d0b67de87147839edaa30d9f009132",
            "b6f7892697c84a8cbbbfd3bd85a0217e", "94187e6e55804baebf8bcec2fd4cee40",
            "fedeb10e43ee496bb7718ad02a676a53", "2f095231467c4f35a76cbad9edb0d3e3",
            "e61e91f2e35d4d3f9d963047f309e222", "7048c7996e3e41858604905474936055",
            "7dba29e78f9d4f97940215b04a562426", "c188f22642f54432a4d90706b73e59c2"]

    topic = "my_topic"

    producer = AIOKafkaProducer(**kafka_conf)
    await producer.start()

    loop = asyncio.get_event_loop()

    try:
        tasks = [loop.create_task(send_msg(producer, topic, account_id)) for account_id in account_ids]
        await asyncio.wait(tasks)
        logging.info('All Done')
    finally:
        await producer.stop()


if __name__ == '__main__':
    start_time = time.time()
    asyncio.run(send_concurrently())
    logging.info(f"time cost: {time.time() - start_time}")

注：需要python3.7以上，因为希望kafka实例为单例模式，以避免频繁创建和销毁kafka实例带来资源消耗，所以将kafka实例的创建放到了async类型的send_concurrently函数中，main中采用了asyncio.run()方式，这样可在多线程中使用同一个kafka单实例。但aiokafka中对于kafka实例的创建并不是单例模式，所以在高并发情况下，往往并不能保证只创建一个实例，这里需要做优化。