kafak怎么快速批量造一百万数据

例如 我需要造的数据模型如下

其中 msg_creation_time ,qualifiedName ,name,guid 是变化的

{
    "version": {
        "version": "1.0.0",
        "versionParts": [
            1
        ]
    },
    "msg_compression_kind": "1",
    "msg_split_idx": 1,
    "msg_source_ip": "10.82.13.74",
    "msg_created_by": "DELL",
    "msg_creation_time": 1711525684104,
    "message": {
        "type": "ENTITY_CREATE_V2",
        "user": "DELL",
        "entities": {
            "entities": [
                {
                    "typeName": "ceshifuzadu2",
                    "attributes": {
                        "qualifiedName": "1711525684104",
                        "tenantId": "tenantId12",
                        "name": "百万消息测试复杂度2024-03-27 07:48:04.104966+00",
                        "userId": "userId12",
                        "version": 1
                    },
                    "guid": "-1711525684104",
                    "isIncomplete": false,
                    "provenanceType": 0,
                    "version": 0,
                    "proxy": false
                }
            ]
        }
    }
}

1. 我在pg数据库当中建立对应的表

create table test_b
(
    version              json,
    msg_compression_kind varchar,
    msg_split_idx        integer,
    msg_source_ip        varchar,
    msg_created_by       varchar,
    msg_creation_time    bigint,
    message              json
);

2. 创建造数脚本 造 100万条

DO $$
    DECLARE k integer := 1;
    DECLARE message json :='{}';
    begin
        WHILE k <= 1000000
            loop
                WITH dynamic_json AS (
                    SELECT json_build_object(
                                   'type', 'ENTITY_CREATE_V2',
                                   'user', 'DELL',
                                   'entities', json_build_object(
                                           'entities', json_build_array(
                                            json_build_object(
                                                    'typeName', 'ceshifuzadu2',
                                                    'attributes', json_build_object(
                                                            'qualifiedName', ''||FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
                                                            'tenantId', 'tenantId12',
                                                            'name', '百万消息测试复杂度' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
                                                            'userId', 'userId12',
                                                            'version', 1
                                                                  ),
                                                    'guid', '-' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
                                                    'isIncomplete', false,
                                                    'provenanceType', 0,
                                                    'version', 0,
                                                    'proxy', false
                                            )
                                                       )
                                               )
                           ) AS messageb )
                SELECT messageb INTO message FROM dynamic_json;
                INSERT INTO public.test_b
                ("version", "msg_compression_kind", "msg_split_idx", "msg_source_ip", "msg_created_by", "msg_creation_time", "message")
                VALUES('{"version":"1.0.0","versionParts":[1]}', 1, 1,
                       '10.82.13.74','DELL',FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000),
                       message);
                k = k + 1;
            end loop;
    END $$;

3. 使用psql 将数据导出

## psql 登录
psql -U postgre -w
## 切换数据库
\c database;
## 执行导出数据的命令
COPY (select row_to_json(t) from test_b t) to '/tmp/testb.json';

4. 到kafka的bin目录当中 执行 这行命令

/opt/kafka_2.13-2.8.1/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic ATLAS_HOOK < /tmp/testb.json