例如 我需要造的数据模型如下
其中 msg_creation_time ,qualifiedName ,name,guid 是变化的
{
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msg_compression_kind": "1",
"msg_split_idx": 1,
"msg_source_ip": "10.82.13.74",
"msg_created_by": "DELL",
"msg_creation_time": 1711525684104,
"message": {
"type": "ENTITY_CREATE_V2",
"user": "DELL",
"entities": {
"entities": [
{
"typeName": "ceshifuzadu2",
"attributes": {
"qualifiedName": "1711525684104",
"tenantId": "tenantId12",
"name": "百万消息测试复杂度2024-03-27 07:48:04.104966+00",
"userId": "userId12",
"version": 1
},
"guid": "-1711525684104",
"isIncomplete": false,
"provenanceType": 0,
"version": 0,
"proxy": false
}
]
}
}
}
- 我在pg数据库当中建立对应的表
create table test_b
(
version json,
msg_compression_kind varchar,
msg_split_idx integer,
msg_source_ip varchar,
msg_created_by varchar,
msg_creation_time bigint,
message json
);
- 创建造数脚本 造 100万条
DO $$
DECLARE k integer := 1;
DECLARE message json :='{}';
begin
WHILE k <= 1000000
loop
WITH dynamic_json AS (
SELECT json_build_object(
'type', 'ENTITY_CREATE_V2',
'user', 'DELL',
'entities', json_build_object(
'entities', json_build_array(
json_build_object(
'typeName', 'ceshifuzadu2',
'attributes', json_build_object(
'qualifiedName', ''||FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
'tenantId', 'tenantId12',
'name', '百万消息测试复杂度' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
'userId', 'userId12',
'version', 1
),
'guid', '-' || FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000)+k,
'isIncomplete', false,
'provenanceType', 0,
'version', 0,
'proxy', false
)
)
)
) AS messageb )
SELECT messageb INTO message FROM dynamic_json;
INSERT INTO public.test_b
("version", "msg_compression_kind", "msg_split_idx", "msg_source_ip", "msg_created_by", "msg_creation_time", "message")
VALUES('{"version":"1.0.0","versionParts":[1]}', 1, 1,
'10.82.13.74','DELL',FLOOR(EXTRACT(EPOCH FROM NOW()) * 1000),
message);
k = k + 1;
end loop;
END $$;
- 使用psql 将数据导出
## psql 登录
psql -U postgre -w
## 切换数据库
\c database;
## 执行导出数据的命令
COPY (select row_to_json(t) from test_b t) to '/tmp/testb.json';
- 到kafka的bin目录当中 执行 这行命令
/opt/kafka_2.13-2.8.1/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic ATLAS_HOOK < /tmp/testb.json