二百六十一、Java——采集Kafka数据，解析成一条条数据，写入另一Kafka中（简单JSON）

一、目的

由于部分数据类型频率为1s，从而数据规模特别大，因此完整的JSON放在Hive中解析起来，尤其是在单机环境下，效率特别慢，无法满足业务需求。

而Flume的拦截器并不能很好的转换数据，因为只能采用Java方式，从Kafka的主题A中采集数据，并解析字段，然后写入到放在Kafka主题B中

二 、原始数据格式

JSON格式比较简单，对象中包含对象

{
    "deviceNo": "39",
    "sourceDeviceType": null,
    "sn": null,
    "model": null,
    "createTime": "2024-09-03 14:55:00",
    "data": {
        "cycle": 300,
        "volumeSum": 34,
        "speedAvg": 22.0,
        "volumeLeft": 12,
        "speedLeft": 22.0,
        "volumeStraight": 22,
        "speedStraight": 22.0,
        "volumeRight": 0,
        "speedRight": 0.0,
        "volumeTurn": null,
        "speedTurn": null
    }
}

三、Java代码

package com.kgc;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerConfig;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.clients.producer.RecordMetadata;
import org.apache.kafka.common.serialization.StringDeserializer;
import org.apache.kafka.common.serialization.StringSerializer;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

public class KafkaKafkaTurnratio {

    // 添加 Kafka Producer 配置
    private static Properties producerProps() {
        Properties props = new Properties();
        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.0.70:9092");
        props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class);
        props.put(ProducerConfig.ACKS_CONFIG, "-1");
        props.put(ProducerConfig.RETRIES_CONFIG, "3");
        props.put(ProducerConfig.BATCH_SIZE_CONFIG, "16384");
        props.put(ProducerConfig.LINGER_MS_CONFIG, "1");
        props.put(ProducerConfig.BUFFER_MEMORY_CONFIG, "33554432");
        return props;
    }

    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.0.70:9092");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, "1000");
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        // 每一个消费，都要定义不同的Group_ID
        prop.put(ConsumerConfig.GROUP_ID_CONFIG, "turnratio_group");

        KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
        consumer.subscribe(Collections.singleton("topic_internal_data_turn_ratio"));
        ObjectMapper mapper = new ObjectMapper();

        // 初始化 Kafka Producer
        KafkaProducer<String, String> producer = new KafkaProducer<>(producerProps());

        while (true) {
            ConsumerRecords<String, String> records = consumer.poll(Duration.ofMillis(1000));
            for (ConsumerRecord<String, String> record : records) {
                try {
                    JsonNode rootNode = mapper.readTree(record.value());

                    System.out.println("原始数据"+rootNode);

                    String device_no = rootNode.get("deviceNo").asText();
                    String source_device_type = rootNode.get("sourceDeviceType").asText();
                    String sn = rootNode.get("sn").asText();
                    String model = rootNode.get("model").asText();
                    String create_time = rootNode.get("createTime").asText();

                    String cycle = rootNode.get("data").get("cycle").asText();
                    String volume_sum = rootNode.get("data").get("volumeSum").asText();
                    String speed_avg = rootNode.get("data").get("speedAvg").asText();
                    String volume_left = rootNode.get("data").get("volumeLeft").asText();
                    String speed_left = rootNode.get("data").get("speedLeft").asText();
                    String volume_straight = rootNode.get("data").get("volumeStraight").asText();
                    String speed_straight = rootNode.get("data").get("speedStraight").asText();
                    String volume_right = rootNode.get("data").get("volumeRight").asText();
                    String speed_right = rootNode.get("data").get("speedRight").asText();
                    String volume_turn = rootNode.get("data").get("volumeTurn").asText();
                    String speed_turn = rootNode.get("data").get("speedTurn").asText();

                    String outputLine = String.format("%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s",
                                device_no, source_device_type, sn, model, create_time,cycle,volume_sum,speed_avg,volume_left,speed_left,volume_straight,speed_straight,volume_right,speed_right,volume_turn,speed_turn);

                    // 发送数据到 Kafka
                    ProducerRecord<String, String> producerRecord = new ProducerRecord<>("topic_db_data_turn_ratio", record.key(), outputLine);
                    producer.send(producerRecord, (RecordMetadata metadata, Exception e) -> {
                            if (e != null) {
                                e.printStackTrace();
                            } else {
                                System.out.println("The offset of the record we just sent is: " + metadata.offset());
                            }
                    });
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
            consumer.commitAsync();
        }
    }

}

剩下的几步参考二百五十九、Java——采集Kafka数据，解析成一条条数据，写入另一Kafka中（一般JSON）这篇博客

四、开启Kafka主题消费者
五、运行测试
六、ODS层新表结构
七、Flume采集配置文件
八、运行Flume任务，检查HDFS文件、以及ODS表数据