Kafka-code_demo

Kafka-code_demo

一.log

1.1.CustomProducer

package com.log;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.io.*;
import java.util.Properties;
import java.util.concurrent.ExecutionException;
public class CustomProducer {
    public static void main(String[] args) throws ExecutionException, InterruptedException, IOException {
        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.1.162:9092");//kafka集群，broker-list
        props.put("acks", "all");
        props.put("retries", 1);//批次大小
        props.put("linger.ms", 1);//等待时重试次数
        props.put("batch.size", 16384);//间
        props.put("buffer.memory", 33554432);//RecordAccumulator缓冲区大小
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        Producer<String, String> producer = new KafkaProducer<>(props);
        //读取文件
        //File file = new File("D:\\code\\log_to_kafka\\src\\main\\resources\\temop");
        File file = new File("D:\\code\\log_to_kafka\\src\\main\\resources\\water_sensor.txt");
        String encoding = "UTF-8";
        StringBuilder result = new StringBuilder();//
        FileInputStream fiStream = new FileInputStream(file);
        InputStreamReader inputStreamReader = new InputStreamReader(fiStream);
        BufferedReader bufferedReader = new BufferedReader(inputStreamReader);
        String line = null;
        while ((line = bufferedReader.readLine()) != null) {// 使用readLine方法，一次读一行
            result.append(line + System.lineSeparator());
            System.out.println("append result");
        }
        bufferedReader.close();
        inputStreamReader.close();
        fiStream.close();
        boolean flag = true;
        while (flag) {
            producer.send(new ProducerRecord<String, String>("water_sensor", result.toString()));
            Thread.sleep(5000);
        }
        producer.close();
    }
}

二.kafka_api

2.1.Create_Topic

package kafka_api;
import org.apache.kafka.clients.admin.*;
import java.util.*;
public class Create_Topic {
    public static void main(String[] args) {
        Properties props = new Properties();
        props.put("bootstrap.servers", "192.168.1.162:9092");//kafka集群，broker-list
        props.put("acks", "all");
        props.put("retries", 1);//批次大小
        props.put("linger.ms", 1);//等待时重试次数
        props.put("batch.size", 16384);//间
        props.put("buffer.memory", 33554432);//RecordAccumulator缓冲区大小
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        AdminClient ac = AdminClient.create(props);
        Collection col = new ArrayList();
        NewTopic newTopic = new NewTopic("water_sensor",3, (short) 2);
        col.add(newTopic);
        CreateTopicsResult topics = ac.createTopics(col);
    }
}

2.2.List_Topics

package kafka_api;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.util.Properties;
public class List_Topics {
    public static void main(String[] args) {
        Properties properties = new Properties();
        properties.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());// key反序列化方式
        properties.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getCanonicalName());// value反系列化方式
        properties.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, true);// 提交方式
        properties.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,
                "hadoop162:9092");// 指定broker地址，来找到group的coordinator
        properties.put(ConsumerConfig.GROUP_ID_CONFIG, "tuyou");// 指定用户组
        KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(properties);
        System.out.println(consumer.listTopics().keySet());
    }
}

数据文件

temop

{"app_id":"4","device_id":"103","distinct_id":"bad691f2-95f3-43de-8804-23d8c6f50afb","event_name":"-","ip":"139.196.217.70","last_event_name":"-","last_page_id":"0","next_event_name":"-","next_page_id":"2","page_id":"1","server_time":"-","uid":"768152"}
{"app_id":"3","device_id":"103","distinct_id":"43a1937d-dd7d-4ac6-9b8e-f580a233fd01","event_name":"-","ip":"36.57.247.68","last_event_name":"-","last_page_id":"1","next_event_name":"-","next_page_id":"3","page_id":"2","server_time":"-","uid":"768153"}
{"app_id":"1","device_id":"102","distinct_id":"ada43d6e-e06a-4407-ae4e-225eaa4b1db3","event_name":"-","ip":"222.24.86.99","last_event_name":"-","last_page_id":"1","next_event_name":"-","next_page_id":"3","page_id":"2","server_time":"-","uid":"768154"}
{"app_id":"4","device_id":"102","distinct_id":"979f196d-8304-4209-a7d8-fb044afdab27","event_name":"-","ip":"210.37.110.50","last_event_name":"-","last_page_id":"0","next_event_name":"-","next_page_id":"2","page_id":"1","server_time":"-","uid":"768155"}

water_sensor.txt

001,123456,100    //id,时间戳,水位高度
002,123457,200
003,123458,300
004,123459,400
005,123460,500

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>
    <groupId>com.kafka.test</groupId>
    <artifactId>log_to_kafka</artifactId>
    <version>1.0-SNAPSHOT</version>
<dependencies>
    <dependency>
        <groupId>org.apache.kafka</groupId>
        <artifactId>kafka-clients</artifactId>
        <version>2.4.1</version>
    </dependency>
</dependencies>
</project>