Txt文本传数据到Kafka的两种方式

方式一 自定义合并分区类

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;

/**
 * @author 张泰
 * @Version v 1.0
 * @date 2020/4/15
 */
public class show {
    public static void main(String[] args) throws IOException {
        Properties props = new Properties();
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        props.put("acks", "-1");
        props.put("retries", 0);
        props.put("batch.size", 16384);
        props.put("linger.ms", 1);
        props.put("buffer.memory", 33554432);
        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
        Producer<String, String> producer = new KafkaProducer<>(props);
//        bin/ kafka-console-consumer.sh --from-beginning --topic rng_comment  --zookeeper node01:2181,node02:2181,node03:2181
        File inputFile = new File("E:\\第四学期视频笔记\\作业\\4.14号练习题\\4.14号练习题\\aaa.txt");
        BufferedReader bufferedReader = new BufferedReader(new FileReader(inputFile));
        String line = null;
        int partition = 0;
        while ((line = bufferedReader.readLine()) != null) {
            try {
                if(Integer.parseInt(line.split("\t")[0]) % 2== 0){
                    partition = 0;
                }else{
                    partition = 1;
                }
            }catch (NumberFormatException e){
                continue;
            }
            producer.send(new ProducerRecord<String, String>("rng_comment1",partition,String.valueOf(partition),line));
        }
        bufferedReader.close();
        producer.close();
    }
}

方式二 自定义分区

package Job;

import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;

public class TxtToKafka {

    /*
    1、	读取数据
    2、	写入到kafka中
    3、	id奇数一个分区，偶数一个分区

     */
    public static void main(String[] args) throws Exception {


      // 、配置kafka集群
        Properties props = new Properties();
        //kafka服务器地址
        props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
        //消息确认机制
        props.put("acks", "all");
        //重试机制
        props.put("retries", 0);
        //批量发送的大小
        props.put("batch.size", 16384);
        //消息延迟
        props.put("linger.ms", 1);
        //批量的缓冲区大小
        props.put("buffer.memory", 33554432);
        //kafka数据中key  value的序列化
        props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
        props.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
        props.put("partitioner.class", "Job.MyPartition");

        //2 写入到kafka中    将"aa"  写入到kafka

        KafkaProducer kafkaProducer = new KafkaProducer(props);


    //1读取数据

          File file = new File("C:\\Users\\Administrator\\Desktop\\SQL\\4.15号练习题\\rng_comment.txt");
          BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
          String Line=null;
          while((Line=bufferedReader.readLine())!=""){
              if(!Line.equals("\t\t\t\t\t\t\t\t\t\t")){

                  kafkaProducer.send(new ProducerRecord("18BD12", Line));

                  System.out.println(Line);
                  Thread.sleep(100);
              }
          }
        kafkaProducer.close();
      //3 id奇数一个分区，偶数一个分区
        //使用自定义分区设置数据
    }
}

package Job;

import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;

import java.util.Map;

public class MyPartition  implements Partitioner {

    //线面的代码块就是自定义分区的代码
    @Override
    public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {

        /*
        0	11381	2018/10/20 21:07	今天你还相信网恋吗：希望越大失望越大，技能省得莫名其妙，小规模团战打得又乱，最后一句，骄兵必败！       	1	219517		3873092044	今天你还相信网恋吗	5	1540040820
        1	683	2018/10/20 21:07	趣游电竞：这几把打的什么呀，       	0	36213		3784336787	趣游电竞	2	1540040820
        2	0	2018/10/20 21:08	阴暗感i：既然选择了RNG，那么我会一直支持RNG！       	0	0		6019401575	阴暗感i	0	1540040880
        3	0	2018/10/20 21:08	买桔子的先森：孙大勇和heart亲妈今晚必死，我不想看到他们       	0	0		5440806717	买桔子的先森	0	1540040880
         */

        //获取数据中的ID
         String idstring = value.toString().split("\t")[0];
          int idint = Integer.parseInt(idstring);

          //2468
          if (idint%2==0){
              return 0;
          }else{
              return 1;
          }
   }

    @Override
    public void close() {

    }

    @Override
    public void configure(Map<String, ?> configs) {

    }
}