方式一 自定义合并分区类
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.Producer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Properties;
/**
* @author 张泰
* @Version v 1.0
* @date 2020/4/15
*/
public class show {
public static void main(String[] args) throws IOException {
Properties props = new Properties();
props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
props.put("acks", "-1");
props.put("retries", 0);
props.put("batch.size", 16384);
props.put("linger.ms", 1);
props.put("buffer.memory", 33554432);
props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
Producer<String, String> producer = new KafkaProducer<>(props);
// bin/ kafka-console-consumer.sh --from-beginning --topic rng_comment --zookeeper node01:2181,node02:2181,node03:2181
File inputFile = new File("E:\\第四学期视频笔记\\作业\\4.14号练习题\\4.14号练习题\\aaa.txt");
BufferedReader bufferedReader = new BufferedReader(new FileReader(inputFile));
String line = null;
int partition = 0;
while ((line = bufferedReader.readLine()) != null) {
try {
if(Integer.parseInt(line.split("\t")[0]) % 2== 0){
partition = 0;
}else{
partition = 1;
}
}catch (NumberFormatException e){
continue;
}
producer.send(new ProducerRecord<String, String>("rng_comment1",partition,String.valueOf(partition),line));
}
bufferedReader.close();
producer.close();
}
}
方式二 自定义分区
package Job;
import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
import java.util.Properties;
public class TxtToKafka {
/*
1、 读取数据
2、 写入到kafka中
3、 id奇数一个分区,偶数一个分区
*/
public static void main(String[] args) throws Exception {
// 、配置kafka集群
Properties props = new Properties();
//kafka服务器地址
props.put("bootstrap.servers", "node01:9092,node02:9092,node03:9092");
//消息确认机制
props.put("acks", "all");
//重试机制
props.put("retries", 0);
//批量发送的大小
props.put("batch.size", 16384);
//消息延迟
props.put("linger.ms", 1);
//批量的缓冲区大小
props.put("buffer.memory", 33554432);
//kafka数据中key value的序列化
props.put("key.serializer","org.apache.kafka.common.serialization.StringSerializer");
props.put("value.serializer","org.apache.kafka.common.serialization.StringSerializer");
props.put("partitioner.class", "Job.MyPartition");
//2 写入到kafka中 将"aa" 写入到kafka
KafkaProducer kafkaProducer = new KafkaProducer(props);
//1读取数据
File file = new File("C:\\Users\\Administrator\\Desktop\\SQL\\4.15号练习题\\rng_comment.txt");
BufferedReader bufferedReader = new BufferedReader(new FileReader(file));
String Line=null;
while((Line=bufferedReader.readLine())!=""){
if(!Line.equals("\t\t\t\t\t\t\t\t\t\t")){
kafkaProducer.send(new ProducerRecord("18BD12", Line));
System.out.println(Line);
Thread.sleep(100);
}
}
kafkaProducer.close();
//3 id奇数一个分区,偶数一个分区
//使用自定义分区设置数据
}
}
package Job;
import org.apache.kafka.clients.producer.Partitioner;
import org.apache.kafka.common.Cluster;
import java.util.Map;
public class MyPartition implements Partitioner {
//线面的代码块就是自定义分区的代码
@Override
public int partition(String topic, Object key, byte[] keyBytes, Object value, byte[] valueBytes, Cluster cluster) {
/*
0 11381 2018/10/20 21:07 今天你还相信网恋吗:希望越大失望越大,技能省得莫名其妙,小规模团战打得又乱,最后一句,骄兵必败! 1 219517 3873092044 今天你还相信网恋吗 5 1540040820
1 683 2018/10/20 21:07 趣游电竞:这几把打的什么呀, 0 36213 3784336787 趣游电竞 2 1540040820
2 0 2018/10/20 21:08 阴暗感i:既然选择了RNG,那么我会一直支持RNG! 0 0 6019401575 阴暗感i 0 1540040880
3 0 2018/10/20 21:08 买桔子的先森:孙大勇和heart亲妈今晚必死,我不想看到他们 0 0 5440806717 买桔子的先森 0 1540040880
*/
//获取数据中的ID
String idstring = value.toString().split("\t")[0];
int idint = Integer.parseInt(idstring);
//2468
if (idint%2==0){
return 0;
}else{
return 1;
}
}
@Override
public void close() {
}
@Override
public void configure(Map<String, ?> configs) {
}
}