本篇文章将使用kafka将数据写入到hbase中
一、使用的包
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>3.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka_2.12</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>1.2.0</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>1.2.0</version>
</dependency>
二、 具体写入流程
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Properties;
import java.util.Random;
public class TrainToHb {
//全局的静态变量用来计数
static long num = 0;
public static void main(String[] args) {
//消费者的配置
Properties prop = new Properties();
prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9092");
prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
prop.put(ConsumerConfig.GROUP_ID_CONFIG,"user_group4");
prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");
//创建消费者并指定要消费的主题
KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(prop);
consumer.subscribe(Collections.singleton("train"));
//连接hbase
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum","hadoop102");
conf.set("hbase.zookeeper.property.clientPort","2181");
conf.set("hbase.rootdir","hdfs://hadoop102:8020/hbase");
Connection connection = null;
try {
//连接Hbase
connection = ConnectionFactory.createConnection(conf);
//确定Hbase的命名空间和表名
TableName tableName = TableName.valueOf("events_db:train");
//如果失败
BufferedMutator.ExceptionListener listener = new BufferedMutator.ExceptionListener(){
@Override
public void onException(RetriesExhaustedWithDetailsException e, BufferedMutator bufferedMutator) throws RetriesExhaustedWithDetailsException {
int nums = e.getNumExceptions();
for (int i = 0; i < nums; i++) {
System.out.println("失败,保存 " + e.getRow(i) + ".");
}
}
};
//采用mutator高效写入hbase
BufferedMutatorParams params = new BufferedMutatorParams(tableName);
params.writeBufferSize(1024*1024);
params.listener(listener);
BufferedMutator mutator = connection.getBufferedMutator(params);
ArrayList<Put> list = new ArrayList<>();
while (true){
list.clear();
//消费者拉取数据
ConsumerRecords<String,String> records = consumer.poll(Duration.ofMillis(100));
//循环读取
for (ConsumerRecord<String, String> record : records){
System.out.println(num);
//分割数据
String[] train = record.value().split(",");
//user,event,invited,timestamp,interested,not_interested
//'eu'
Random random = new Random();
int num = random.nextInt(10);
//指定rowkey
Put put = new Put((train[0]+train[1]+num).getBytes());
//指定列族,列限定符和具体的数据
put.addColumn("eu".getBytes(),"user".getBytes(),train[0].getBytes());
put.addColumn("eu".getBytes(),"event".getBytes(),train[1].getBytes());
put.addColumn("eu".getBytes(),"invited".getBytes(),train[2].getBytes());
put.addColumn("eu".getBytes(),"timestamp".getBytes(),train[3].getBytes());
put.addColumn("eu".getBytes(),"interested".getBytes(),train[4].getBytes());
put.addColumn("eu".getBytes(),"not_interested".getBytes(),train[5].getBytes());
list.add(put);
}
if (list.size()>0){
num+=list.size();
mutator.mutate(list);
}
}
} catch (IOException e) {
e.printStackTrace();
}finally {
try {
if (connection!=null){
connection.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
}