Kafka数据写入Hbase

本篇文章将使用kafka将数据写入到hbase中

一、使用的包

 <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka-clients</artifactId>
      <version>3.0.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.kafka</groupId>
      <artifactId>kafka_2.12</artifactId>
      <version>2.0.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-client</artifactId>
      <version>1.2.0</version>
    </dependency>
    <dependency>
      <groupId>org.apache.hbase</groupId>
      <artifactId>hbase-server</artifactId>
      <version>1.2.0</version>
    </dependency>
 

二、 具体写入流程

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Properties;
import java.util.Random;


public class TrainToHb {
    //全局的静态变量用来计数
    static long num = 0;
    public static void main(String[] args) {
        //消费者的配置
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"hadoop102:9092");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        prop.put(ConsumerConfig.GROUP_ID_CONFIG,"user_group4");
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");
        //创建消费者并指定要消费的主题
        KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(prop);
        consumer.subscribe(Collections.singleton("train"));

        //连接hbase
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum","hadoop102");
        conf.set("hbase.zookeeper.property.clientPort","2181");
        conf.set("hbase.rootdir","hdfs://hadoop102:8020/hbase");

        Connection connection = null;

        try {
            //连接Hbase
            connection = ConnectionFactory.createConnection(conf);
            //确定Hbase的命名空间和表名
            TableName tableName = TableName.valueOf("events_db:train");
            //如果失败
            BufferedMutator.ExceptionListener listener = new BufferedMutator.ExceptionListener(){
                @Override
                public void onException(RetriesExhaustedWithDetailsException e, BufferedMutator bufferedMutator) throws RetriesExhaustedWithDetailsException {
                    int nums = e.getNumExceptions();
                    for (int i = 0; i < nums; i++) {
                        System.out.println("失败，保存 " + e.getRow(i) + ".");
                    }
                }
            };
            //采用mutator高效写入hbase
            BufferedMutatorParams params = new BufferedMutatorParams(tableName);
            params.writeBufferSize(1024*1024);
            params.listener(listener);
            BufferedMutator mutator = connection.getBufferedMutator(params);

            ArrayList<Put> list = new ArrayList<>();

            while (true){
                list.clear();
                //消费者拉取数据
                ConsumerRecords<String,String> records = consumer.poll(Duration.ofMillis(100));
                //循环读取
                for (ConsumerRecord<String, String> record : records){
                    System.out.println(num);
                    //分割数据
                    String[] train = record.value().split(",");
                    //user,event,invited,timestamp,interested,not_interested
                    //'eu'
                    Random random = new Random();
                    int num = random.nextInt(10);
                    //指定rowkey
                    Put put = new Put((train[0]+train[1]+num).getBytes());
                    //指定列族，列限定符和具体的数据
                    put.addColumn("eu".getBytes(),"user".getBytes(),train[0].getBytes());
                    put.addColumn("eu".getBytes(),"event".getBytes(),train[1].getBytes());
                    put.addColumn("eu".getBytes(),"invited".getBytes(),train[2].getBytes());
                    put.addColumn("eu".getBytes(),"timestamp".getBytes(),train[3].getBytes());
                    put.addColumn("eu".getBytes(),"interested".getBytes(),train[4].getBytes());
                    put.addColumn("eu".getBytes(),"not_interested".getBytes(),train[5].getBytes());
                    list.add(put);
                }
                if (list.size()>0){
                    num+=list.size();
                    mutator.mutate(list);
                }
            }

        } catch (IOException e) {
            e.printStackTrace();
        }finally {
            try {
                if (connection!=null){
                    connection.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    }
}