使用面向对象将Kafka数据导入hbase

pom导包

<dependency>
  <groupId>junit</groupId>
  <artifactId>junit</artifactId>
  <version>4.11</version>
  <scope>test</scope>
</dependency>
<dependency>
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka-clients</artifactId>
    <version>2.0.0</version>
</dependency>
<dependency>
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka_2.11</artifactId>
  <version>2.0.0</version>
</dependency>
<dependency>
  <groupId>org.apache.kafka</groupId>
  <artifactId>kafka-streams</artifactId>
  <version>2.0.0</version>
</dependency>
<dependency>
  <groupId>org.apache.hbase</groupId>
  <artifactId>hbase-client</artifactId>
  <version>2.4.0</version>
</dependency>
<dependency>
  <groupId>org.apache.hbase</groupId>
  <artifactId>hbase-server</artifactId>
  <version>2.4.0</version>
</dependency>
<dependency>
  <groupId>redis.clients</groupId>
  <artifactId>jedis</artifactId>
  <version>3.7.0</version>
</dependency>

 先写接口Iworker

package oop.handle.Write;

import java.io.IOException;

public interface Iworker {
    // 抽象方法，用于填充数据，抛出IOException异常
 abstract void fillData() throws IOException;
}

 IWrite接口

 

package oop.handle.Write;
import org.apache.kafka.clients.consumer.ConsumerRecords;

import java.io.IOException;

public interface IWrite {
    // 向ConsumerRecords中写入数据
   void write(ConsumerRecords<String,String> records) throws IOException;
}

 HbaseWorker类实现连接Kafka和导数据

 

package oop.handle.Write;

import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;

import java.io.IOException;
import java.time.Duration;
import java.util.Collections;
import java.util.Properties;

public class HbaseWorker implements Iworker {
    KafkaConsumer<String,String> consumer;
    IWrite write;
    // 构造函数，初始化KafkaConsumer和IWrite对象
    public HbaseWorker(String topicName,String consumerGroupName,IWrite write){
        this.write= write;
        // 创建Properties对象，用于配置KafkaConsumer
        Properties prop = new Properties();
        // 设置Kafka集群的地址
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.52.146:9092");
        // 设置key的反序列化类
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        // 设置value的反序列化类
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
        // 设置是否自动提交offset
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
        // 设置从何处开始消费
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");
        // 设置消费组的名称
        prop.put(ConsumerConfig.GROUP_ID_CONFIG,consumerGroupName);
        // 设置自动提交offset的时间间隔
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");
        // 创建KafkaConsumer对象
        consumer = new KafkaConsumer<>(prop);
        // 订阅指定的topic
        consumer.subscribe(Collections.singleton(topicName));
    }
    // 实现fillData方法，从Kafka中拉取数据并写入Hbase
    @Override
    public void fillData() throws IOException {
        while (true){
            // 从Kafka中拉取数据
            ConsumerRecords<String,String> records = consumer.poll(Duration.ofMillis(100));
            // 将数据写入Hbase
            write.write(records);
        }
    }
}

 HbaseWrite类，实现连接hbase

 

package oop.handle.Write;

import oop.handle.IParseRecord;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;
import org.apache.kafka.clients.consumer.ConsumerRecords;

import java.io.IOException;
import java.util.ArrayList;

public class HbaseWrite implements IWrite {
    BufferedMutator mutator;
    IParseRecord handler;
    // 构造函数，传入目标表名、处理记录的handler、主机名、端口、hdfs路径
    public HbaseWrite(String targetTableName, IParseRecord handler, String host, String port, String hdfspath)throws IOException {
        this.handler=handler;
        // 创建HBase配置
        Configuration conf = HBaseConfiguration.create();
        // 设置Zookeeper主机名
        conf.set("hbase.zookeeper.quorum",host);
        // 设置Zookeeper客户端端口
        conf.set("hbase.zookeeper.property.clientPort",port);
        // 设置HBase根目录
        conf.set("hbase.rootdir",hdfspath);
        // 创建HBase连接
        Connection connection = ConnectionFactory.createConnection(conf);
        // 获取目标表名
        TableName tableName = TableName.valueOf(targetTableName);
        // 创建BufferedMutator异常监听器
        BufferedMutator.ExceptionListener listener = new BufferedMutator.ExceptionListener() {
            @Override
            public void onException(RetriesExhaustedWithDetailsException e, BufferedMutator bufferedMutator) throws RetriesExhaustedWithDetailsException {
                // 获取异常数量
                int nums = e.getNumExceptions();
                // 遍历异常
                for (int i = 0; i < nums; i++) {
                    // 打印异常信息
                    System.out.println("失败 保存"+e.getRow(i)+".");
                }
            }
        };
        // 创建BufferedMutator参数
        BufferedMutatorParams params = new BufferedMutatorParams(tableName);
        // 设置写缓冲区大小
        params.writeBufferSize(1024*1024);
        // 设置异常监听器
        params.listener(listener);
        // 获取BufferedMutator
        BufferedMutator mutator = connection.getBufferedMutator(params);
    }
    // 构造函数，传入目标表名、处理记录的handler
    public HbaseWrite(String targetTableName, IParseRecord handler) throws IOException {
      // 调用另一个构造函数，传入默认的主机名、端口、hdfs路径
      this(targetTableName,handler,"192.168.52.146","2181","hdfs://localhost:9000/hbase");
}
    // 重写write方法，传入ConsumerRecords
    @Override
    public void write(ConsumerRecords<String, String> records) throws IOException {
        // 如果records不为空且handler不为空
        if (records.count()>0 && handler!=null){
            // 解析records
            ArrayList<Put>list=handler.parse(records);
            // 如果解析结果不为空
            if (list.size()>0){
                // 保存解析结果
                mutator.mutate(list);
            }
        }
    }
}

EventAttendeeToHb2启动类，调用类方法就可以了

 

package oop.handle.Write;

import oop.handle.EventAttendeeHandler;

import java.io.IOException;

public class EventAttendeeToHb2 {
    public static void main(String[] args) throws IOException {
        // 创建一个HbaseWrite对象，用于写入数据到Hbase数据库中
        IWrite write=new HbaseWrite("events_db:event_attendee",new EventAttendeeHandler());
        // 创建一个HbaseWorker对象，用于处理数据
        Iworker worker=new HbaseWorker("train","train_group1",write);
        // 填充数据
        worker.fillData();
    }
}

 使用面向对象可以在后续开发时更好进行特定修改和扩展