Kafka中的数据导入Hbase

Kafka中的数据导入Hbase

Hbase新建空表

start-hbase.sh
hbase shell
create_namespace 'events_db'
create 'events_db:user_friend', 'uf'

编写代码

public class UserFriendToHB {
    static int num =0;
    public static void main(String[] args) {
        //kafka消费端属性  配置
        Properties prop=new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG,"192.168.146.222:9092");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG,StringDeserializer.class);
        //设置是否自动提交，获取数据的状态，false 手动提交，true 自动提交
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG,"false");
        prop.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG,"1000");

        prop.put(ConsumerConfig.GROUP_ID_CONFIG,"user_friend_group");
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG,"earliest");   //earliest latest none

        KafkaConsumer<String,String> consumer = new KafkaConsumer<>(prop);
        consumer.subscribe(Collections.singleton("user_friends"));

        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.rootdir","hdfs://192.168.146.222:9000/tmp/hbase-root/hbase");
        conf.set("hbase.zookeeper.quorum","192.168.146.222");
        conf.set("hbase.zookeeper.property.clientPort","2181");


        try {
            Connection connection = ConnectionFactory.createConnection(conf);
            Table userFriendTable = connection.getTable(TableName.valueOf("events_db:user_friend"));

            while (true) {
                ConsumerRecords<String, String> poll = consumer.poll(100);
                List<Put> datas=new ArrayList<>();
                for (ConsumerRecord<String, String> record : poll) {
                    System.out.println(record.value().toString());
                    String[] split = record.value().toString().split(",");
                    Put put = new Put(Bytes.toBytes((split[0] + split[1]).hashCode()));
                    put.addColumn("uf".getBytes(),"userid".getBytes(),split[0].getBytes());
                    put.addColumn("uf".getBytes(),"friendid".getBytes(),split[1].getBytes());
                    datas.add(put);
                }
                num+=datas.size();
                System.out.println("----------num:"+num);
                userFriendTable.put(datas);

            }
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}