java读取kafka数据写入hbase

创建kafka消费者对象和hbase实例：

                Properties procounsumer = new Properties();
        //kafka是服务器地址
        procounsumer.put("zookeeper.connect", "dap03:2181/kafka");
        //定义消费者组
        procounsumer.put("group.id", "aaa");

        procounsumer.put("zookeeper.session.timeout.ms", "10000");
        procounsumer.put("zookeeper.sync.time.ms", "200");
        procounsumer.put("enable.auto.commit", "true");
        procounsumer.put("fetch.message.max.bytes", "10485760");
        procounsumer.put("serializer.class", "kafka.serializer.StringEncoder");
        procounsumer.put("rebalance.max.retries", "10");
        procounsumer.put("rebalance.backoff.ms", "20000");
        // hbase的配置
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.rootdir", "hdfs://dap03:9000/hbase");
        conf.set("hbase.zookeeper.quorum", "dap03");
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        return new ConsumerConfig(procounsumer);

下面的代码因为封装了一些参数，可能看不太明白，我也没有原版的代码了，将就着用，这里hbase的rowkey是加盐反转时间戳设计的

public class ETCToHbase4 implements Runnable{
    private static Logger logger = LoggerFactory.getLogger(ETCToHbase4.class);
    public static void main(String[] args) {
        new ETCToHbase4().run();
    }

    @Override
    public void run() {
        // 创建消费者对象
        ConsumerConnector connector = Consumer
                .createJavaConsumerConnector(new toKafkaUtils().getConsumer());
//        ConsumerConfig config = new ConsumerConfig(con);
//        // 消费者实例
//        ConsumerConnector connector = Consumer
//                .createJavaConsumerConnector(config);
        StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
        StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());


        Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
//        String topicName = "ods_gantry_info";
        // 指定tohbase的topic
        topicCountMap.put(Common.ALL_TOHBASE.get(Constants.MJXT4_TOPIC),1);
        Map<String, List<KafkaStream<String, String>>> messageStreams = connector.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
        List<KafkaStream<String, String>> streams = messageStreams.get(Common.ALL_TOHBASE.get(Constants.MJXT4_TOPIC));

        SimpleDateFormat sdf0 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
        SimpleDateFormat sdf1 = new SimpleDateFormat("yyyyMMddHHmmss");
        String row = null;
        // 设置登录用户
        System.setProperty(Constants.HADOOP_USER_NAME,Common.ALL_TOHBASE.get(Constants.HADOOP_USER_NAME));
        // 指定列族
        String columnFamilys = Common.ALL_TOHBASE.get(Constants.COLUMN_FAMILYS);
        int count = 0;
        Date startDate= new Date();
        try {
            // 指定hbase的表
            HTable table = new HTable(new toKafkaUtils().getconf(), Common.ALL_TOHBASE.get(Constants.MJXT4_HBASE_TABLE));
            if (null != streams && !streams.isEmpty()) {
                int c = 0;
                for (int i = 0; i < streams.size(); i++) {
                    ConsumerIterator<String, String> it = streams.get(i).iterator();
                    while (it.hasNext()) {
                        //创建List，保存Put类型的数据，需要方在while里定义，否则集合会无限增大，最终会导致OOM
                        List<Put> puts=new ArrayList<>();
                        String line = it.next().message();
                        System.out.println(line);

                        if (line.startsWith("{") && line.endsWith("}") && line.contains("licensecode")){
                            LinkedHashMap json = JSON.parseObject(line, LinkedHashMap.class);
                            JSONObject jsonObject = new JSONObject(true);

                            jsonObject.putAll(json);
                            System.out.println(jsonObject);
//                            row = new StringBuilder(jsonObject.getString("licensecode")).reverse().toString();
                            row=new StringBuilder(jsonObject.getString("licensecode")).reverse()+"_"+sdf1.format(sdf0.parse(jsonObject.getString("pictime").replace("T", " ")));
                            Put put = new Put(Bytes.toBytes(row));
                            for (String key : jsonObject.keySet()) {
                                put.addColumn(Bytes.toBytes(columnFamilys), Bytes.toBytes(key), Bytes.toBytes(jsonObject.get(key).toString()));
                            }
                            puts.add(put);
                            // 将集合的数据传入hbase表
                            count++;
                            System.out.println(count);
                            table.setAutoFlush(false);
                            Date endtime = new Date();
                            if (count>=200){
                                table.flushCommits();
                                count = 0;
                                startDate=new Date();
                                logger.info("数据到200 mjxt4刷新一次");
                            }
                            if (endtime.getTime() -startDate.getTime() >= 10000){
                                table.flushCommits();
                                startDate=endtime;
                                logger.info("10秒钟 mjxt4刷新一次");
                            }
                            table.put(puts);
                        }else {
                            logger.info("json格式错误mjxt4");
                            continue;
                        }
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
}

这里是一部分的配置文件：