创建kafka消费者对象和hbase实例:
Properties procounsumer = new Properties();
//kafka是服务器地址
procounsumer.put("zookeeper.connect", "dap03:2181/kafka");
//定义消费者组
procounsumer.put("group.id", "aaa");
procounsumer.put("zookeeper.session.timeout.ms", "10000");
procounsumer.put("zookeeper.sync.time.ms", "200");
procounsumer.put("enable.auto.commit", "true");
procounsumer.put("fetch.message.max.bytes", "10485760");
procounsumer.put("serializer.class", "kafka.serializer.StringEncoder");
procounsumer.put("rebalance.max.retries", "10");
procounsumer.put("rebalance.backoff.ms", "20000");
// hbase的配置
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.rootdir", "hdfs://dap03:9000/hbase");
conf.set("hbase.zookeeper.quorum", "dap03");
conf.set("hbase.zookeeper.property.clientPort", "2181");
return new ConsumerConfig(procounsumer);
下面的代码因为封装了一些参数,可能看不太明白,我也没有原版的代码了,将就着用,这里hbase的rowkey是加盐反转时间戳设计的
public class ETCToHbase4 implements Runnable{
private static Logger logger = LoggerFactory.getLogger(ETCToHbase4.class);
public static void main(String[] args) {
new ETCToHbase4().run();
}
@Override
public void run() {
// 创建消费者对象
ConsumerConnector connector = Consumer
.createJavaConsumerConnector(new toKafkaUtils().getConsumer());
// ConsumerConfig config = new ConsumerConfig(con);
// // 消费者实例
// ConsumerConnector connector = Consumer
// .createJavaConsumerConnector(config);
StringDecoder keyDecoder = new StringDecoder(new VerifiableProperties());
StringDecoder valueDecoder = new StringDecoder(new VerifiableProperties());
Map<String, Integer> topicCountMap = new HashMap<String, Integer>();
// String topicName = "ods_gantry_info";
// 指定tohbase的topic
topicCountMap.put(Common.ALL_TOHBASE.get(Constants.MJXT4_TOPIC),1);
Map<String, List<KafkaStream<String, String>>> messageStreams = connector.createMessageStreams(topicCountMap, keyDecoder, valueDecoder);
List<KafkaStream<String, String>> streams = messageStreams.get(Common.ALL_TOHBASE.get(Constants.MJXT4_TOPIC));
SimpleDateFormat sdf0 = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
SimpleDateFormat sdf1 = new SimpleDateFormat("yyyyMMddHHmmss");
String row = null;
// 设置登录用户
System.setProperty(Constants.HADOOP_USER_NAME,Common.ALL_TOHBASE.get(Constants.HADOOP_USER_NAME));
// 指定列族
String columnFamilys = Common.ALL_TOHBASE.get(Constants.COLUMN_FAMILYS);
int count = 0;
Date startDate= new Date();
try {
// 指定hbase的表
HTable table = new HTable(new toKafkaUtils().getconf(), Common.ALL_TOHBASE.get(Constants.MJXT4_HBASE_TABLE));
if (null != streams && !streams.isEmpty()) {
int c = 0;
for (int i = 0; i < streams.size(); i++) {
ConsumerIterator<String, String> it = streams.get(i).iterator();
while (it.hasNext()) {
//创建List,保存Put类型的数据,需要方在while里定义,否则集合会无限增大,最终会导致OOM
List<Put> puts=new ArrayList<>();
String line = it.next().message();
System.out.println(line);
if (line.startsWith("{") && line.endsWith("}") && line.contains("licensecode")){
LinkedHashMap json = JSON.parseObject(line, LinkedHashMap.class);
JSONObject jsonObject = new JSONObject(true);
jsonObject.putAll(json);
System.out.println(jsonObject);
// row = new StringBuilder(jsonObject.getString("licensecode")).reverse().toString();
row=new StringBuilder(jsonObject.getString("licensecode")).reverse()+"_"+sdf1.format(sdf0.parse(jsonObject.getString("pictime").replace("T", " ")));
Put put = new Put(Bytes.toBytes(row));
for (String key : jsonObject.keySet()) {
put.addColumn(Bytes.toBytes(columnFamilys), Bytes.toBytes(key), Bytes.toBytes(jsonObject.get(key).toString()));
}
puts.add(put);
// 将集合的数据传入hbase表
count++;
System.out.println(count);
table.setAutoFlush(false);
Date endtime = new Date();
if (count>=200){
table.flushCommits();
count = 0;
startDate=new Date();
logger.info("数据到200 mjxt4刷新一次");
}
if (endtime.getTime() -startDate.getTime() >= 10000){
table.flushCommits();
startDate=endtime;
logger.info("10秒钟 mjxt4刷新一次");
}
table.put(puts);
}else {
logger.info("json格式错误mjxt4");
continue;
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
这里是一部分的配置文件: