文章目录
package my.test.kafka_hbase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.StringDeserializer;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
public class UserFriendhb {
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.226.111:9092");
prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
prop.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000");
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
prop.put(ConsumerConfig.GROUP_ID_CONFIG, "aa99");
KafkaConsumer<String, String> consumer = new KafkaConsumer<>(prop);
consumer.subscribe(Collections.singletonList("train_1"));
Configuration config = HBaseConfiguration.create();
config.set("hbase.rootdir", "hdfs://192.168.226.111:9000/hbase");
config.set("hbase.zookeeper.quorum", "192.168.226.111");
config.set("hbase.zookeeper.property.clientPort", "2181");
try {
Connection connection = ConnectionFactory.createConnection(config);
Table table = connection.getTable(TableName.valueOf("events_db:train"));
while (true) {
ConsumerRecords<String, String> records = consumer.poll(100);
List<Put> putList = new ArrayList<>();
for (ConsumerRecord<String, String> record : records) {
System.out.println(record);
String[] infos = record.value().split(",");
Put put = new Put(Bytes.toBytes((infos[0] + infos[1]).hashCode()));
put.addColumn("eu".getBytes(), "user".getBytes(), infos[0].getBytes());
put.addColumn("eu".getBytes(), "event".getBytes(), infos[1].getBytes());
put.addColumn("eu".getBytes(), "invited".getBytes(), infos[2].getBytes());
put.addColumn("eu".getBytes(), "timestamp".getBytes(), infos[3].getBytes());
put.addColumn("eu".getBytes(), "interested".getBytes(), infos[4].getBytes());
put.addColumn("eu".getBytes(), "not_interested".getBytes(), infos[5].getBytes());
putList.add(put);
System.out.println("--------------");
}
table.put(putList);
table.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}