#组重置
kafka-consumer-groups.sh --bootstrap-server 192.168.64.210:9092 --group cm --reset-offsets --all-topics --to-earliest --execute
#进行行数据统计
#进入hbase shell
count 'events'
# 在数据库外
$HBASE_HOME/bin/hbase org.apache.hadoop.hbase.mapreduce.RowCounter 'events'
修改 group id
cd /opt/soft/kafka200/config/
vim consumer.properties
#查看topic
kafka-topics.sh --zookeeper 192.168.64.210:2181 --list
#启动zookeeper
zkServer.sh start
#启动kafka
start-all.sh
kafka-server-start.sh /opt/soft/kafka200/config/server.properties
#启动hbase
start-hbase.sh
#监控kafka
kafka-console-consumer.sh --bootstrap-server 192.168.64.200:9092 --topic events
#启动mapreduce
hbase shell
count event
#行
hbase org.apache.hadoop.hbase.mapreduce.RowCounter 'userfriends'
#linux复制命令
#复制
yy
#粘贴
p
#退出hdoop安全模式
hadoop dfsadmin -safemode leave
#hbase删除
disable 'eventAttendees'
drop 'eventAttendees'
#hbase 创建表
create 'eventAttendees' 'base'
count 'eventAttendees'
#配置zookeeper 最大超时时间
vim zof.config
maxSessionTimeout=1800000
#IDEA代码格式化快捷键
Ctrl+Alt+L
简直不要太好用!!!
#不用框架直接读写到hbase
public class Kth {
static int i = 0;
static Connection conn;
static {
try {
Configuration cnf = HBaseConfiguration.create();
cnf.set("hbase.zookeeper.quorum", "192.168.64.210:2181");
conn = ConnectionFactory.createConnection(cnf);
} catch (IOException e) {
e.printStackTrace();
}
}
public static void main(String[] args) {
Properties prop = new Properties();
prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.64.210:9092");
prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
prop.put(ConsumerConfig.GROUP_ID_CONFIG, "cm");
prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(prop);
consumer.subscribe(Arrays.asList(new String[]{"user_friends_raw"}));
while (true) {
ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(10));
System.out.println("获取数据:" + records.count() + "===========" + (++i));
List<Put> puts = new ArrayList<>();
for (ConsumerRecord<String, String> record : records) {
//调用hbase写入数据
String[] lines = record.value().split(",", -1);
if (lines.length > 1) {
String[] fids = lines[1].split(" ");
for (String fid : fids) {
Put put = new Put((lines[0] + "-" + fid).getBytes());
put.addColumn("base".getBytes(), "userid".getBytes(), lines[0].getBytes());
put.addColumn("base".getBytes(), "friendid".getBytes(), fid.getBytes());
puts.add(put);
}
}
}
// //找到表
try {
Table htable = conn.getTable(TableName.valueOf("userfriends"));
htable.put(puts);
} catch (IOException e) {
e.printStackTrace();
}
//清空一下list集合
puts.clear();
// consumer.commitAsync();
consumer.commitSync();
}
}
}
#浏览器访问hadoop
192.168.64.210:50070