Hbase常用命令 kafka组重置

kjshuan

已于 2022-08-16 11:16:27 修改

阅读量516

点赞数 1

分类专栏： Java 大数据文章标签： hbase kafka 大数据

于 2022-08-13 15:47:29 首次发布

本文链接：https://blog.csdn.net/just_learing/article/details/126320521

版权

Java 同时被 2 个专栏收录

111 篇文章 6 订阅

订阅专栏

大数据

11 篇文章 0 订阅

订阅专栏

#组重置

kafka-consumer-groups.sh --bootstrap-server 192.168.64.210:9092 --group cm --reset-offsets --all-topics --to-earliest --execute

#进行行数据统计
#进入hbase shell
count 'events'
# 在数据库外
 $HBASE_HOME/bin/hbase org.apache.hadoop.hbase.mapreduce.RowCounter 'events'

修改 group id

cd /opt/soft/kafka200/config/
vim consumer.properties
在这里插入图片描述

#查看topic

 kafka-topics.sh --zookeeper 192.168.64.210:2181 --list

#启动zookeeper

zkServer.sh start

#启动kafka

start-all.sh

kafka-server-start.sh /opt/soft/kafka200/config/server.properties

#启动hbase

start-hbase.sh

#监控kafka

kafka-console-consumer.sh --bootstrap-server 192.168.64.200:9092 --topic events

#启动mapreduce

hbase shell
count event

#行
hbase org.apache.hadoop.hbase.mapreduce.RowCounter 'userfriends'

#linux复制命令

#复制
yy
#粘贴
p

#退出hdoop安全模式

hadoop dfsadmin -safemode leave

#hbase删除

disable 'eventAttendees'
drop 'eventAttendees'

#hbase 创建表

create 'eventAttendees' 'base'
count 'eventAttendees'

#配置zookeeper 最大超时时间

vim zof.config
maxSessionTimeout=1800000

#IDEA代码格式化快捷键

Ctrl+Alt+L
简直不要太好用！！！

#不用框架直接读写到hbase

public class Kth {

    static int i = 0;
    static Connection conn;

    static {
        try {
            Configuration cnf = HBaseConfiguration.create();
            cnf.set("hbase.zookeeper.quorum", "192.168.64.210:2181");
            conn = ConnectionFactory.createConnection(cnf);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        Properties prop = new Properties();
        prop.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "192.168.64.210:9092");
        prop.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
        prop.put(ConsumerConfig.GROUP_ID_CONFIG, "cm");
        prop.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        prop.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class.getName());
        prop.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, false);
        KafkaConsumer<String, String> consumer = new KafkaConsumer<String, String>(prop);
        consumer.subscribe(Arrays.asList(new String[]{"user_friends_raw"}));
        while (true) {

            ConsumerRecords<String, String> records = consumer.poll(Duration.ofSeconds(10));
            System.out.println("获取数据:" + records.count() + "===========" + (++i));
            List<Put> puts = new ArrayList<>();
            for (ConsumerRecord<String, String> record : records) {
                //调用hbase写入数据
                String[] lines = record.value().split(",", -1);
                if (lines.length > 1) {
                    String[] fids = lines[1].split(" ");
                    for (String fid : fids) {
                        Put put = new Put((lines[0] + "-" + fid).getBytes());
                        put.addColumn("base".getBytes(), "userid".getBytes(), lines[0].getBytes());
                        put.addColumn("base".getBytes(), "friendid".getBytes(), fid.getBytes());
                        puts.add(put);
                    }
                }
            }
//            //找到表
            try {
                Table htable = conn.getTable(TableName.valueOf("userfriends"));
                htable.put(puts);
            } catch (IOException e) {
                e.printStackTrace();
            }
            //清空一下list集合
            puts.clear();
//            consumer.commitAsync();
            consumer.commitSync();
        }
    }
}

在这里插入图片描述