编写自动化脚本
vim zhh.sh ============================= #shell脚本判断参 if [ $1 == "start" ];then echo '开启zookeeper\hadoop\hbase' /opt/soft/zk345/bin/zkServer.sh start /opt/soft/hadoop260/sbin/start-all.sh /opt/soft/hbase120/bin/start-hbase.sh else echo '关闭hbase\hadoop\zookeeper' /opt/soft/hbase120/bin/stop-hbase.sh /opt/soft/hadoop260/sbin/stop-all.sh /opt/soft/zk345/bin/zkServer.sh stop fi ============================= #授权 chmod 700 zhh.sh ls #启动 ./zhh.sh start #关闭 ./zhh.sh stop
一个表有n个列簇 一般建议不超过3个
进入数据库
hbase shell #版本 version #状态 status #查看 whoami list desc 'mydemo:userinfos' #插入数据(姓名张三 年纪40) put 'mydemo:userinfos','1','base:name','zhangsan' put 'mydemo:userinfos','1','base:age',40 #查询值 get 'mydemo:userinfos','1'
#批量插入
#首先退出数据库 exit hdfs dfs -mkdir -p /opt hdfs dfs -put /opt/hbase01.csv /opt #这里最好一句一句粘贴 插入数据 hbase org.apache.hadoop.hbase.mapreduce.ImportTsv -Dimporttsv.separator=, -Dimporttsv.columns="HBASE_ROW_KEY,base:name,base:age" 'mydemo:userinfos' /opt/hbase01.csv #进入数据库 hbase shell #查看数据 count 'mydemo:userinfos'
hbase常用命令
#扫描 scan 'mydemo:userinfos' #删除 get 'mydemo:userinfos','1' delete 'mydemo:userinfos','1',"base:age" #删除表 disable 'mydemo:userinfos' drop 'mydemo:userinfos' list #创建表 create "mydemo:userinfos","base" #分页扫描 scan 'mydemo:userinfos',{LIMIT=>3} scan 'mydemo:userinfos',{FILTER=>"ValueFilter(=,'substring:a')"} scan 'mydemo:userinfos',{FILTER=>"SingleColumnValueFilter('base','name',=,'binary:aahepo')"}
java编写代码
#1 导包 <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-client</artifactId> <version>1.2.0</version> </dependency>
#2 编写连接设置 public class App { public static void main( String[] args ) throws Exception { //配置zookeeper地址 Configuration cfg = HBaseConfiguration.create(); cfg.set("hbase.zookeeper.quorum","192.168.64.210:2181"); //获得数据库连接对象 Connection connection = ConnectionFactory.createConnection(cfg); //获取数据表 Table table = connection.getTable(TableName.valueOf("mydemo:userinfos")); //查询表中的数据 ResultScanner re = table.getScanner(new Scan()); //遍历循环数据 for (Result result : re) { String name = new String(result.getValue("base".getBytes(),"name".getBytes())); String age = new String(result.getValue("base".getBytes(), "age".getBytes())); System.out.println(name+"<============>"+age); } connection.close(); } }
#插入一条数据 public class App { public static void main( String[] args ) throws Exception { //配置zookeeper地址 Configuration cfg = HBaseConfiguration.create(); cfg.set("hbase.zookeeper.quorum","192.168.64.210:2181"); //获得数据库连接对象 Connection connection = ConnectionFactory.createConnection(cfg); //获取数据表 Table table = connection.getTable(TableName.valueOf("mydemo:userinfos")); //准备一个put命令 Put put=new Put("10000".getBytes()); put.addColumn("base".getBytes(),"name".getBytes(),"zhangsanfeng".getBytes()); put.addColumn("base".getBytes(),"age".getBytes(),"200".getBytes()); //插入一条数据 table.put(put); connection.close(); }
#查询一条消息 public class App { public static void main( String[] args ) throws Exception { //配置zookeeper地址 Configuration cfg = HBaseConfiguration.create(); cfg.set("hbase.zookeeper.quorum","192.168.64.210:2181"); //获得数据库连接对象 Connection connection = ConnectionFactory.createConnection(cfg); //获取数据表 Table table = connection.getTable(TableName.valueOf("mydemo:userinfos")); // //准备一个put命令 // Put put=new Put("10000".getBytes()); // put.addColumn("base".getBytes(),"name".getBytes(),"zhangsanfeng".getBytes()); // put.addColumn("base".getBytes(),"age".getBytes(),"200".getBytes()); // //插入一条数据 // table.put(put); //查询一条 Get get = new Get("10000".getBytes()); Result result = table.get(get); System.out.println(new String(result.getValue("base".getBytes(),"name".getBytes()))); connection.close(); } }
#批量插入 数据库查看 scan 'mydemo:userinfos',{FILTER=>"PrefixFilter('101')"} ====================== public class App { public static void main( String[] args ) throws Exception { //配置zookeeper地址 Configuration cfg = HBaseConfiguration.create(); cfg.set("hbase.zookeeper.quorum","192.168.64.210:2181"); //获得数据库连接对象 Connection connection = ConnectionFactory.createConnection(cfg); //获取数据表 Table table = connection.getTable(TableName.valueOf("mydemo:userinfos")); List<Put> puts = new ArrayList<>(); for (int i = 1010; i < 1020; i++) { Put put = new Put((i + "").getBytes()); put.addColumn("base".getBytes(),"name".getBytes(),("test"+i).getBytes()); puts.add(put); } table.put(puts); connection.close(); } }
springboot整合hbase
#1 导包 pom <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-shaded-client</artifactId> <version>1.2.0</version> </dependency>
#2 config配置 @Configuration public class HbaseConfig { @Value("${hbase.zookeeper.quorum}") private String zookeeper_url; @Bean public org.apache.hadoop.conf.Configuration hbaseConfiguration(){ org.apache.hadoop.conf.Configuration cfg= HBaseConfiguration.create(); cfg.set(HConstants.ZOOKEEPER_QUORUM,zookeeper_url); return cfg; } @Bean @Scope("prototype") public Connection getConnection() { Connection connection=null; try { connection = ConnectionFactory.createConnection(hbaseConfiguration()); } catch (IOException e) { e.printStackTrace(); } return connection; } @Bean public Supplier<Connection> hbaseConSupplier(){ return ()->{ return getConnection(); }; } }
#3 配置实体类 @Data @AllArgsConstructor @NoArgsConstructor @Builder public class Userinfos { private String rowkey; private String name; private String age; }
#4 配置service层 @Service public class HbaseService { @Resource private Connection hbasecon; public List<Userinfos> findAll(){ ArrayList<Userinfos> users = new ArrayList<>(); //获取表 try { Table tab = hbasecon.getTable(TableName.valueOf("mydemo:userinfos".getBytes())); Scan scan = new Scan(); ResultScanner results = tab.getScanner(scan); for (Result result : results) { String name=""; String age=""; if(result.containsColumn("base".getBytes(), "name".getBytes())){ name = new String(result.getValue("base".getBytes(), "name".getBytes())); } if(result.containsColumn("base".getBytes(), "age".getBytes())){ age = new String(result.getValue("base".getBytes(), "age".getBytes())); } Userinfos ui = Userinfos.builder().rowkey(new String(result.getRow())) .name(name) .age(age).build(); users.add(ui); } } catch (IOException e) { e.printStackTrace(); } return users; } }
#5 配置 yml server: port: 8081 spring: application: name: sbhbase hbase: zookeeper: quorum: 192.168.64.210:2181
#6 测试 @SpringBootTest class SbhbaseApplicationTests { @Resource private HbaseService hbaseService; @Test void contextLoads() { System.out.println(hbaseService.findAll()); } }
创建名空间
create_namespace 'towercrane' create 'towercrane:tchandler','base' exit
查看大数据
cd /opt/data ls cat event_attendees.csv | head -1 cat user_friends.csv | head -1