hbase java rest_Hbase Java Api操作

1 packagecom.vue.demo.utils;2

3

4 importorg.apache.hadoop.conf.Configuration;5 importorg.apache.hadoop.hbase.Cell;6 importorg.apache.hadoop.hbase.CompareOperator;7 importorg.apache.hadoop.hbase.NamespaceDescriptor;8 importorg.apache.hadoop.hbase.TableName;9 import org.apache.hadoop.hbase.client.*;10 import org.apache.hadoop.hbase.filter.*;11 importorg.apache.hadoop.hbase.util.Bytes;12 importorg.apache.hadoop.hbase.client.Result;13 importorg.slf4j.Logger;14 importorg.slf4j.LoggerFactory;15

16 importjava.io.Closeable;17 importjava.io.IOException;18 importjava.util.ArrayList;19 importjava.util.HashMap;20 importjava.util.List;21 importjava.util.Map;22

23 /**

24 *@authoryangwj25 * @date 2020/9/7 10:0926 * @desc27 */

28 public class HBaseHelper implementsCloseable {29

30 private static final Logger log = LoggerFactory.getLogger(HBaseHelper.class);31

32 private Configuration configuration = null;33 private Connection connection = null;34 private Admin admin = null;35

36 /**构造方法*/

37 private HBaseHelper(Configuration configuration) throwsIOException {38 this.configuration =configuration;39 this.connection = ConnectionFactory.createConnection(this.configuration);40 admin = this.connection.getAdmin();41 }42

43 /**获取对象*/

44 public static HBaseHelper getHBaseHelper(Configuration configuration) throwsIOException {45 return newHBaseHelper(configuration);46 }47

48 @Override49 public void close() throwsIOException {50 admin.close();51 connection.close();52 }53

54 publicConnection getConnection() {55 returnconnection;56 }57

58 publicConfiguration getConfiguration() {59 returnconfiguration;60 }61

62 /**创建命名空间:数据隔离*/

63 public voidcreateNamespace(String namespace) {64 try{65 NamespaceDescriptor nd =NamespaceDescriptor.create(namespace).build();66 admin.createNamespace(nd);67 } catch(Exception e) {68 log.error("[createNamespace]Error: {}",e.getMessage());69 }70 }71

72 /**删除命名空间*/

73 public void dropNamespace(String namespace, booleanforce) {74 try{75 if(force) {76 TableName[] tableNames =admin.listTableNamesByNamespace(namespace);77 for(TableName name : tableNames) {78 admin.disableTable(name);79 admin.deleteTable(name);80 }81 }82 } catch(Exception e) {83 log.error("[dropNamespace]Error: {}",e.getMessage());84 }85 try{86 admin.deleteNamespace(namespace);87 } catch(IOException e) {88 log.error("[dropNamespace]Error: {}",e.getMessage());89 }90 }91

92 /**判断是否存在表*/

93 public booleanexistsTable(String table)94 throwsIOException {95 returnexistsTable(TableName.valueOf(table));96 }97

98 public booleanexistsTable(TableName table)99 throwsIOException {100 returnadmin.tableExists(table);101 }102

103 /**

104 * 创建表105 *@paramtable 表名106 *@paramcolfams 列簇107 *@throwsIOException108 */

109 public voidcreateTable(String table, String... colfams)110 throwsIOException {111 createTable(TableName.valueOf(table), 1, null, colfams);112 }113

114 public voidcreateTable(TableName table, String... colfams)115 throwsIOException {116 createTable(table, 1, null, colfams);117 }118

119 public void createTable(String table, intmaxVersions, String... colfams)120 throwsIOException {121 createTable(TableName.valueOf(table), maxVersions, null, colfams);122 }123

124 /**

125 * 创建表126 *@paramtable 表名127 *@parammaxVersions timestamp128 *@paramcolfams 列簇129 *@throwsIOException130 */

131 public void createTable(TableName table, intmaxVersions, String... colfams)132 throwsIOException {133 createTable(table, maxVersions, null, colfams);134 }135

136

137 public void createTable(String table, byte[][] splitKeys, String... colfams)138 throwsIOException {139 createTable(TableName.valueOf(table), 1, splitKeys, colfams);140 }141

142 /**

143 * 创建表144 *@paramtable 表名145 *@parammaxVersions timestamp146 *@paramsplitKeys147 *@paramcolfams 列簇148 *@throwsIOException149 */

150 public void createTable(TableName table, int maxVersions, byte[][] splitKeys,151 String... colfams)152 throwsIOException {153 //表描述器构造器

154 TableDescriptorBuilder tableDescriptorBuilder =TableDescriptorBuilder.newBuilder(table);155

156 //列族描述构造器

157 ColumnFamilyDescriptorBuilder cfDescBuilder;158

159 //列描述器

160 ColumnFamilyDescriptor cfDesc;161

162

163 for(String cf : colfams) {164 cfDescBuilder =ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(cf));165 cfDescBuilder.setMaxVersions(maxVersions);166 cfDesc =cfDescBuilder.build();167 tableDescriptorBuilder.setColumnFamily(cfDesc);168 }169 //获得表描述器

170 TableDescriptor tableDescriptor =tableDescriptorBuilder.build();171

172 if (splitKeys != null) {173 admin.createTable(tableDescriptor, splitKeys);174 } else{175 admin.createTable(tableDescriptor);176 }177 }178

179 /**禁用表*/

180 public void disableTable(String table) throwsIOException {181 disableTable(TableName.valueOf(table));182 }183

184 public void disableTable(TableName table) throwsIOException {185 admin.disableTable(table);186 }187

188 /**删除表*/

189 public void dropTable(String table) throwsIOException {190 dropTable(TableName.valueOf(table));191 }192

193 public void dropTable(TableName table) throwsIOException {194 if(existsTable(table)) {195 if(admin.isTableEnabled(table)) {disableTable(table);}196 admin.deleteTable(table);197 }198 }199

200

201 public voidput(String table, String row, String fam, String qual,202 String val) throwsIOException {203 put(TableName.valueOf(table), row, fam, qual, val);204 }205

206 /**插入或更新单行*/

207 public voidput(TableName table, String row, String fam, String qual,208 String val) throwsIOException {209 Table tbl =connection.getTable(table);210 Put put = newPut(Bytes.toBytes(row));211 put.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual), Bytes.toBytes(val));212 tbl.put(put);213 tbl.close();214 }215

216 public void put(String table, String row, String fam, String qual, longts,217 String val) throwsIOException {218 put(TableName.valueOf(table), row, fam, qual, ts, val);219 }220

221 /**带时间戳插入或更新单行*/

222 public void put(TableName table, String row, String fam, String qual, longts,223 String val) throwsIOException {224 Table tbl =connection.getTable(table);225 Put put = newPut(Bytes.toBytes(row));226 put.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual), ts,227 Bytes.toBytes(val));228 tbl.put(put);229 tbl.close();230 }231

232 /**插入或者更新一个rowKey数据，一个Put里有一个rowKey，可能有多个列族和列名*/

233 public void put(String tableNameString, Put put) throwsIOException {234 TableName tableName =TableName.valueOf(tableNameString);235 Table table =connection.getTable(tableName);236 if (put != null && put.size() > 0) {237 table.put(put);238 }239 table.close();240 }241

242 public voidput(String table, String[] rows, String[] fams, String[] quals,243 long[] ts, String[] vals) throwsIOException {244 put(TableName.valueOf(table), rows, fams, quals, ts, vals);245 }246

247 /**用于测试数据*/

248 public voidput(TableName table, String[] rows, String[] fams, String[] quals,249 long[] ts, String[] vals) throwsIOException {250 Table tbl =connection.getTable(table);251 for(String row : rows) {252 Put put = newPut(Bytes.toBytes(row));253 for(String fam : fams) {254 int v = 0;255 for(String qual : quals) {256 String val = vals[v < vals.length ? v : vals.length - 1];257 long t = ts[v < ts.length ? v : ts.length - 1];258 System.out.println("Adding: " + row + " " + fam + " " + qual +

259 " " + t + " " +val);260 put.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual), t,261 Bytes.toBytes(val));262 v++;263 }264 }265 tbl.put(put);266 }267 tbl.close();268 }269

270 public voiddump(String table, String[] rows, String[] fams, String[] quals)271 throwsIOException {272 dump(TableName.valueOf(table), rows, fams, quals);273 }274

275 /**测试dump数据*/

276 public voiddump(TableName table, String[] rows, String[] fams, String[] quals)277 throwsIOException {278 Table tbl =connection.getTable(table);279 List gets = new ArrayList();280 for(String row : rows) {281 Get get = newGet(Bytes.toBytes(row));282 get.readAllVersions();283 if (fams != null) {284 for(String fam : fams) {285 for(String qual : quals) {286 get.addColumn(Bytes.toBytes(fam), Bytes.toBytes(qual));287 }288 }289 }290 gets.add(get);291 }292 org.apache.hadoop.hbase.client.Result[] results =tbl.get(gets);293 for(org.apache.hadoop.hbase.client.Result result : results) {294 for(Cell cell : result.rawCells()) {295 System.out.println("Cell: " + cell +

296 ", Value: " +Bytes.toString(cell.getValueArray(),297 cell.getValueOffset(), cell.getValueLength()));298 }299 }300 tbl.close();301 }302

303

304 public void dump(String table) throwsIOException {305 dump(TableName.valueOf(table));306 }307

308 public void dump(TableName table) throwsIOException {309 try(310 Table t =connection.getTable(table);311 ResultScanner scanner = t.getScanner(newScan());312 ) {313 for(org.apache.hadoop.hbase.client.Result result : scanner) {314 dumpResult(result);315 }316 }317 }318

319 /**从Cell取Array要加上位移和长度，不然数据不正确*/

320 public voiddumpResult(Result result) {321 for(Cell cell : result.rawCells()) {322 System.out.println("Cell: " + cell +

323 ", Value: " +Bytes.toString(cell.getValueArray(),324 cell.getValueOffset(), cell.getValueLength()));325 }326 }327

328 public void dumpCells(String key, Listlist) {329 for(Cell cell : list) {330 String columnFamily =Bytes.toString(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength());331 String columnName =Bytes.toString(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength());332 String value =Bytes.toString(cell.getValueArray(), cell.getValueOffset(), cell.getValueLength());333 System.out.printf("[key:%s]\t[family:%s] [column:%s] [value:%s]\n",334 key, columnFamily, columnName, value);335 }336 }337

338

339 /**批量插入数据,list里每个map就是一条数据，并且按照rowKey columnFamily columnName columnValue放入map的key和value*/

340 public void bulkInsert(String tableNameString, List> list) throwsIOException {341 Table table =connection.getTable(TableName.valueOf(tableNameString));342 List puts = new ArrayList();343 if (list != null && list.size() > 0) {344 for (Mapmap : list) {345 Put put = new Put(Bytes.toBytes(map.get("rowKey").toString()));346 put.addColumn(Bytes.toBytes(map.get("columnFamily").toString()),347 Bytes.toBytes(map.get("columnName").toString()),348 Bytes.toBytes(map.get("columnValue").toString()));349 puts.add(put);350 }351 }352 table.put(puts);353 table.close();354 }355

356 /**批量插入*/

357 public void bulkInsert2(String tableNameString, List puts) throwsIOException {358 Table table =connection.getTable(TableName.valueOf(tableNameString));359 if (puts != null && puts.size() > 0) {360 table.put(puts);361 }362 table.close();363 }364

365 /**根据rowKey删除所有行数据*/

366 public void deleteByKey(String tableNameString, String rowKey) throwsIOException {367 Table table =connection.getTable(TableName.valueOf(tableNameString));368 Delete delete = newDelete(Bytes.toBytes(rowKey));369

370 table.delete(delete);371 table.close();372 }373

374 /**根据rowKey和列族删除所有行数据*/

375 public void deleteByKeyAndFamily(String tableNameString, String rowKey, String columnFamily) throwsIOException {376 Table table =connection.getTable(TableName.valueOf(tableNameString));377 Delete delete = newDelete(Bytes.toBytes(rowKey));378 delete.addFamily(Bytes.toBytes(columnFamily));379

380 table.delete(delete);381 table.close();382 }383

384 /**根据rowKey、列族删除多个列的数据*/

385 public voiddeleteByKeyAndFC(String tableNameString, String rowKey,386 String columnFamily, List columnNames) throwsIOException {387 Table table =connection.getTable(TableName.valueOf(tableNameString));388 Delete delete = newDelete(Bytes.toBytes(rowKey));389 for(String columnName : columnNames) {390 delete.addColumns(Bytes.toBytes(columnFamily), Bytes.toBytes(columnName));391 }392 table.delete(delete);393 table.close();394 }395

396

397 /**根据rowkey，获取所有列族和列数据*/

398 public List getRowByKey(String tableNameString, String rowKey) throwsIOException {399 Table table =connection.getTable(TableName.valueOf(tableNameString));400

401 Get get = newGet(Bytes.toBytes(rowKey));402

403 org.apache.hadoop.hbase.client.Result result =table.get(get);404

405 //Cell[] cells = result.rawCells();

406 List list =result.listCells();407 table.close();408 returnlist;409 }410

411 /**根据rowKey，family,qualifier获取列值*/

412 public List getRowByKeyAndColumn(String tableNameString, String rowKey, String cf, String clName) throwsIOException {413 Table table =connection.getTable(TableName.valueOf(tableNameString));414 Get get = newGet(Bytes.toBytes(rowKey));415 get.addColumn(Bytes.toBytes(cf), Bytes.toBytes(clName));416

417 org.apache.hadoop.hbase.client.Result result =table.get(get);418 List list =result.listCells();419 table.close();420 returnlist;421 }422

423 /**根据rowkey，获取所有列族和列数据*/

424 public Map> getRowByKeys(String tableNameString, String... rowKeys) throwsIOException {425 Table table =connection.getTable(TableName.valueOf(tableNameString));426

427 List gets = new ArrayList<>();428 for(String rowKey : rowKeys) {429 Get get = newGet(Bytes.toBytes(rowKey));430 gets.add(get);431 }432

433 org.apache.hadoop.hbase.client.Result[] results =table.get(gets);434

435 Map> map = new HashMap<>();436 for(org.apache.hadoop.hbase.client.Result res : results) {437 map.put(Bytes.toString(res.getRow()), res.listCells());438 }439

440 table.close();441 returnmap;442 }443

444 private Map> formatToMap(String tableNameString,Scan scan) throwsIOException{445 //确保table和scanner被释放

446 try (Table table =connection.getTable(TableName.valueOf(tableNameString));447 ResultScanner scanner =table.getScanner(scan);448 ) {449 Map> map = new HashMap<>();450 for(org.apache.hadoop.hbase.client.Result result : scanner) {451 map.put(Bytes.toString(result.getRow()), result.listCells());452 }453 returnmap;454 }455 }456

457 /**根据rowKey过滤数据，rowKey可以使用正则表达式458 * 返回rowKey和Cells的键值对459 **/

460 public Map> filterByRowKeyRegex(String tableNameString, String rowKey, CompareOperator operator) throwsIOException {461

462 Scan scan = newScan();463 //使用正则

464 RowFilter filter = new RowFilter(operator, newRegexStringComparator(rowKey));465

466 //包含子串匹配,不区分大小写。467 //RowFilter filter = new RowFilter(operator,new SubstringComparator(rowKey));

468

469 scan.setFilter(filter);470

471 returnformatToMap(tableNameString,scan);472 }473

474 //根据列族，列名，列值(支持正则)查找数据475 //返回值：如果查询到值，会返回所有匹配的rowKey下的各列族、列名的所有数据(即使查询的时候这些列族和列名并不匹配)

476 public Map>filterByValueRegex(String tableNameString, String family, String colName,477 String value, CompareOperator operator) throwsIOException {478 Scan scan = newScan();479

480 //正则匹配

481 SingleColumnValueFilter filter = newSingleColumnValueFilter(Bytes.toBytes(family),482 Bytes.toBytes(colName), operator, newRegexStringComparator(value));483

484 //完全匹配485 //SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes(family),486 //Bytes.toBytes(colName),operator,Bytes.toBytes(value));487

488 //SingleColumnValueExcludeFilter排除列值489

490 //要过滤的列必须存在，如果不存在，那么这些列不存在的数据也会返回。如果不想让这些数据返回,设置setFilterIfMissing为true

491 filter.setFilterIfMissing(true);492 scan.setFilter(filter);493

494 returnformatToMap(tableNameString,scan);495 }496

497 //根据列名前缀过滤数据

498 public Map> filterByColumnPrefix(String tableNameString, String prefix) throwsIOException {499

500 //列名前缀匹配

501 ColumnPrefixFilter filter = newColumnPrefixFilter(Bytes.toBytes(prefix));502

503 //QualifierFilter 用于列名多样性匹配过滤504 //QualifierFilter filter = new QualifierFilter(CompareOperator.EQUAL,new SubstringComparator(prefix));505

506 //多个列名前缀匹配507 //MultipleColumnPrefixFilter multiFilter = new MultipleColumnPrefixFilter(new byte[][]{});

508

509 Scan scan = newScan();510 scan.setFilter(filter);511

512 returnformatToMap(tableNameString,scan);513 }514

515 //根据列名范围以及列名前缀过滤数据

516 public Map>filterByPrefixAndRange(String tableNameString, String colPrefix,517 String minCol, String maxCol) throwsIOException {518

519 //列名前缀匹配

520 ColumnPrefixFilter filter = newColumnPrefixFilter(Bytes.toBytes(colPrefix));521

522 //列名范围扫描，上下限范围包括

523 ColumnRangeFilter rangeFilter = new ColumnRangeFilter(Bytes.toBytes(minCol), true,524 Bytes.toBytes(maxCol), true);525

526 FilterList filterList = newFilterList(FilterList.Operator.MUST_PASS_ALL);527 filterList.addFilter(filter);528 filterList.addFilter(rangeFilter);529

530 Scan scan = newScan();531 scan.setFilter(filterList);532

533 returnformatToMap(tableNameString,scan);534 }535

536

537 }