HBase API 实战

最新推荐文章于 2023-03-08 16:42:38 发布
最新推荐文章于 2023-03-08 16:42:38 发布
阅读量526 收藏
点赞数
分类专栏： hbase 文章标签： hbase
版权声明：本文为博主原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接和本声明。
本文链接：https://blog.csdn.net/mlljava1111/article/details/48662717
版权

BulkLoad

package com.chinahadoop.testbasicapi;

import java.io.IOException;
import java.util.Arrays;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.HFileOutputFormat;
import org.apache.hadoop.hbase.mapreduce.KeyValueSortReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

@SuppressWarnings("deprecation")
public class BulkLoad {

	public static class BulkLoadMapper extends Mapper<LongWritable, Text, ImmutableBytesWritable, KeyValue>  {
		@Override
		protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
			String[] values = value.toString().split("\t");
			System.out.println(Arrays.toString(values));
			byte[] row = Bytes.toBytes(values[0]);
			ImmutableBytesWritable k = new ImmutableBytesWritable(row);
			KeyValue kvProtocol = new KeyValue(row, Bytes.toBytes("testfml"), null, values[1].getBytes());
			context.write(k, kvProtocol);
		}
	}

	public static void main(String[] args) throws IOException, InterruptedException, ClassNotFoundException {
		Configuration conf = HBaseConfiguration.create();
		Job job = new Job(conf, "BulkLoad");
		
		job.setJarByClass(BulkLoad.class);
		job.setOutputKeyClass(ImmutableBytesWritable.class);
		job.setOutputValueClass(KeyValue.class);
		job.setMapperClass(BulkLoadMapper.class);
		job.setReducerClass(KeyValueSortReducer.class);
		
//		job.setOutputFormatClass(HFileOutputFormat2.class);
		job.setOutputFormatClass(HFileOutputFormat.class);
		HTable table = new HTable(conf, "test_table");
		HFileOutputFormat.configureIncrementalLoad(job, table);
		FileInputFormat.addInputPath(job, new Path("/user/hbase/bulkloadfile2"));
		FileOutputFormat.setOutputPath(job, new Path("/user/hbase/bulkloadfile2res"));

		System.exit(job.waitForCompletion(true) ? 0 : 1);
	}
}

FixRepair 

package com.chinahadoop.testrepair;

import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;


public class FixRepair {

	public static void main(String[] args) throws Exception {
		Configuration conf = HBaseConfiguration.create();
		HTable table = new HTable(conf, Bytes.toBytes(".META."));
		Scan scan = new Scan();
		scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
		ResultScanner scanner = table.getScanner(scan);
		Result rs = null;
		Map<byte[], List<HRegionInfo>> startKeyToRegionInfos = new TreeMap<byte[], List<HRegionInfo>>(Bytes.BYTES_COMPARATOR);
		while ((rs = scanner.next()) != null) {
			HRegionInfo regionInfo = Writables.getHRegionInfo(
					rs.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
			if (args[0].equals(regionInfo.getTableDesc().getNameAsString())) {
				byte[] startKey = regionInfo.getStartKey();
				List<HRegionInfo> infos = startKeyToRegionInfos.get(startKey);
				if (infos == null) {
					infos = new LinkedList<HRegionInfo>();
					startKeyToRegionInfos.put(startKey, infos);
				}
				infos.add(regionInfo);
			}
		}
		HBaseAdmin ha = new HBaseAdmin(conf);
		FileSystem fs = FileSystem.get(conf);
		for (List<HRegionInfo> infos : startKeyToRegionInfos.values()) {
			if (infos.size() > 1) {
				HRegionInfo parentRegion = (Bytes.compareTo(infos.get(0).getEndKey(), infos.get(1).getEndKey()) > 0) ?
						infos.get(0) : infos.get(1);
				HRegionInfo subRegionInfo = (infos.get(0) == parentRegion) ? infos.get(1) : infos.get(0);
				List<HRegionInfo> foundRegionInfos = new LinkedList<HRegionInfo>();
				foundRegionInfos.add(subRegionInfo);
				while (Bytes.compareTo(subRegionInfo.getEndKey(), parentRegion.getEndKey()) <= 0) {
					List<HRegionInfo> next = startKeyToRegionInfos.get(subRegionInfo.getEndKey());
					if (next == null) {
						break;
					}
					foundRegionInfos.add(next.get(0));
					subRegionInfo = next.get(0);
				}
				if (Bytes.compareTo(subRegionInfo.getEndKey(), parentRegion.getEndKey()) == 0) {
					for (HRegionInfo regionInfo : foundRegionInfos) {
						regionInfo.setOffline(true);
						Put p = new Put(regionInfo.getRegionName());
						p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, 
								Writables.getBytes(regionInfo));
						table.put(p);
						table.flushCommits();
						ha.closeRegion(regionInfo.getRegionName());
					}
					Path parentPath = new Path("/hbase/" + args[0] + "/" + parentRegion.getEncodedName());
					for (HColumnDescriptor fmlDesc : parentRegion.getTableDesc().getFamilies()) {
						Path parentFmlPath = new Path(parentPath, fmlDesc.getNameAsString());
						for (HRegionInfo regionInfo : foundRegionInfos) {
							Path subPath = new Path("/hbase/" + args[0] + "/" + regionInfo.getEncodedName());
							Path subFmlPath = new Path(subPath, fmlDesc.getNameAsString());
							for (FileStatus storeFile : fs.listStatus(subFmlPath)) {
								Path storeFilePath = storeFile.getPath();
								if (!storeFilePath.getName().contains(".")) {
									fs.rename(storeFilePath, new Path(parentFmlPath, storeFilePath.getName())); 
								}
							}
						}
					}
					ha.closeRegion(parentRegion.getRegionName());
				} else {
					throw new Exception("Can not find right subregion set of " + parentRegion.toString());
				}
			}
		}
	}

}


RepareMeta 

package com.chinahadoop.testrepair;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Writables;

public class RepareMeta {

	public static void main(String[] args) throws IOException {
		String regionFilePath = args[0];
		Configuration conf = HBaseConfiguration.create();
		HTable table = new HTable(conf, Bytes.toBytes(".META.")); 
		FileSystem fs = FileSystem.get(conf);
		FSDataInputStream in = fs.open(new Path(regionFilePath));
		HRegionInfo regionInfo = new HRegionInfo();
		regionInfo.readFields(in);
		System.out.println(regionInfo.toString());
		Put p = new Put(regionInfo.getRegionName());
		p.add(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER, 
				Writables.getBytes(regionInfo));
		table.put(p);
		table.close();
		in.close();
		fs.close();
	}
	
}

TestAdvanceApi 

package com.chinahadoop.testbasicapi;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.util.Bytes;

public class TestAdvanceApi {

	public static void main(String[] args) throws IOException {
		Configuration conf = HBaseConfiguration.create();
		HTable table = new HTable(conf, "test_table");
		Scan scan = new Scan();
		scan.setFilter(new RowFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("test_key3"))));
		ResultScanner scanner = table.getScanner(scan);
		Result resultInScan = null;
		while ((resultInScan = scanner.next()) != null) {
			String key = Bytes.toString(resultInScan.getRow());
			String value = Bytes.toString(resultInScan.getValue(Bytes.toBytes("testfml"), null));
			System.out.println("key:" + key);
			System.out.println("value:" + value); 
		}
		scanner.close();
		table.incrementColumnValue(Bytes.toBytes("test_key3"), Bytes.toBytes("testfml"), Bytes.toBytes("testcounter"), 1L);
		Increment icr = new Increment(Bytes.toBytes("test_key3"));
		icr.addColumn(Bytes.toBytes("testfml"), Bytes.toBytes("testcounter"), 1L);
		table.increment(icr);
		Get get = new Get(Bytes.toBytes("test_key3"));
		get.addColumn(Bytes.toBytes("testfml"), Bytes.toBytes("testcounter"));
		Result result = table.get(get);
		String key = Bytes.toString(result.getRow());
		long value = Bytes.toLong(result.getValue(Bytes.toBytes("testfml"), Bytes.toBytes("testcounter")));
		System.out.println("key:" + key);
		System.out.println("value:" + value); 
		table.close();
	}
	
}


TestMrApi 

package com.chinahadoop.testbasicapi;

import java.io.IOException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableReducer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.mapreduce.Job;

public class TestMrApi {
	public static void main(String[] args) throws Exception {
		Configuration conf = HBaseConfiguration.create();
		Job job = createSubmittableJob(conf);
		if (job != null) {
			System.exit(job.waitForCompletion(true) ? 0 : 1);
		}
	}
	static class TestTableMap extends
			TableMapper<ImmutableBytesWritable, ImmutableBytesWritable> {
		@Override
		public void map(ImmutableBytesWritable row, Result result,
				Context context) throws IOException, InterruptedException {
			for (Cell cell : result.rawCells()) {
				context.write(new ImmutableBytesWritable(CellUtil.cloneRow(cell)), 
						new ImmutableBytesWritable(CellUtil.cloneValue(cell)));
			}
		}
	}
	static class TestTableReducer extends
			TableReducer<ImmutableBytesWritable, ImmutableBytesWritable, ImmutableBytesWritable> {
		protected void reduce(ImmutableBytesWritable key, Iterable<ImmutableBytesWritable> values, 
				Context context) throws IOException, InterruptedException {
			for(ImmutableBytesWritable value: values) {
				Put p = new Put(key.get());
				p.add(Bytes.toBytes("testfml"), Bytes.toBytes("testcl"), value.get());
				context.write(key, p);
			}
		}
	}
	private static Job createSubmittableJob(Configuration conf) throws IOException {
	    conf.set(TableInputFormat.INPUT_TABLE, "test_table");
	    conf.set(TableInputFormat.SCAN_COLUMNS, "testfml");
		@SuppressWarnings("deprecation")
		Job job = new Job(conf, "Scan_test_table");
		job.setJarByClass(TestMrApi.class);
		job.setInputFormatClass(TableInputFormat.class);
		job.setMapperClass(TestTableMap.class);
		job.setMapOutputKeyClass(ImmutableBytesWritable.class);
		job.setMapOutputValueClass(ImmutableBytesWritable.class);
		TableMapReduceUtil.initTableReducerJob("test_table4", TestTableReducer.class, job);
		return job;
	}
}


TestObserve 

package com.chinahadoop.testbasicapi;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;

public class TestObserve extends BaseRegionObserver {
	public static final Log LOG = LogFactory.getLog(TestObserve.class);
	private HTableInterface table2 = null;
	@Override
	public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e,
			final Put put, final WALEdit edit, final Durability durability)
			throws IOException {
		if (table2 == null) {
			synchronized (this) {
				if (table2 == null) {
					table2 = e.getEnvironment().getTable(TableName.valueOf("test_table2"));
				}
			}
		} 
		Put newPut = new Put(put.getRow());
		for(Map.Entry<byte [], List<Cell>> entry: put.getFamilyCellMap().entrySet()) {
			newPut.getFamilyCellMap().put(entry.getKey(), entry.getValue());
	    }
		put.setDurability(durability);
		table2.put(put);
	}
	@Override
	protected void finalize() throws Throwable {
		if (table2 != null) {
			try { 
				table2.close();
			} catch (Exception e) {
				LOG.error("Error when finalize", e);
			}
		}
		super.finalize();
	}
	
}



mllhxn
关注
  • 0
    点赞
  • 踩
  • 0
    收藏
    觉得还不错? 一键收藏
  • 0
    评论
专栏目录
Hbase+Spring boot实战分布式文件存储-第4章-HBase的优化策略
muyingmiao的专栏
09-07 768
4-1 HBase优化策略一：服务端优化策略 什么会导致HBase的性能下降1.JVM内存分配与GC回收策略 这个是大部分java应用程序都会遇到的问题，这里不再阐述2.与HBase运行机制相关的部分配置不合理 HBase写入时当memstore达到一定的大小会flush到磁盘保存成HFile，当HFile小文件太多会执行compact操作进行合并，对HBase来说，当每一个store只包含一个...
Hbase与JAVA API实验报告
weixin_44919491的博客
10-11 1355
一、安装HBase 在start-hbase.Sh中添加一行export JAVA_HOME=/usr。。。。。。 配置环境变量 使用vi命令进入~/.bashrc 更改配置 经检查hbase安装成功 配置Java环境变量 用vi编辑hbase-env.sh 二、编程实践 create Describe Put Get delete deleteall 三、JAVA实例 import org.apache.hadoop.conf.Configuration;import org.a
HBase+SpringBoot实战分布式文件存储
12-04
HBase+SpringBoot实战分布式文件存储 资源永久有效哦
HBaseAPI
树深时见鹿_ic的博客
10-24 560
HBase的API： HBaseUtils public class HBaseUtils { private static Logger logger = LoggerFactory.getLogger(HBaseUtils.class); public static Admin getAdmin() { Configuration conf = HBaseCo...
大数据技术之HBase（三）HBase API的使用
最新发布
five小点心的博客
03-08 587
将hbase中的jar包导入到hive中。二、修改Hive配置文件。
HBase Java API编程实例
大数据基础入门教程
12-17 1390
在本实例中，首先创建一个学生成绩表 scores，用来存储学生各门课程的考试成绩，然后向 scores 添加数据。 表 scores 的概念视图如图 1 所示，用学生的名字 name 作为行键，年级 grade 是一个只有一个列的列族，score 是一个列族，每一门课程都是 score 的一个列，如 english、math、Chinese 等。score 的列可以随时添加。 例如，后续学生又参加...
【HBase基础教程】5、HBase API访问
andieguo的专栏
03-05 4537
开发环境硬件环境：Centos 6.5 服务器4台（一台为Master节点，三台为Slave节点） 软件环境：Java 1.7.0_45、Eclipse Juno Service Release 2、hadoop-1.2.1、hbase-0.94.20。 1、Hbase API介绍 几个相关类与HBase数据模型之间的对应关系HbaseAdmin用于数据库的创建与删除，HbaseConfigura
HBase+Springboot实战分布式文件存储 笔记
深度
04-30 1297
本系统解决的问题 Hadoop文件系统不适合存储大量小文件，但如果我们的业务还是需要存储大量小文件，就需要另一种解决方案 1.HBase简介 HBase是一个分布式的，面向列的开源数据库，适合非结构化存储 HBase是基于Hadoop的核心HDFS系统进行数据存储 HBase可以存储超大数据并适合用来进行大数据的实时查询 HBase是解决大数据领域内实时查询问题的 ...
Hbase的API
北京小辉
05-23 1090
package cn.yc.com.base.data; /** * * ClassName: RecordBean * @Description: 实时访客Hbase表映射 * @author 余辉 * @date 2015-8-18 */ public class RecordBean { private String rowkey;
HBase + SpringBoot分布式文件存储实战(一)-简介
weixin_34355715的博客
06-14 380
技术分析前提 ...
Hbase API
开水烫鱼的博客
12-10 180
public class HbaseTest { //注意要配host,我第一次没配出现了RetriesExhaustedException: Can't get the locations public static Configuration conf; //获取配置信息 static { conf = HBaseConfiguration.c...
基于Hbase和SpringBoot的分布式HOS文件存储系统
击水三千里的专栏
03-31 948
本文详细剖析了HOS分层对象存储系统的内部结构，探讨了RowKey设计、seqId的巧妙应用以及表结构优化对于提升HBase性能和数据管理有效性的关键作用，揭秘如何在大数据环境中实现高效存取与管理。
（第6篇）大数据发展背后的强力推手——HBase分布式存储系统
chitan2235的专栏
03-22 161
分享一套今年最新Hadoop大数据教程和100道Hadoop大数据必会面试题。 因为链接经常被和谐，需要的朋友请加微信 ganshiyun666来获取最新下载链接，注明“OSC” 教程已帮助300+人成功转型Hadoop开发，90%起薪超过20K，工资比之前翻了一倍。 百度Ha...
HBase学习之路 （八）HBase大牛博客
weixin_30399155的博客
04-02 68
主要是记录一下链接 http://hbasefly.com 转载于:https://www.cnblogs.com/qingyunzong/p/8696845.html
HBase 基本API操作
liangzelei的博客
05-13 365
import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration;...
HBASE客户端的一些参数设置
ahu6741的博客
04-18 191
HConstants.ZOOKEEPER_QUORUM, HConstants.ZOOKEEPER_ZNODE_PARENT, HConstants.ZOOKEEPER_CLIENT_PORT,HConstants.ZOOKEEPER_RECOVERABLE_WAITTIME,HConstants.HBASE_CLIENT_PAUSE, HConstants.HBASE_CLIEN...
HBase Java API实战：Eclipse环境配置与示例代码
"本教程主要介绍如何使用HBase Java API进行编程实践，包括启动Hadoop和HBase服务，创建Java项目，导入必要的jar包，以及编写示例代码进行数据操作。" 在HBase的Java API编程实践中，首先需要确保Hadoop和HBase环境...
评论
添加红包

请填写红包祝福语或标题

个

红包个数最小为10个

元

红包金额最低5元

当前余额3.43元 前往充值 >
需支付：10.00元
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付元
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明：

1.余额是钱包充值的虚拟货币，按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载，可以购买VIP、付费专栏及课程。

余额充值