Hbase2.x协处理器的使用

最新推荐文章于 2022-03-11 15:32:27 发布

天地人-神君

最新推荐文章于 2022-03-11 15:32:27 发布

阅读量1.9k

点赞数 2

分类专栏： java hadoop 文章标签： HBASE JAVA 大数据

本文链接：https://blog.csdn.net/qq_41833259/article/details/92790187

版权

java 同时被 2 个专栏收录

19 篇文章 0 订阅

订阅专栏

hadoop

2 篇文章 0 订阅

订阅专栏

简介

HBASE的协处理器分为endpoint和observer，网上介绍很多，这里不再赘述，直接上代码。

创建Maven工程

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
	<modelVersion>4.0.0</modelVersion>
	<groupId>com.klcwqy.hbase</groupId>
	<artifactId>demo</artifactId>
	<version>0.0.1-SNAPSHOT</version>

	<dependencies>
		<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-client -->
		<dependency>
			<groupId>org.apache.hbase</groupId>
			<artifactId>hbase-client</artifactId>
			<version>2.0.1</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.apache.hbase/hbase-server -->
		<dependency>
		    <groupId>org.apache.hbase</groupId>
		    <artifactId>hbase-server</artifactId>
		    <version>2.0.1</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
		<dependency>
		    <groupId>com.google.protobuf</groupId>
		    <artifactId>protobuf-java</artifactId>
		    <version>2.6.1</version>
		</dependency>
		
	</dependencies>

	<build>
		<finalName>demo</finalName>
		<plugins>
			<plugin>
				<groupId>org.apache.maven.plugins</groupId>
				<artifactId>maven-compiler-plugin</artifactId>
				<version>3.1</version>
				<configuration>
					<source>1.8</source>
					<target>1.8</target>
					<encoding>UTF-8</encoding>
				</configuration>
			</plugin>
		</plugins>
	</build>
</project>

自定义Observer

Hbase2.x定义observer需要实现RegionObserver, RegionCoprocessor，这点与1.x版本不同。下例为向一张表插入数据之前，取出另一张表有用的数据，并插入之。

package com.klcwqy.hbase.demo.coprocessor.observers;

import java.io.IOException;
import java.util.Optional;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.coprocessor.RegionObserver;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WALEdit;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;


public class ReverseInfoObserver implements RegionObserver, RegionCoprocessor{
	
	private static final Logger logger = LoggerFactory.getLogger(ReverseInfoObserver.class);
	
	private static Configuration conf = null;
	private static Connection connection = null;
	private static Table order = null;
	private RegionCoprocessorEnvironment env = null;
	
	static{
		 conf = HBaseConfiguration.create();
		 conf.set("hbase.zookeeper.quorum", "slave1,slave2,slave3");
		 conf.set("hbase.zookeeper.property.clientPort", "2181");
		 try {
			connection = ConnectionFactory.createConnection(conf);
			order = connection.getTable(TableName.valueOf("order"));
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
	
	

	@Override
	public void start(CoprocessorEnvironment e) throws IOException {
		this.env = (RegionCoprocessorEnvironment) e;
	}



	@Override
	public void stop(CoprocessorEnvironment env) throws IOException {
		
	}

	/**
	 * 加入该方法，否则无法生效
	 */
	@Override
	public Optional<RegionObserver> getRegionObserver() {
		return Optional.of(this);
	}



	@Override
	public void prePut(ObserverContext<RegionCoprocessorEnvironment> c,Put put, WALEdit edit, Durability durability) throws IOException {
		logger.info("run ReverseInfoObserver............prePut...........................");
		try {
			System.out.println("---------------------------------------------------------------------");
			byte[] user = put.getRow();
			Cell cell = put.get(Bytes.toBytes("info"), Bytes.toBytes("order")).get(0);
			Put o_put = new Put(cell.getValueArray(),cell.getValueOffset(),cell.getValueLength());
			o_put.addColumn(Bytes.toBytes("info"), Bytes.toBytes("user"), user);
			order.put(o_put);
			order.close();
		} catch (IOException e) {
			logger.error(e.getMessage());
			throw e;
		}
	}

	
}

测试Observer

hbase加载协处理器分为静态和动态两种，此处使用动态加载。
1、禁用表

disable 'account'

2、注册协处理器

alter 'account' , METHOD =>'table_att','coprocessor'=>'hdfs://master:9000/test/hbase/coprocessor/d.jar|com.klcwqy.hbase.demo.coprocessor.observers.ReverseInfoObserver|1001|'

3、测试

put 'account','klcwqy','info:order','200'

4、结果
扫描order表，发现了一条数据：
klcwqy column=info:order, timestamp=1560868236903, value=200 测试成功

自定义Endpoint（本例带参数）

1、创建.proto

option java_package = "com.klcwqy.hbase.demo.coprocessor.endpoints.sum";
option java_outer_classname = "SumRows";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;

message SumRequest {
	required string family = 2;
	required string column = 3;
}

message SumResponse {
    required int64 count = 1 [default=0];
}

service SumRowService {
    rpc getSum(SumRequest)
        returns (SumResponse);
}

2、创建RegionCoprocessor的子类

package com.klcwqy.hbase.demo.coprocessor.endpoints.sum;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessor;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;

import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import com.klcwqy.hbase.demo.coprocessor.endpoints.sum.SumRows.SumRequest;
import com.klcwqy.hbase.demo.coprocessor.endpoints.sum.SumRows.SumResponse;

@SuppressWarnings("all")
public class GetSumEndpoint extends SumRows.SumRowService implements RegionCoprocessor{

	private RegionCoprocessorEnvironment rce = null;

	@Override
    public void stop(CoprocessorEnvironment env) throws IOException {
    	
    }

	@Override
    public void start(CoprocessorEnvironment env) throws IOException {
        rce = (RegionCoprocessorEnvironment) env;
    }
    
    @Override
    public Iterable<Service> getServices() {
        return Collections.singleton(this);
    }

	@Override
	public void getSum(RpcController controller, SumRequest request,RpcCallback<SumResponse> done) {
		//获取列族
		byte[] family = Bytes.toBytes(request.getFamily());
		byte[] column = Bytes.toBytes(request.getColumn());
		int count = this.getCount(family,column);
		//获取response
		SumResponse response = SumRows.SumResponse.newBuilder().setCount(count).build();
		done.run(response);
	}

	private int getCount(byte[] family, byte[] column){
		try {
			if (rce == null) {
			    return 0;
			}
			int count = 0;
			byte[] currentRow = null;
			Scan scan = new Scan();
			scan.addColumn(family, column);
			RegionScanner scanner = rce.getRegion().getScanner(scan);
			List<Cell> cells = new ArrayList<Cell>();
			boolean hasMore;
			String value = null;
			do {
			    hasMore = scanner.nextRaw(cells);
			    for (Cell cell : cells) {
			        if (currentRow == null || !CellUtil.matchingRows(cell, currentRow)) {
			            currentRow = CellUtil.cloneRow(cell);
			            value = Bytes.toString(cell.getValueArray(),cell.getValueOffset(),cell.getValueLength());
			            count += Integer.parseInt(value);
			            break;
			        }
			    }
			    cells.clear();
			} while (hasMore);
			
			return count;
		} catch (Exception e) {
			e.printStackTrace();
		}
		return -99999;
	}
	
}

3、创建Call

package com.klcwqy.hbase.demo.coprocessor.endpoints.sum;

import java.io.IOException;

import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcUtils;

import com.klcwqy.hbase.demo.coprocessor.endpoints.sum.SumRows.SumRequest;
import com.klcwqy.hbase.demo.coprocessor.endpoints.sum.SumRows.SumRowService;

public class GetSumCallable implements Batch.Call<SumRows.SumRowService, Integer>{
	
	private SumRows.SumRequest request;
	
	public GetSumCallable(SumRequest request) {
		super();
		this.request = request;
	}

	@Override
	public Integer call(SumRowService service) throws IOException {
		CoprocessorRpcUtils.BlockingRpcCallback<SumRows.SumResponse> rpcCallback = new CoprocessorRpcUtils.BlockingRpcCallback<SumRows.SumResponse>();
		service.getSum(null, request, rpcCallback);
		final SumRows.SumResponse response = rpcCallback.get();
	    return (int) (response.hasCount() ? response.getCount() : 0);
	}
}

4、注册到HBASE，注册过程与Observer一样，不在赘述。

5、客户端测试

public static void getSum(){
		try (final Table table = connection.getTable(TableName.valueOf("account"))) {
			SumRows.SumRequest request = SumRows.SumRequest.newBuilder().setFamily("info").setColumn("order").build();
			final Map<byte[], Integer> longMap = table.coprocessorService(SumRows.SumRowService.class, null, null, new GetSumCallable(request));
			long totalRows = 0;
			final Set<Map.Entry<byte[], Integer>> entries = longMap.entrySet();
			for (Map.Entry<byte[], Integer> entry : entries) {
				totalRows += entry.getValue();
			}
			System.out.println("总和:" + totalRows);
		} catch (Throwable e) {
			e.printStackTrace();
		}
	}

执行结果：总和:660
测试成功。

6、注意点
更新协处理器的时候（动态加载），需要将jar包重命名，否则新的协处理器不生效，因为JVM没重启。这不仅仅只是执行
alter 'account',METHOD => 'table_att_unset',NAME =>'coprocessor$1'
卸载协处理器那么简单，切记。