hbase协处理器endpoint示例（求最值）

---

hbase协处理器endpoint示例（求最值）

---

废话不多说，hbase原理等知识点网上太多了，也可以参考官网，在这里笔者就不过多赘述了，以下是经笔者测试的一个案例：

create 'test2', 'f1'

向f1加入字段 PBRL

1. 创建一个count_max.proto的文件，添加如下内容：

syntax = "proto2";
option java_package = "com.endpoint.test.max";
option java_outer_classname = "maxProtocol";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;

message maxRequest {
    required string family = 1;
    required string column = 2;
}
message maxResponse {
    required double max = 1 [default = 0];
   
}
service maxService {
  rpc getmax(maxRequest)
    returns (maxResponse);
}

`
2. 使用protoc生成类文件：
cmd进入proto文件的目录下，执行如下命令

protoc --java_out=./ count_max.proto

命令执行完成后会在当前目录下生产一个名称为maxProtocol的类，将这个类复制到项目的com.endpoint.test.max包下，这些是需要注意的地方：

• maxRequest 内部类，表示请求信息
• maxResponse 内部类，表示返回信息
• maxService 内部类，表示所提供的服务，这个类还有一个内部接口，这个接口定义了 getmax()这个方法

3. 服务端代码

package com.endpoint.test.max;

import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.util.Bytes;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;

public class MaxServer extends maxProtocol.maxService implements Coprocessor,CoprocessorService{

    private RegionCoprocessorEnvironment env;   // 定义环境

    @Override
    public void getmax(RpcController controller, maxProtocol.maxRequest request, RpcCallback<maxProtocol.maxResponse> done) {
        String family = request.getFamily();
        if (null == family || "".equals(family)) {
            throw new NullPointerException("you need specify the family");
        }
        String column = request.getColumn();
        if (null == column || "".equals(column))
            throw new NullPointerException("you need specify the column");


        Scan scan = new Scan();

        // 设置扫描对象
        scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(column));


        // 定义变量
        maxProtocol.maxResponse response = null;
        InternalScanner scanner = null;

        // 扫描每个region，取值后求和
        try {
            scanner = env.getRegion().getScanner(scan);
            List<Cell> results = new ArrayList<Cell>();
            boolean hasMore = false;
            Double max = null;
            do {
                hasMore = scanner.next(results);
                //for (Cell cell : results) {
                if (results.isEmpty()) {
                    continue;
                }
                Cell kv = results.get(0);
                try {
                    Double temp = Double.parseDouble(new String(CellUtil.cloneValue(kv)));
                    max = max != null && (temp == null || compare(temp, max) <= 0) ? max : temp;
                } catch (Exception e) {
                }


                results.clear();
                //sum += Long.parseLong(new String(CellUtil.cloneValue(cell)));

                //results.clear();
            } while (hasMore);
            // 设置返回结果
            response = maxProtocol.maxResponse.newBuilder().setMax((max!=null?max.doubleValue():Double.MAX_VALUE)).build();
        } catch (IOException e) {
            ResponseConverter.setControllerException(controller, e);
        } finally {
            if (scanner != null) {
                try {
                    scanner.close();
                } catch (IOException e) {
                    //e.printStackTrace();
                }
            }
        }
        // 将rpc结果返回给客户端
        done.run(response);

    }



    public static int compare(Double l1, Double l2) {
        if (l1 == null ^ l2 == null) {
            return l1 == null ? -1 : 1; // either of one is null.
        } else if (l1 == null)
            return 0; // both are null
        return l1.compareTo(l2); // natural ordering.
    }
    @Override
    public Service getService() {

        return this;
    }

    @Override
    public void start(CoprocessorEnvironment env) throws IOException {
        if (env instanceof RegionCoprocessorEnvironment) {
            this.env = (RegionCoprocessorEnvironment)env;
        } else {
            throw new CoprocessorException("no load region");
        }

    }

    @Override
    public void stop(CoprocessorEnvironment env) throws IOException {

    }

    /**
     * <code>rpc getmax(.maxRequest) returns (.maxResponse);</code>
     *
     * @param controller
     * @param request
     * @param done
     */

}

4. 动态部署coprocessor

进入hbase shell :

• disable ‘test2’

• alter ‘test2’, METHOD => ‘table_att’, ‘Coprocessor’=>‘hdfs://linux5:8020/tmp/coprosessor/CoprosessorServer-1.0-SNAPSHOT1.jar|com.endpoint.test.max.MinServer|’

• enable ‘test2’

• 卸载协处理器：

alter 'test2', METHOD => 'table_att_unset', NAME => 'coprocessor$1'

5. Client端代码

在idea中操作即可运行

package com.endpoint.test.max;


import com.endpoint.test.max.maxProtocol;
import com.google.protobuf.ServiceException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.coprocessor.Batch;
import org.apache.hadoop.hbase.ipc.BlockingRpcCallback;
import org.apache.hadoop.hbase.util.Bytes;

import java.io.IOException;
import java.util.Map;

public class maxClient {

   //设置参数
    private static final String TABLE_NAME = "test2";
    private static final String FAMILY = "f1";
    private static final String COLUMN = "PBRL";
    private static final byte[] STRAT_KEY = Bytes.toBytes("000");
    private static final byte[] END_KEY = Bytes.toBytes("100");

    public static void main(String[] args) throws Exception {



        // 配置HBse
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "linux3,linux4,linux5");
        conf.set("hbase.zookeeper.property.clientPort", "2181");
        conf.setLong("hbase.rpc.timeout", 600000);
        System.setProperty("hadoop.home.dir", "F:/ruanjian/hadoop-2.6.0-cdh5.14.0");

        // 建立一个连接
        Connection conn = ConnectionFactory.createConnection(conf);
        // 获取表
        HTable table = (HTable) conn.getTable(TableName.valueOf(TABLE_NAME));


        // 设置请求对象
        final maxProtocol.maxRequest request = maxProtocol.maxRequest.newBuilder().setFamily(FAMILY).setColumn(COLUMN).build();

        try {
            // 获得返回值
            Map<byte[], Double> result = table.coprocessorService(maxProtocol.maxService.class, STRAT_KEY,  END_KEY,
                    new Batch.Call<maxProtocol.maxService, Double>() {

                        @Override
                        public Double call(maxProtocol.maxService maxService) throws IOException {
                            BlockingRpcCallback<maxProtocol.maxResponse> rpcCallback = new BlockingRpcCallback<maxProtocol.maxResponse>();
                            maxService.getmax(null, request, rpcCallback);
                            maxProtocol.maxResponse response = rpcCallback.get();
                            return response.getMax();
                        }


                    });
            Double max = null;
            // 将返回值进行迭代相加
            for (Double temp : result.values()) {
                max = max != null && (temp == null || compare(temp, max) <= 0) ? max : temp;
            }
            // 结果输出
            System.out.println("max: " + max);

        } catch (ServiceException e) {
            e.printStackTrace();
        }catch (Throwable e) {
            e.printStackTrace();
        }
        table.close();
        conn.close();

    }

    public static int compare(Double l1, Double l2) {
        if (l1 == null ^ l2 == null) {
            return l1 == null ? -1 : 1; // either of one is null.
        } else if (l1 == null)
            return 0; // both are null
        return l1.compareTo(l2); // natural ordering.
    }

}

github代码参考：https://github.com/kaiqid/CoprosessorServer/tree/master/src/main/java/com/endpoint/test/max
https://github.com/kaiqid/CoprocessorClient/tree/master/src/main/java/com/endpoint/test/max
里面有maxProtocol类