基于hbase的协处理器更新es索引数据(hbase二级索引解决方案)

elasticsearch使用的是7.x版本
hbase使用的是1.2.x版本
maven依赖

<dependencies>
        <dependency>
            <groupId>org.elasticsearch.client</groupId>
            <artifactId>elasticsearch-rest-high-level-client</artifactId>
            <version>7.1.0</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-server</artifactId>
            <version>1.2.1</version> 
        </dependency>
    </dependencies>

elasticsearch连接

package com.xxy.hbase_es;

import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import java.io.IOException;


/**
 * ES Cleint class
 */
public class ESClient {

    // ElasticSearch Client
    public static RestHighLevelClient client;

    /**
     * 创建连接
     */
    public static void initEsClient() {
        client = new RestHighLevelClient(
                RestClient.builder(
                        new HttpHost("192.168.23.171", 9200, "http")));
    }

    /**
     * 关闭连接
     */
    public static void closeEsClient() throws IOException {
        client.close();
    }
}

hbase协处理类

package com.xxy.hbase_es;

import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.log4j.Logger;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.client.RequestOptions;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;

public class HbaseObserver extends BaseRegionObserver {
    private static final Logger LOG = Logger.getLogger(HbaseObserver.class);
    /**
     * 协作器开始
     * @param e
     * @throws IOException
     */
    @Override
    public void start(CoprocessorEnvironment e) throws IOException {
        ESClient.initEsClient();
    }

    /**
     * 协作器结束
     * @param e
     * @throws IOException
     */
    @Override
    public void stop(CoprocessorEnvironment e) throws IOException {
        ESClient.closeEsClient();
    }

    /**
     * 将数据存入hbase之后同步存入es
     * @param e
     * @param put
     * @param edit
     * @param durability
     * @throws IOException
     */
    @Override
    public void postPut(ObserverContext<RegionCoprocessorEnvironment> e, Put put, WALEdit edit, Durability durability) throws IOException {
//        super.postPut(e, put, edit, durability);
        // 获取hbase的rowkey作为es的id
        String id = new String(put.getRow());
        // 根据hbase的rowkey组建es数据格式
        NavigableMap<byte[], List<Cell>> familyMap = put.getFamilyCellMap();
        Map<String, Object> json = new HashMap<String, Object>();
        for (Map.Entry<byte[], List<Cell>> entry : familyMap.entrySet()) {
            for (Cell cell : entry.getValue()) {
                String key = Bytes.toString(CellUtil.cloneQualifier(cell));
                String value = Bytes.toString(CellUtil.cloneValue(cell));
                json.put(key, value);
            }
        }
        // 提交该数据
        IndexRequest indexRequest = new IndexRequest("stu")
                .id(id).source(json);
        // todo 后期根据indexResponse判断数据是否插入成功，此处暂时不做判断
        IndexResponse indexResponse = ESClient.client.index(indexRequest, RequestOptions.DEFAULT);
    }

    /**
     * 将hbase数据删除后的操作同步删除es索引
     * @param e
     * @param delete
     * @param edit
     * @param durability
     * @throws IOException
     */
    @Override
    public void postDelete(ObserverContext<RegionCoprocessorEnvironment> e, Delete delete, WALEdit edit, Durability durability) throws IOException {
//        super.postDelete(e, delete, edit, durability);
        // 获取hbase的rowkey作为es的id
        String id = new String(delete.getRow());
        DeleteRequest request = new DeleteRequest(
                "stu",    //索引
                id);
        // 提交该数据
        // todo 后期根据indexResponse判断数据是否插入成功，此处暂时不做判断
        DeleteResponse deleteResponse = ESClient.client.delete(
                request, RequestOptions.DEFAULT);
    }

    /**
     * hbase桶导入后同步添加es索引
     * @param ctx
     * @param familyPaths
     * @throws IOException
     */
    @Override
    public boolean postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx, List<Pair<byte[], String>> familyPaths, boolean hasLoaded) throws IOException {
        for (Pair<byte[], String> familyPath : familyPaths) {
            LOG.info(Bytes.toString(familyPath.getFirst())+"===>"+familyPath.getSecond());
        }
        return super.postBulkLoadHFile(ctx, familyPaths, hasLoaded);
    }
}

打包上传至hdfs并且更改权限
create ‘stu’,‘info’

disable ‘stu’

alter ‘stu’, METHOD => ‘table_att’, ‘coprocessor’ => ‘hdfs://node01:9000/tmp/hbase_es/hbase_es.jar|com.xxy.hbase_es.HbaseObserver’|1001

enable ‘stu’

put ‘stu’,‘001’,‘info:name’,‘zhangsan’

此时查看elasticsearch stu索引也动态更新了