试着写了个协处理器。感觉协处理器不好使,一旦出错,搞得整个Hbase集群都挂了…
后来选择用Spark建索引放ES,来改善查询这块儿的效率
具体步骤有点忘,后面补充。
1.
option java_package = "com.cloud.coprocessor";
option java_outer_classname = "RAProto";
option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
message RARequest {
required string cloumnFamily = 1;
required string keyword=2;
required string timeSetStirng=3;
}
message RAStat {
required string resourceType=1;
required int32 resourceCnt=2;
required int64 resourceDuration=3;
}
message RAResponse{
repeated RAStat raStat=1;
}
service RAService {
rpc getRAStat(RARequest)
returns (RAResponse);
}
执行某个命令之后会生成:RAProto.java
然后写自己的service
package com.cloud.coprocessor;
import com.google.protobuf.RpcCallback;
import com.google.protobuf.RpcController;
import com.google.protobuf.Service;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.Coprocessor;
import org.apache.hadoop.hbase.CoprocessorEnvironment;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.RegexStringComparator;
import org.apache.hadoop.hbase.filter.RowFilter;
import org.apache.hadoop.hbase.protobuf.ResponseConverter;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class RAEndPoint extends RAProto.RAService implements Coprocessor, CoprocessorService {
private RegionCoprocessorEnvironment env;
@Override
public void getRAStat(RpcController controller, RAProto.RARequest request, RpcCallback done) {
Scan scan = new Scan();
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
String keyword = request.getKeyword();
String timeSetString = request.getTimeSetStirng();
String[] timeArray = timeSetString.split("\\,");
String regex;
for (String time : timeArray) {
if (!"".equals(keyword)) {
regex = "(.*" + keyword + ")(.*" + time + "$)";
} else {
regex = time + "$";
}
RowFilter filter = new RowFilter(CompareFilter.CompareOp.EQUAL,
new RegexStringComparator(regex));
filterList.addFilter(filter);
}
scan.setFilter(filterList);
RAProto.RAResponse.Builder response = RAProto.RAResponse.newBuilder();
InternalScanner scanner = null;
try {
scanner = env.getRegion().getScanner(scan);
Map<String, RAProto.RAStat> resultMap = new HashMap<>();
List<Cell> results = new ArrayList<>();
boolean hasMore;
do {
results.clear();
hasMore = scanner.next(results);
for (Cell cell : results) {
RAProto.RAStat.Builder builder = RAProto.RAStat.newBuilder();
String qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
if (!qualifier.equals("app_id")) {
String[] qualifierArray = qualifier.split("\\_");
String resourceType = qualifierArray[0];
builder.setResourceType(resourceType);
int resourceCnt = 0;
long resourceDuration = 0L;
switch (qualifierArray[1]) {
case "cnt":
resourceCnt = Bytes.toInt(CellUtil.cloneValue(cell));
break;
case "duration":
resourceDuration = Bytes.toLong(CellUtil.cloneValue(cell));
break;
default:
break;
}
if (resultMap.containsKey(resourceType)) {
RAProto.RAStat valueInMap = resultMap.get(resourceType);
builder.setResourceCnt(valueInMap.getResourceCnt() + resourceCnt);
builder.setResourceDuration(valueInMap.getResourceDuration() + resourceDuration);
} else {
builder.setResourceCnt(resourceCnt);
builder.setResourceDuration(resourceDuration);
}
resultMap.put(resourceType, builder.build());
}
}
} while (hasMore);
resultMap.values().forEach(response::addRaStat);
} catch (IOException ioe) {
ResponseConverter.setControllerException(controller, ioe);
} finally {
if (scanner != null) {
try {
scanner.close();
} catch (IOException ignored) {
}
}
}
done.run(response.build());
}
@Override
public Service getService() {
return this;
}
@Override
public void start(CoprocessorEnvironment env) throws IOException {
if (env instanceof RegionCoprocessorEnvironment) {
this.env = (RegionCoprocessorEnvironment) env;
} else {
throw new CoprocessorException("Must be loaded on a table region!");
}
}
@Override
public void stop(CoprocessorEnvironment env) throws IOException {
// nothing to do
}
}