背景
- 大数据架构业务场景中需要实时数据落入ES,基本上是业务数据,目的是为了封装后作为规则引擎的变量提供,是变量系统的一部分;
- 架构数据流来源于Maxwell,Spark Streaming做数据流处理,落库使用RestHighLevelClient的同步提交Bulk写入;增删改此文档不涉及,主要就是客户端以及查询的封装,为后续变量系统的应用部分;
客户端
streaming直接使用
def createESClientNew ( ) : RestHighLevelClient = {
new RestHighLevelClient (
RestClient . builder (
new HttpHost ( "host" , 9200 , "http" ) ,
new HttpHost ( "host" , 9200 , "http" ) ,
new HttpHost ( "host" , 9200 , "http" )
)
)
}
集群节点之间使用,序列化
public class BaseEsClientSerializable implements Serializable {
public RestHighLevelClient getClient ( ) {
return null ;
}
}
public class EsClientSerializable extends BaseEsClientSerializable {
public static RestHighLevelClient client = new RestHighLevelClient (
RestClient . builder ( new HttpHost ( "host" , 9200 , "http" ) ) ) ;
@Override
public RestHighLevelClient getClient ( ) {
return client;
}
}
配置连接池查询
public class EsClientPoolFactory implements PooledObjectFactory < RestHighLevelClient > {
@Override
public PooledObject < RestHighLevelClient > makeObject ( ) throws Exception {
RestHighLevelClient client = null ;
try {
client = new RestHighLevelClient ( RestClient . builder (
new HttpHost ( "host" , 9200 , "http" ) ) ) ;
} catch ( Exception e) {
e. printStackTrace ( ) ;
}
return new DefaultPooledObject < > ( client) ;
}
@Override
public void destroyObject ( PooledObject < RestHighLevelClient > pooledObject) throws Exception {
RestHighLevelClient highLevelClient = pooledObject. getObject ( ) ;
highLevelClient. close ( ) ;
}
@Override
public boolean validateObject ( PooledObject < RestHighLevelClient > pooledObject) {
return true ;
}
@Override
public void activateObject ( PooledObject < RestHighLevelClient > pooledObject) throws Exception {
System . out. println ( "activateObject" ) ;
}
@Override
public void passivateObject ( PooledObject < RestHighLevelClient > pooledObject) throws Exception {
System . out. println ( "passivateObject" ) ;
}
}
public class ElasticSearchPoolUtil {
private static GenericObjectPoolConfig poolConfig = new GenericObjectPoolConfig ( ) ;
static {
poolConfig. setMaxTotal ( 8 ) ;
}
private static EsClientPoolFactory esClientPoolFactory = new EsClientPoolFactory ( ) ;
private static GenericObjectPool < RestHighLevelClient > clientPool = new GenericObjectPool < > ( esClientPoolFactory,
poolConfig) ;
public static RestHighLevelClient getClient ( ) throws Exception {
RestHighLevelClient client = clientPool. borrowObject ( ) ;
return client;
}
public static void returnClient ( RestHighLevelClient client) {
clientPool. returnObject ( client) ;
}
}
查询
没有封装的很完全,多个熟悉之后按照业务场景添加就好,目的就是后续查询不需要开发,直接配置条件即可;
多条件match,过滤查询
def searchHitsByMatchs(
client: RestHighLevelClient,
index: String ,
matchs: JSONObject,
size: Int
) : SearchResponse = {
val searchRequest = new SearchRequest( index)
val searchSourceBuilder = new SearchSourceBuilder( ) . size( size)
val boolQueryBuilder = QueryBuilders. boolQuery( )
matchs. keySet( ) . toArray( ) . foreach( jsonKey => {
val matchQueryBuilder = QueryBuilders. matchQuery( jsonKey. toString, matchs. getString( jsonKey. toString) )
boolQueryBuilder. must( ) . add( matchQueryBuilder)
} )
searchSourceBuilder. query( boolQueryBuilder)
searchRequest. source( searchSourceBuilder)
client. search(
searchRequest,
RequestOptions. DEFAULT
)
}
多条件term,过滤查询
def searchHitsByTerms( client: RestHighLevelClient,
index: String ,
terms: JSONObject,
size: Int
) : SearchResponse = {
val searchRequest = new SearchRequest( index)
val searchSourceBuilder = new SearchSourceBuilder( ) . size( size)
val boolQueryBuilder = QueryBuilders. boolQuery( )
terms. keySet( ) . toArray( ) . foreach( jsonKey => {
val termQueryBuilder = QueryBuilders. termQuery( jsonKey. toString, terms. getString( jsonKey. toString) )
boolQueryBuilder. must( ) . add( termQueryBuilder)
} )
searchSourceBuilder. query( boolQueryBuilder)
searchRequest. source( searchSourceBuilder)
client. search(
searchRequest,
RequestOptions. DEFAULT
)
}
多条件match查询返回count
def searchCountByMatchs(
client: RestHighLevelClient,
index: String ,
matchs: JSONObject
) : CountResponse = {
val countRequest: CountRequest = new CountRequest( index)
val searchSourceBuilder = new SearchSourceBuilder( )
val boolQueryBuilder = QueryBuilders. boolQuery( )
matchs. keySet( ) . toArray( ) . foreach( jsonKey => {
val matchQueryBuilder = QueryBuilders. matchQuery( jsonKey. toString, matchs. getString( jsonKey. toString) )
boolQueryBuilder. must( ) . add( matchQueryBuilder)
} )
searchSourceBuilder. query( boolQueryBuilder)
countRequest. source( searchSourceBuilder)
client. count(
countRequest,
RequestOptions. DEFAULT
)
}
多条件match聚合查询
def searchSumByMatchs(
client: RestHighLevelClient,
index: String ,
matchs: JSONObject,
returnFieldName: String ,
sumField: String ,
aggregationType: String
) : SearchResponse = {
val searchRequest = new SearchRequest( index)
val searchSourceBuilder = new SearchSourceBuilder( )
val boolQueryBuilder = QueryBuilders. boolQuery( )
matchs. keySet( ) . toArray( ) . foreach( jsonKey => {
val matchQueryBuilder = QueryBuilders. matchQuery( jsonKey. toString, matchs. getString( jsonKey. toString) )
boolQueryBuilder. must( ) . add( matchQueryBuilder)
} )
searchSourceBuilder. query( boolQueryBuilder)
val aggregationBuilder: ValuesSourceAggregationBuilder. LeafOnly[ _ > : ValuesSource. Numeric < : ValuesSource, _ > :
SumAggregationBuilder with MaxAggregationBuilder with MinAggregationBuilder with AvgAggregationBuilder with ValueCountAggregationBuilder < :
ValuesSourceAggregationBuilder. LeafOnly[ _ > :
ValuesSource. Numeric < : ValuesSource, _ > : SumAggregationBuilder with MaxAggregationBuilder with MinAggregationBuilder with AvgAggregationBuilder with ValueCountAggregationBuilder] ] =
aggregationType match {
case "SUM" =>
AggregationBuilders. sum( returnFieldName) . field( sumField)
case "MAX" =>
AggregationBuilders. max( returnFieldName) . field( sumField)
case "MIN" =>
AggregationBuilders. min( returnFieldName) . field( sumField)
case "AVG" =>
AggregationBuilders. avg( returnFieldName) . field( sumField)
case _ =>
AggregationBuilders. count( returnFieldName) . field( sumField)
}
searchSourceBuilder. aggregation( aggregationBuilder)
searchRequest. source( searchSourceBuilder)
client. search(
searchRequest,
RequestOptions. DEFAULT
)
}
数据解析
解析searchResponse为SearchHit 集合
def analysisSearchResponseToHits(
searchResponse : SearchResponse
) : Array[ SearchHit] = {
searchResponse. getHits. getHits
}
解析CountResponse为long值
def analysisCountResponseToLong( countResponse : CountResponse) : Long = {
countResponse. getCount
}
解析searchResponse为各种聚合Double值
def analysisSearchResponseToAggregationFloat(
searchResponse : SearchResponse ,
returnFieldName: String ,
aggregationType: String
) : Double = {
val aggregations = searchResponse. getAggregations
val aggregation: Aggregation = aggregations. get( returnFieldName) . asInstanceOf[ Aggregation]
aggregationType match {
case "SUM" =>
aggregation. asInstanceOf[ ParsedSum] . value( )
case "MAX" =>
aggregation. asInstanceOf[ ParsedMax] . value( )
case "MIN" =>
aggregation. asInstanceOf[ ParsedMin] . value( )
case "AVG" =>
aggregation. asInstanceOf[ ParsedAvg] . value( )
case _ =>
aggregation. asInstanceOf[ ParsedValueCount] . value( )
}
}
调用使用
val client = EsClientSerializable. client
val matchs = new JSONObject( )
matchs. put( "cert_no.keyword" , "xxx" )
val countResponse = searchCountByMatchs(
client,
"vs_fms_repay_plan_wide_table_new" ,
matchs
)
println( SingleResponseAnalysisUtils. analysisCountResponseToLong( countResponse) )
val searchResponse = searchSumByMatchs(
client,
"vs_fms_repay_plan_wide_table_new" ,
matchs,
"result_principal" ,
"principal" ,
"MAX"
)
println( SingleResponseAnalysisUtils. analysisSearchResponseToAggregationFloat(
searchResponse,
"result_principal" ,
"MAX"
) )
说明
后续会将ES实际项目中的部分逐渐更新出来; 包含在大数据场景下:实时、离线不同项目中的运用;