elasticsearch 脚本排序

elasticsearch 脚本排序主要用于复杂场景的综合排序,脚本语言有 java ,painless,groovy,目前调研的为painless。

 

  • kibana 脚本创建mapping
DELETE ksc.metadata
PUT ksc.metadata
{
  "mappings": {
    "doc": {
      "properties": {
        "hotnum": {
          "type": "long"
        },
        "ds": {
          "type": "object",
          "properties": {
            "dsid": {
              "type": "long"
            },
            "dsname": {
              "type": "text",
              "fielddata": true
            }
          }
        },
        "tlbtype": {
          "type":"keyword"
        },
        "dbname": {
          "type": "text",
          "fielddata": true
        },
        "createdate": {
          "type": "date",
          "format": "yyyy-MM-dd HH:mm:ss"
        }
      }
    }
  }
}  

 

  • java 代码倒入数据
package com.hys.es;

import com.alibaba.fastjson.JSONObject;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.hys.Application;
import javafx.scene.input.DataFormat;
import lombok.Data;
import lombok.Getter;
import lombok.Setter;
import org.apache.http.HttpHost;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.alias.Alias;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequestBuilder;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateRequest;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.lucene.search.function.CombineFunction;
import org.elasticsearch.common.lucene.search.function.FunctionScoreQuery;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilder;
import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.elasticsearch.rest.RestClientProperties;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.core.ElasticsearchTemplate;
import org.springframework.data.elasticsearch.core.query.NativeSearchQuery;
import org.springframework.data.elasticsearch.core.query.NativeSearchQueryBuilder;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;

import java.io.IOException;
import java.text.SimpleDateFormat;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.stream.Collectors;

/**
 * es 的测试单元
 */
@ActiveProfiles("es")
@RunWith(SpringRunner.class)
@SpringBootTest(classes = Application.class,webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class EsTest {


    @Autowired
    ElasticsearchTemplate elasticsearchTemplate;

    @Autowired
    RestClientProperties restClientProperties;

    RestHighLevelClient restHighLevelClient = null;

    String index = "test";

    Client client = null;

    String type = "student";

    @Before
    public void inserIndexIfNotExist(){
        if(!elasticsearchTemplate.indexExists(index)){
             elasticsearchTemplate.createIndex(this.index);
        }
        client = elasticsearchTemplate.getClient();
        HttpHost[] httpHosts = new HttpHost[restClientProperties.getUris().size()];
        int index = 0;
        restClientProperties.getUris().forEach(uri->{
            httpHosts[index] = HttpHost.create(uri);
        });
        restHighLevelClient = new RestHighLevelClient(
                RestClient.builder(
                        httpHosts
                ));
    }

    /**
     * 插入模板
     */
    @Test
    public void insertTemplate(){
        String setting = "{\n" +
                "    \"template\" : \"te*\",\n" +
                "    \"order\" : 1,\n" +
                "    \"settings\" : {\n" +
                "        \"number_of_shards\" : 1\n" +
                "    },\n" +
                "    \"mappings\" : {\n" +
                "        \"type1\" : {\n" +
                "            \"_source\" : { \"enabled\" : true }\n" +
                "        }\n" +
                "    }\n" +
                "}";
        boolean ok = elasticsearchTemplate.createIndex(this.index, setting);
        System.out.println(ok);

    }

    @Test
    public void insertTemplate2() throws IOException {
        String setting = "{\n" +
                "    \"template\" : \"te*\",\n" +
                "    \"order\" : 1,\n" +
                "    \"settings\" : {\n" +
                "        \"number_of_shards\" : 1\n" +
                "    },\n" +
                "    \"mappings\" : {\n" +
                "        \"type1\" : {\n" +
                "            \"_source\" : { \"enabled\" : true }\n" +
                "        }\n" +
                "    }\n" +
                "}";


        CreateIndexRequest request = new CreateIndexRequest(index);//创建索引
        //创建的每个索引都可以有与之关联的特定设置。
        request.settings(Settings.builder().loadFromSource(setting,XContentType.JSON)
        );
        //创建索引时创建文档类型映射
        request.mapping("tweet",//类型定义
                "  {\n" +
                        "    \"tweet\": {\n" +
                        "      \"properties\": {\n" +
                        "        \"message\": {\n" +
                        "          \"type\": \"text\"\n" +
                        "        }\n" +
                        "      }\n" +
                        "    }\n" +
                        "  }",//类型映射,需要的是一个JSON字符串
                XContentType.JSON);

        //为索引设置一个别名
        request.alias(
                new Alias("twitter_alias")
        );
        //可选参数
        request.timeout(TimeValue.timeValueMinutes(2));//超时,等待所有节点被确认(使用TimeValue方式)
        //request.timeout("2m");//超时,等待所有节点被确认(使用字符串方式)

        request.masterNodeTimeout(TimeValue.timeValueMinutes(1));//连接master节点的超时时间(使用TimeValue方式)
        //request.masterNodeTimeout("1m");//连接master节点的超时时间(使用字符串方式)

        request.waitForActiveShards(2);//在创建索引API返回响应之前等待的活动分片副本的数量,以int形式表示。
        //request.waitForActiveShards(ActiveShardCount.DEFAULT);//在创建索引API返回响应之前等待的活动分片副本的数量,以ActiveShardCount形式表示。

        //同步执行
        CreateIndexResponse createIndexResponse = restHighLevelClient.indices().create(request);
        //异步执行
        //异步执行创建索引请求需要将CreateIndexRequest实例和ActionListener实例传递给异步方法:
        //CreateIndexResponse的典型监听器如下所示:
        //异步方法不会阻塞并立即返回。
        ActionListener<CreateIndexResponse> listener = new ActionListener<CreateIndexResponse>() {
            @Override
            public void onResponse(CreateIndexResponse createIndexResponse) {
                //如果执行成功,则调用onResponse方法;
            }
            @Override
            public void onFailure(Exception e) {
                //如果失败,则调用onFailure方法。
            }
        };
        restHighLevelClient.indices().createAsync(request, listener);//要执行的CreateIndexRequest和执行完成时要使用的ActionListener

        //返回的CreateIndexResponse允许检索有关执行的操作的信息,如下所示:
        boolean acknowledged = createIndexResponse.isAcknowledged();//指示是否所有节点都已确认请求
        boolean shardsAcknowledged = createIndexResponse.isShardsAcknowledged();//指示是否在超时之前为索引中的每个分片启动了必需的分片副本数
        System.out.println(acknowledged);

    }

    @Test
    public void insertTemplate3() throws IOException {


        PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys");

        List<String> indexPatterns = new ArrayList<String>();
        indexPatterns.add("ubi*");
        putIndexTemplateRequest.patterns(indexPatterns);
        Map<String, Object> settings = new HashMap<>();
        settings.put("number_of_shards", 1);
        putIndexTemplateRequest.settings(settings);
        PutIndexTemplateResponse putIndexTemplateResponse = restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);


        System.out.println(putIndexTemplateResponse.isAcknowledged());
    }


    @Test
    public void insertTemplate4() throws IOException {


        PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys1");
        putIndexTemplateRequest.source("{\n" +
                "    \"template\" : \"*\",\n" +
                "    \"order\" : 0,\n" +
                "    \"settings\" : {\n" +
                "        \"number_of_shards\" : 1\n" +
                "    },\n" +
                "    \"mappings\" : {\n" +
                "        \"type1\" : {\n" +
                "            \"_source\" : { \"enabled\" : false }\n" +
                "        }\n" +
                "    }\n" +
                "}",XContentType.JSON);
        PutIndexTemplateResponse putIndexTemplateResponse = restHighLevelClient.indices().putTemplate(putIndexTemplateRequest, RequestOptions.DEFAULT);


        System.out.println(putIndexTemplateResponse.isAcknowledged());
    }


    @Test
    public void validate(){
        PutIndexTemplateRequest putIndexTemplateRequest = new PutIndexTemplateRequest("templateahys1");
        putIndexTemplateRequest.source("{\n" +
                "    \"template\" : \"*\",\n" +
                "    \"order\" : 0,\n" +
                "    \"settings1\" : {\n" +
                "        \"number_of_shards\" : 1\n" +
                "    },\n" +
                "    \"mappings\" : {\n" +
                "        \"type1\" : {\n" +
                "            \"_source\" : { \"enabled\" : false }\n" +
                "        }\n" +
                "    }\n" +
                "}",XContentType.JSON);
        System.out.println("ok");
    }

    //插入type
    @Test
    public void querySoreSingle() throws IOException {

        //https://blog.csdn.net/u012270682/article/details/80165836

        String index = "ksc.metadata";
        String type = "doc";
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
        queryBuilder.must(QueryBuilders.multiMatchQuery("ds62","ds.dsname"));

       // String idOrCode = "params.current1-doc['createdate'].date";
        //根据热度排序

        String hotNumCode = "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}" +
                "else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;";
        Map<String,Object> params = new HashMap<>();
       // params.put("current1",new SimpleDateFormat("yyyy-MM-dd").format(LocalDateTime.now()));
        params.put("current1",new Date());
        params.put("hotnumSize",100);
        params.put("hotnumMinscore",10);
        params.put("hotnumMaxscore",100);
        //params.put("hotnumsize",100);
        Script script = new Script(ScriptType.INLINE,"painless",hotNumCode,params);
        ScoreFunctionBuilder scoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(script);
        FunctionScoreQueryBuilder scoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,scoreFunctionBuilder)
                .scoreMode(FunctionScoreQuery.ScoreMode.SUM).boostMode(CombineFunction.REPLACE);
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        builder.withQuery(scoreQueryBuilder).withIndices(index).withTypes(type);
        NativeSearchQuery nativeSearchQuery = builder.build();
        String queryc = nativeSearchQuery.getQuery().toString();
        System.out.println(queryc);
        List<Map> maps = elasticsearchTemplate.queryForList(nativeSearchQuery, Map.class);
        System.out.println(maps);


    }


    @Test
    public void querySoreMult() throws IOException {

        //https://blog.csdn.net/u012270682/article/details/80165836

        String index = "ksc.metadata";
        String type = "doc";
        BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
        queryBuilder.must(QueryBuilders.multiMatchQuery("ds62","ds.dsname"));

        // String idOrCode = "params.current1-doc['createdate'].date";
        //根据热度排序
        String hotNumCode = "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}" +
                "else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;";
        Map<String,Object> params = new HashMap<>();
        // params.put("current1",new SimpleDateFormat("yyyy-MM-dd").format(LocalDateTime.now()));
        params.put("current1",new Date());
        params.put("hotnumSize",100);
        params.put("hotnumMinscore",10);
        params.put("hotnumMaxscore",100);
        Script hotScript = new Script(ScriptType.INLINE,"painless",hotNumCode,params);
        ScoreFunctionBuilder hotScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(hotScript);


        //时间排序
        String timeCode = "Calendar c = Calendar.getInstance(); c.setTime(new Date());c.add(Calendar.DATE, - 7);long beforeNowSeven = c.getTimeInMillis();long differ = (doc['createdate'].value.millis - beforeNowSeven)/86400000;if(differ> 7){differ = 1;}return differ;";
        Script timeScript = new Script(ScriptType.INLINE,"painless",timeCode,params);
        ScoreFunctionBuilder timeScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(timeScript);


        //类型排序
        String tlbTypeCode = "if(doc['tlbtype'].value=='table'){return 2;}else{return 1;}";
        Script tlbTypeScript = new Script(ScriptType.INLINE,"painless",tlbTypeCode,params);
        ScoreFunctionBuilder tlbScriptScoreFunctionBuilder = ScoreFunctionBuilders.scriptFunction(tlbTypeScript);

        FunctionScoreQueryBuilder.FilterFunctionBuilder[] filterFunctionBuilders = new FunctionScoreQueryBuilder.FilterFunctionBuilder[]{
            new FunctionScoreQueryBuilder.FilterFunctionBuilder(hotScriptScoreFunctionBuilder),
                new FunctionScoreQueryBuilder.FilterFunctionBuilder(timeScriptScoreFunctionBuilder),
                new FunctionScoreQueryBuilder.FilterFunctionBuilder(tlbScriptScoreFunctionBuilder)
        };
        FunctionScoreQueryBuilder scoreQueryBuilder = new FunctionScoreQueryBuilder(queryBuilder,filterFunctionBuilders)
                .scoreMode(FunctionScoreQuery.ScoreMode.SUM).boostMode(CombineFunction.REPLACE);
        NativeSearchQueryBuilder builder = new NativeSearchQueryBuilder();
        builder.withQuery(scoreQueryBuilder).withIndices(index).withTypes(type);
        NativeSearchQuery nativeSearchQuery = builder.build();
        String queryc = nativeSearchQuery.getQuery().toString();
        System.out.println(queryc);
        List<Map> maps = elasticsearchTemplate.queryForList(nativeSearchQuery, Map.class);
        System.out.println(maps);


    }




    
    @Test
    public void batchData() throws IOException {
        String index = "ksc.metadata";
        String type = "doc";
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout(TimeValue.timeValueMinutes(2));
        bulkRequest.timeout("2m");
        DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");

        for (int i = 0;i<1000;i++) {
            int r = new Random().nextInt(50);
            int d = new Random().nextInt(100);
            Ds ds = new Ds();
            ds.setDsid(d);
            ds.setDsname("ds"+d);
            LocalDateTime dateTime = LocalDateTime.of(2019, 8,getZ(10) , getZ(24) ,getZ(60) );

            Doc doc = new Doc();
            doc.setCreatedate(dateTime.format(formatter));
            doc.setHotnum(new Random().nextInt(1000));
            doc.setDbname("db"+r);
            doc.setTlbtype();
            doc.setDs(ds);
            bulkRequest.add(new IndexRequest(index, type, UUID.randomUUID().toString())
                    .source(JSONObject.toJSONString(doc),XContentType.JSON));

        }
        BulkResponse response = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        //response.hasFailures()
        System.out.println(response.status());
    }

    private int getZ(int seed){
        int i = 0;
        for(;;){
            i=new Random().nextInt(seed);
            if (i > 0){
                return  i;
            }
        }

    }

    @Test
    public void batch100Minlin() throws IOException {

        for (int i = 0;i<2;i++){
            batchData();
        }
    }

    @Test
    public void queryScript(){

        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");Calendar c = Calendar.getInstance();
        //过去七天
        c.setTime(new Date());
        c.add(Calendar.DATE, - 7);
        Date d = c.getTime();
        long time = d.getTime();
        //86400000
        System.out.println(24*60*60*1000);

        String day = format.format(d);
        System.out.println("过去七天:"+day);
        
    }

    @After
    public void  close() throws IOException {

        if(client!=null){
            client.close();
        }

        if(restHighLevelClient!=null){
            restHighLevelClient.close();
        }
    }
    
    
    @Document(indexName = "test", type = "student", createIndex = false)
    public static class Student {

        @Field
        String id;

        @Field
        String name;

        @Field
        Long age;


        @Field(format = DateFormat.year_month_day)
        Date birthDay;


    }


    @Data
    public static class Doc{

        Integer hotnum;

        String dbname;

        //@JsonFormat (shape = JsonFormat.Shape.STRING, pattern ="yyyy-MM-dd HH:mm:ss")
        //LocalDateTime createdate;
        String createdate;

        String tlbtype;

        public void setTlbtype(){
            boolean aBoolean = new Random().nextBoolean();
            if(aBoolean){
                tlbtype = "table";
            }else {
                tlbtype = "col";
            }
        }

        Ds ds;


    }

    @Data
    public static class  Ds{

        Integer dsid;

        String dsname;
    }



}

 

  • kibana脚本查询
GET ksc.metadata/_search
{
  "from": 0,
  "size": 50,
  "query": {
    "function_score": {
      "query": {
        "multi_match": {
          "query": "ds62",
          "fields": [
            "ds.dsname"
          ]
        }
      },
      "score_mode": "sum",
      "boost_mode" : "replace",
      "functions": [
        {
          "script_score": {
            "script": {
              "inline": "if(doc['tlbtype'].value=='table'){return 2;}else{return 1;}",
              "params": {

              },
              "lang": "painless"
            }
          }
        },
        {
          "script_score": {
            "script": {
              "inline": "Calendar c = Calendar.getInstance(); c.setTime(new Date());c.add(Calendar.DATE, - 7);long beforeNowSeven = c.getTimeInMillis();long differ = (doc['createdate'].value.millis - beforeNowSeven)/86400000;if(differ> 7){differ = 1;}return differ;",
              "params": {

              }
              , "lang": "painless"
            }
          }
        },
        {
          "script_score": {
            "script": {
              "inline": "long hotscore = doc['hotnum'].value/params['hotnumSize']*10;if(hotscore < params['hotnumMinscore']){hotscore =params['hotnumMinscore'];}else if(hotscore > params['hotnumMaxscore']){hotscore =params['hotnumMaxscore'];}return hotscore;",
              "lang": "painless",
              "params": {
                "current1": "2019-08-10T10:33:53.470Z",
                "hotnumSize": 100,
                "hotnumMinscore": 10,
                "hotnumMaxscore": 100
              }
            }
          }
        }
      ]
    }
  }
}

文档批量更新接口

POST 1_test_metadata_index/_update_by_query
{
  "script": {
    "lang": "painless",
    "params": {
      "bizIds":[74]
    }, 
    "inline": "ctx._source.bizCatalogList = ctx._source.bizCatalogList.stream().filter(x -> !params.bizIds.contains(x.id)).collect(Collectors.toList());"
  },
  "query": {
    "term": {
      "id.keyword": {
        "value": "78013a35c69d28afcddc7b85c9c3b7da"
      }
    }
  }
}
  • java 代码实现搜索
  • 1
    点赞
  • 4
    收藏
    觉得还不错? 一键收藏
  • 0
    评论

“相关推荐”对你有帮助么?

  • 非常没帮助
  • 没帮助
  • 一般
  • 有帮助
  • 非常有帮助
提交
评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值