2021-06-25

kibana使用scroll查询es

scroll游标使用
公司需要一次返回所有数据，数据有点大，4w多条。所以使用游标查询，说白了就是分次数查询，每次查询会记录一个游标。然后用下面那个查询就可以从游标开始查询。每次游标都是一样的，有效时间保持一分钟。也就是说，第一次使用上面的查询方式，第二次滚动下面的查询就可以了。

GET gerrit-changes/_search?scroll=1m
{
  "size": 1000, 
  "query": {
    "match_all": {}
  }
}

GET /_search/scroll
{
  "scroll" : "1m",
  "scroll_id" : "FGluY2x1ZGVfY29udGV4dF91dWlkDXF1ZXJ5QW5kRmV0Y2gBFEVYd3RRbm9CM2FKdFBzTVc4RDB3AAAAAAASjK4WVWZ5TkZUTmpUdUt3QzVMMkZWVjY0Zw=="
}

go实现代码,代码为截取片段，不可直接复制，仅供参考。其实看上面的kibana就能够写出所有版本的代码了应该。

//正常查询用这个方法
func Search(bg context.Context,index string,buf io.Reader,includes ...string)(res *esapi.Response, err error){
	client := GetEsClient()
	res, err = client.Search(
		client.Search.WithContext(bg),
		client.Search.WithIndex(index),
		client.Search.WithBody(buf),
		client.Search.WithTrackTotalHits(true),
		client.Search.WithSourceIncludes(includes...),
		client.Search.WithPretty(),
	)
	return
}
//游标查询

func query(){
// 创建term查询条件，用于精确查询
	var buf bytes.Buffer
	query := map[string]interface{}{
		//"from":(params.Current - 1)*params.PageSize,
		//"size":params.PageSize,
		"size":1024, //设置每次查询返回数据的大小
		"query": getProjectQuerySentence(beginstamp,endstamp),

	}
	if err := json.NewEncoder(&buf).Encode(query); err != nil {
		xlog.Fatalf("Error encoding query: %s", err)
	}
	开始查询
	hits, err := utils.ScrollSearch(params.Flag,context.Background(),"gerrit-changes",&buf,
		"project","url","createdOn","status","owner","branch","lastUpdated","commitMessage",
		"patchSets.sizeInsertions","patchSets.sizeDeletions","comments","id")}
}

func ScrollSearch(flag int,bg context.Context,index string,buf io.Reader,includes ...string)(hits []interface{}, err error){
	client := GetEsClient()
	res, err := client.Search(
		client.Search.WithContext(bg),
		client.Search.WithIndex(index),
		client.Search.WithBody(buf),
		client.Search.WithScroll(6000),
		client.Search.WithTrackTotalHits(true),
		client.Search.WithSourceIncludes(includes...),
		client.Search.WithPretty(),
	)
	var r map[string]interface{}
	if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
		xlog.Info("json parsing", xlog.Any("error", "Error parsing the response body:"+err.Error()))
	}
	scrollId := r["_scroll_id"].(string)
	hits = r["hits"].(map[string]interface{})["hits"].([]interface{}) //将其转换成数组形式，后面就可以使用len和range了
	if flag == 0 { //flag = 0 就直接返回少量数据
		return
	}

	var scrollHits []interface{}
	i := 0
	for scrollHits == nil || len(scrollHits) >10{
		i++
		scrollRes,err := client.Scroll(
			client.Scroll.WithScroll(6000),
			client.Scroll.WithScrollID(scrollId),
		)
		if err != nil {
			xlog.Info("scroll query", xlog.Any("error", "scroll query error:"+err.Error()))
		}
		if err := json.NewDecoder(scrollRes.Body).Decode(&r); err != nil {
			xlog.Info("json parsing", xlog.Any("error", "Error parsing the response body:"+err.Error()))
		}
		scrollHits = r["hits"].(map[string]interface{})["hits"].([]interface{})
		hits = append(hits,scrollHits...)
	}


	return
}