golang 查询 Elasticsearch

说明：本文以日志查询为例，ES集群版本为7.2。实现效果如下：

具体步骤如下：

1.定义mapping，创建索引

PUT log
{
  "settings": {
    "index": {
      "number_of_shards": 1,
      "number_of_replicas": 0
    }
  },
  "mappings": {
    "properties": {
      "timestamp": {
        "type": "date"
      },
      "hostname": {
        "type": "text",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 25
          }
        }
      },
      "ip": {
        "type": "ip",
        "fields": {
          "keyword": {
            "type": "keyword",
            "ignore_above": 25
          }
        }
      },
      "level": {
        "type": "keyword"
      }
    }
  }
}

2.批量导入数据，确保数据导入成功
POST _bulk
{ "create" : { "_index" : "log", "_id" : "1" } }
{ "timestamp" : 1605337534872,"hostname":"HF-DC-FR-001","ip":"192.168.0.1","level":"INFO" }
{ "create" : { "_index" : "log", "_id" : "2" } }
{ "timestamp" : 1605332550895,"hostname":"HF-DC2-FR-011","ip":"192.168.1.1","level":"ERROR" }
{ "create" : { "_index" : "log", "_id" : "3" } }
{ "timestamp" : 1605226232774,"hostname":"HF-DC-OP-001","ip":"192.167.0.1","level":"INFO" }
{ "create" : { "_index" : "log", "_id" : "4" } }
{ "timestamp" : 1605226554847,"hostname":"HF-DC2Y-FR-001","ip":"192.158.0.1","level":"WARN" }
{ "create" : { "_index" : "log", "_id" : "5" } }
{ "timestamp" : 1605139523094,"hostname":"HF-UE-WU-001","ip":"10.168.0.1","level":"WARN" }
{ "create" : { "_index" : "log", "_id" : "6" } }
{ "timestamp" : 1605139523095,"hostname":"HP-OU-FR-001","ip":"10.158.0.1","level":"INFO" }
//查询数据是否导入成功
GET log/_search
{
}

3.es查询语句

GET log/_search
{
  "from":0,
  "size":10,
  "query": {
    "bool": {
      "filter": [
        {
          "term": {
            "level": "INFO"
          }
        },
        {
          "range": {
            "timestamp": {
              "gte": 1605139523095,
              "lte": 1605337534872
            }
          }
        },
        {
          "wildcard": {
            "ip.keyword": {
              "value": "192.*"
            }
          }
        },
        {
          "match": {
            "hostname": {
              "query": "HF-DC",
              "operator": "or"
            }
          }
        }
      ]
    }
  },
  "sort": [
    {
      "timestamp": {
        "order": "desc"
      }
    }
  ]
}

//查询结果

4.golang实现es的查询语句
golang调用包：https://github.com/olivere/elastic 因为我的es版本是7.2，所以应使用版本7
文档：https://godoc.org/github.com/olivere/elastic
ES官网：https://www.elastic.co/guide/en/elasticsearch/reference/7.x/index.html
代码实现：

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"github.com/olivere/elastic/v7"
)

//传入参数
type listParam struct{
	page int
	size int
	ip string
	startTime int64
	endTime int64
	level string
	hostname string
}

//返回查询结果
type listRtn struct{
	total int64
	logs []log
}
type log struct{
	id string
	Ip string `json:"ip"`
	Timestamp int64 `json:"timestamp`
	Level string `json:"level"`
	Hostname string `json:"hostname"`
}

func main(){
	//传入参数
	var listPostParam=listParam{
		page:0,
		size:10,
		ip:"192.*",
		startTime:1605139523095,
		endTime:1605337534872,
		level:"INFO",
		hostname:"HF-DC",
	}

	//连接es
	client, err := elastic.NewClient(elastic.SetURL("http://127.0.0.1:9200/"),elastic.SetBasicAuth("elastic", "elastic-pwd"))
	if err != nil {
		panic(err)
	}

	//es查询
	var res *elastic.SearchResult
	indices:="log"
	boolQ := elastic.NewBoolQuery()
	if len(strings.TrimSpace(listPostParam.level))>0{
		boolQ.Filter(elastic.NewTermQuery("level", listPostParam.level))//level需准确匹配
	}

	if listPostParam.endTime>0 && listPostParam.startTime <= 0{//日期区间查询
		boolQ.Filter(elastic.NewRangeQuery("timestamp").Lte(listPostParam.endTime))
	}else if listPostParam.endTime<=0 && listPostParam.startTime > 0{
		boolQ.Filter(elastic.NewRangeQuery("timestamp").Gte(listPostParam.startTime))
	}else if listPostParam.endTime > 0 && listPostParam.startTime > 0{
		boolQ.Filter(elastic.NewRangeQuery("timestamp").Lte(listPostParam.endTime).Gte(listPostParam.startTime))
	}
	//注意：ip和hostname都需要实现模糊查询，ip通过wildcard实现，hostname通过match实现
	// match查询不一定完全符合，这里只是提供一种思路，比如用户填入内容为HF-DC-OP，同样会查出记录HF-DC-FR-001，这个就需要考虑分词了
	/*
	GET log/_analyze
	{
	  "field": "hostname",
	  "text":  "HF-DC-OP"
	}
	结果：
	{
	  "tokens" : [
	    {
	      "token" : "hf",
	      "start_offset" : 0,
	      "end_offset" : 2,
	      "type" : "<ALPHANUM>",
	      "position" : 0
	    },
	    {
	      "token" : "dc",
	      "start_offset" : 3,
	      "end_offset" : 5,
	      "type" : "<ALPHANUM>",
	      "position" : 1
	    },
	    {
	      "token" : "op",
	      "start_offset" : 6,
	      "end_offset" : 8,
	      "type" : "<ALPHANUM>",
	      "position" : 2
	    }
	  ]
	}
	 */
	if len(strings.TrimSpace(listPostParam.ip))>0{
		boolQ.Filter(elastic.NewWildcardQuery("ip.keyword",fmt.Sprintf("*%s*",listPostParam.ip)))//ip模糊查询
	}
	if len(strings.TrimSpace(listPostParam.hostname))>0{
		boolQ.Filter(elastic.NewMatchQuery("hostname",listPostParam.hostname).Operator("or"))//hostname模糊查询，试着将or调整为and看看结果
	}
	res, err = client.Search(indices).
		Query(boolQ).
		Sort("timestamp",false).
		From(listPostParam.page*listPostParam.size).
		Size(listPostParam.size).
		Do(context.Background()) //分页查询
	if err!=nil{
		panic(err)
	}

	//es查询结果处理：不清楚怎么处理结果，可以参考第三步的查询结果，是相互对应的
	var listRtn listRtn
	if res!=nil && res.Hits.TotalHits.Value > 0 {
		listRtn.total = res.Hits.TotalHits.Value
		for _, hit := range res.Hits.Hits {
			var log log
			err=json.Unmarshal(hit.Source,&log)
			if err !=nil{
				fmt.Println(err.Error())
				continue
			}
			log.id = hit.Id
			listRtn.logs = append(listRtn.logs, log)
		}
	}else{
		listRtn.total = 0
	}
	fmt.Printf("%v",listRtn)
}