elasticsearch之多索引查询

一、问题源起

在elasticsearch的查询中，我们一般直接通过URL来设置要search的index； 如果我们需要查询的索引比较多并且没有什么规律的话，就会面临一个尴尬的局面，超过URL的长度限制；

二、测试环境

elasticsearch 6.8.12

测试数据

新增三个测试的index，每个index里边一个document；

PUT test1/_doc/1
{
  "id":1,
  "name":"test1-1"
}


# {
#   "_index" : "test1",
#   "_type" : "_doc",
#   "_id" : "1",
#   "_version" : 1,
#   "result" : "created",
#   "_shards" : {
#     "total" : 2,
#     "successful" : 1,
#     "failed" : 0
#   },
#   "_seq_no" : 0,
#   "_primary_term" : 1
# }

PUT test2/_doc/1
{
  "id":1,
  "name":"test2-1"
}


# {
#   "_index" : "test2",
#   "_type" : "_doc",
#   "_id" : "1",
#   "_version" : 1,
#   "result" : "created",
#   "_shards" : {
#     "total" : 2,
#     "successful" : 1,
#     "failed" : 0
#   },
#   "_seq_no" : 0,
#   "_primary_term" : 1
# }

PUT test3/_doc/1
{
  "id":1,
  "name":"test3-1"
}

# {
#   "_index" : "test3",
#   "_type" : "_doc",
#   "_id" : "1",
#   "_version" : 1,
#   "result" : "created",
#   "_shards" : {
#     "total" : 2,
#     "successful" : 1,
#     "failed" : 0
#   },
#   "_seq_no" : 0,
#   "_primary_term" : 1
# }

三、URL中指定multi index

直接在URL中指定搜索特定的index

POST test1/_search 
{
    "query": {
        "match_all": {}
    }
}


# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 5,
#     "successful" : 5,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 1,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       }
#     ]
#   }
# }

可以通过都好分割同时搜索多个index；

POST test1,test2/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 1,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 10,
#     "successful" : 10,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 2,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       }
#     ]
#   }
# }

我们可以使用关键字_all指定搜索所有的index；

POST _all/_search 
{
    "query": {
        "match_all": {}
    }
}

{
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 15,
#     "successful" : 15,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 3,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       },
#       {
#         "_index" : "test3",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test3-1"
#         }
#       }
#     ]
#   }
# }

也可以使用通配符*来匹配一些名字有共同特征的index；

POST test*/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 1,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 15,
#     "successful" : 15,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 3,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       },
#       {
#         "_index" : "test3",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test3-1"
#         }
#       }
#     ]
#   }
# }

还可以使用-来排除某个index；

POST test*,-test2/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 10,
#     "successful" : 10,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 2,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test3",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test3-1"
#         }
#       }
#     ]
#   }
# }

四、URL中multi index的一些控制选项

如果我们显示search一个不存在的或者关闭的index就会报错；

POST test4/_search
{
    "query": {
        "match_all": {}
    }
}


# {
#   "error" : {
#     "root_cause" : [
#       {
#         "type" : "index_not_found_exception",
#         "reason" : "no such index",
#         "resource.type" : "index_or_alias",
#         "resource.id" : "test4",
#         "index_uuid" : "_na_",
#         "index" : "test4"
#       }
#     ],
#     "type" : "index_not_found_exception",
#     "reason" : "no such index",
#     "resource.type" : "index_or_alias",
#     "resource.id" : "test4",
#     "index_uuid" : "_na_",
#     "index" : "test4"
#   },
#   "status" : 404
# }

POST test3/_close
# 
# {
#   "acknowledged" : true
# }

POST test3/_search
{
    "query": {
        "match_all": {}
    }
}


# {
#   "error": {
#     "root_cause": [
#       {
#         "type": "index_closed_exception",
#         "reason": "closed",
#         "index_uuid": "KI7Iv4eGRIOk6MsycXokNQ",
#         "index": "test3"
#       }
#     ],
#     "type": "index_closed_exception",
#     "reason": "closed",
#     "index_uuid": "KI7Iv4eGRIOk6MsycXokNQ",
#     "index": "test3"
#   },
#   "status": 400
# }

我们可以使用ignore_unavailable来忽略不存在或者关闭的index；

POST test4/_search?ignore_unavailable=true
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 0,
#     "successful" : 0,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 0,
#     "max_score" : 0.0,
#     "hits" : [ ]
#   }
# }


POST test3/_search?ignore_unavailable=true
{
    "query": {
        "match_all": {}
    }
}


# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 0,
#     "successful" : 0,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 0,
#     "max_score" : 0.0,
#     "hits" : [ ]
#   }
# }

如果通过通配符、_all隐式的指定search的index，如果不存在则默认不会报错，不过可以通过allow_no_indices=false来让elasticsearch报错；

POST noexist*/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 0,
#     "successful" : 0,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 0,
#     "max_score" : 0.0,
#     "hits" : [ ]
#   }
# }


POST noexist*/_search?allow_no_indices=false
{
    "query": {
        "match_all": {}
    }
}

# {
#   "error" : {
#     "root_cause" : [
#       {
#         "type" : "index_not_found_exception",
#         "reason" : "no such index",
#         "resource.type" : "index_or_alias",
#         "resource.id" : "noexist*",
#         "index_uuid" : "_na_",
#         "index" : "noexist*"
#       }
#     ],
#     "type" : "index_not_found_exception",
#     "reason" : "no such index",
#     "resource.type" : "index_or_alias",
#     "resource.id" : "noexist*",
#     "index_uuid" : "_na_",
#     "index" : "noexist*"
#   },
#   "status" : 404
# }



POST test3*/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 0,
#     "successful" : 0,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 0,
#     "max_score" : 0.0,
#     "hits" : [ ]
#   }
# }

POST test3*/_search?allow_no_indices=false
{
    "query": {
        "match_all": {}
    }
}

# {
#   "error" : {
#     "root_cause" : [
#       {
#         "type" : "index_not_found_exception",
#         "reason" : "no such index",
#         "resource.type" : "index_or_alias",
#         "resource.id" : "test3*"
#       }
#     ],
#     "type" : "index_not_found_exception",
#     "reason" : "no such index",
#     "resource.type" : "index_or_alias",
#     "resource.id" : "test3*"
#   },
#   "status" : 404
# }

我们也可以使用expand_wildcards来控制展开哪些index，可选值open、closed、none、all；

默认只扩展open；

POST test*/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 10,
#     "successful" : 10,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 2,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       }
#     ]
#   }
# }


POST test*/_search?expand_wildcards=all
{
    "query": {
        "match_all": {}
    }
}

# {
#   "error": {
#     "root_cause": [
#       {
#         "type": "index_closed_exception",
#         "reason": "closed",
#         "index_uuid": "KI7Iv4eGRIOk6MsycXokNQ",
#         "index": "test3"
#       }
#     ],
#     "type": "index_closed_exception",
#     "reason": "closed",
#     "index_uuid": "KI7Iv4eGRIOk6MsycXokNQ",
#     "index": "test3"
#   },
#   "status": 400
# }

POST test*/_search?expand_wildcards=all&ignore_unavailable=true
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 10,
#     "successful" : 10,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 2,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       }
#     ]
#   }
# }

五、使用index aliases封装物理index

aliases是物理索引的别名，请求api的时候，elasticsearch会自动将aliases转化为对应的物理index name；

别名既可以映射到某个特定的index，也可以映射到多个index；

别名也可以同时应用过滤条件，实现只对index的局部数据进行搜索；

POST /_aliases
{
    "actions" : [
        { "add" : { "index" : "test*", "alias" : "all_test_indices" } }
    ]
}

# {
#   "acknowledged" : true
# }

POST all_test_indices/_search
{
    "query": {
        "match_all": {}
    }
}

# {
#   "took" : 0,
#   "timed_out" : false,
#   "_shards" : {
#     "total" : 10,
#     "successful" : 10,
#     "skipped" : 0,
#     "failed" : 0
#   },
#   "hits" : {
#     "total" : 2,
#     "max_score" : 1.0,
#     "hits" : [
#       {
#         "_index" : "test1",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test1-1"
#         }
#       },
#       {
#         "_index" : "test2",
#         "_type" : "_doc",
#         "_id" : "1",
#         "_score" : 1.0,
#         "_source" : {
#           "id" : 1,
#           "name" : "test2-1"
#         }
#       }
#     ]
#   }
# }

六、multi search–通过body指定index

Multi Search API的主要目的是实现在一个API里边实现多个search请求，其通过如下格式分别通过header指定index，body指定查询语句；

header\n
body\n
header\n
body\n

Multi Search API除了与前两者具有相同的指定index name的能力，最大的优势就是通过body传递index name，轻松突破URL的长度限制的局限性；

还有一点就是Multi Search API支持大量的没有特定规律的index name，例如跟时间序列有关的index name等；

GET _msearch
{"index":"test*"}
{"query" : {"match_all" : {}}}

# {
#   "responses" : [
#     {
#       "took" : 0,
#       "timed_out" : false,
#       "_shards" : {
#         "total" : 10,
#         "successful" : 10,
#         "skipped" : 0,
#         "failed" : 0
#       },
#       "hits" : {
#         "total" : 2,
#         "max_score" : 1.0,
#         "hits" : [
#           {
#             "_index" : "test1",
#             "_type" : "_doc",
#             "_id" : "1",
#             "_score" : 1.0,
#             "_source" : {
#               "id" : 1,
#               "name" : "test1-1"
#             }
#           },
#           {
#             "_index" : "test2",
#             "_type" : "_doc",
#             "_id" : "1",
#             "_score" : 1.0,
#             "_source" : {
#               "id" : 1,
#               "name" : "test2-1"
#             }
#           }
#         ]
#       },
#       "status" : 200
#     }
#   ]
# }