Elk的查询使用方法
1 elk通过调用其api的使用方法
elasticsearch官网提供了这个cat的api方法，具体使用如下：
elasticsearch本身提供了9200端口（如果未做修改的话）,可以在es的ip和端口后加/_cat查看可用的cat api
以下为列举出的一般用法，更多用法可以在/_cat来查看
1.1 /_cat/indices?index=a_log_*&v 查询指定index的状态，documents数量，占用大小等信息
[root@es2-6 conf]# curl '192.168.243.155:9200/_cat/indices?index=logstash-2019.03&v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open logstash-2019.03 Un_vcYjaQSeB5RLiYXtLEw 5 1 1761866 0 673.2mb 337mb
1.2 /_cat/indices?v 查询所有index的状态，documents数量，占用大小，等信息
[root@es2-6 conf]# curl '192.168.243.155:9200/_cat/indices?v'
health status index uuid pri rep docs.count docs.deleted store.size pri.store.size
green open metricbeat-6.5.4-2019.03.03 vJag6asmS0ifG6Ty-AEj9Q 1 1 270319 0 86.6mb 43.3mb
green open .monitoring-es-6-2019.03.01 eLpZBY0cTk6XOgKuJNgsUQ 1 1 111017 128 133.2mb 67.2mb
green open .monitoring-es-6-2019.02.28 U7_Ct9HfTsCKYzB8H5QBng 1 1 493554 1436 559.8mb 279.9mb
green open metricbeat-6.5.2-2019.03.01 HZypdjWOSnCMwc8K6pjS0A 1 1 3145778 0 1.6gb 862.1mb
green open .monitoring-kibana-6-2019.03.01 RFE_LA3rTGSL4l8g50AWfA 1 1 15421 0 8.5mb 4.2mb
green open metricbeat-6.5.4-2019.03.01 TS8rSj6HSXWOy96va5U8lw 1 1 154844 0 50.6mb 25.4mb
green open logstash-2019.03 Un_vcYjaQSeB5RLiYXtLEw 5 1 1761676 0 670.5mb 337mb
green open metricbeat-6.5.2-2019.03.03 jfScLYzMSLiBC4_xi_OlZQ 1 1 5578728 0 2.9gb 1.4gb
green open .monitoring-kibana-6-2019.03.04 8aFHIaF4R8un1VV1AbtODw 1 1 1284 0 852kb 376.3kb
green open .monitoring-kibana-6-2019.03.02 1uiWDq28T9eDun8KLMxu8A 1 1 17261 0 9.2mb 4.5mb
green open metricbeat-6.5.4-2019.03.04 mduHD1JoR_WG9jCZ7DsbWg 1 1 111434 0 35.5mb 17.7mb
green open metricbeat-6.5.2-2019.03.04 o0ng_Xb4QQOfKMi40z9AJg 1 1 2292421 0 1.6gb 677.6mb
green open .monitoring-kibana-6-2019.03.03 hmWFv9ykQEyJxZbXCOE5XQ 1 1 17258 0 7.4mb 3.6mb
green open metricbeat-6.5.2-2019.03.02 7YvwpDDdT1a4oAhXIt6lAw 1 1 5577032 0 3gb 1.4gb
green open .monitoring-es-6-2019.03.02 iVHyzaqvSoCTAMFEp1rQPQ 1 1 152831 234 195.2mb 97.4mb
green open .monitoring-es-6-2019.03.04 OulM1NwWQ7yLhRtoVNdpqg 1 1 16398 138 24.2mb 12.1mb
green open .monitoring-es-6-2019.03.03 RoeZ5g1uTM6fQM38GY0AOA 1 1 187386 420 249.8mb 124.9mb
green open .kibana dH7_KDcLTM27femts5bjoA 1 1 4 0 39.2kb 19.6kb
green open metricbeat-6.5.4-2019.03.02 vpFfj92gR8qiZ4t2iAAWzw 1 1 271854 0 87.2mb 43.5mb
1.3 /_cat/allocation?v 查询当前索引的filter以及routing所配置的别名信息
[root@es2-6 conf]# curl '192.168.243.155:9200/_cat/allocation?v'
shards disk.indices disk.used disk.avail disk.total disk.percent host ip node
16 5.4gb 23.1gb 26.7gb 49.9gb 46 172.31.182.104 172.31.182.104 es2-6.5-2
15 2.4gb 19.2gb 30.7gb 49.9gb 38 172.31.182.134 172.31.182.134 es2-6.5-3
15 3.4gb 29.6gb 20.3gb 49.9gb 59 192.168.243.155 192.168.243.155 es2-6.5-1
1.4 /_cat/count?v&index=a_log_* 查询index=a_log_* 的所有日志总数
[root@es2-6 conf]# curl '192.168.243.155:9200/_cat/count?v&index=logstash-2019.03'
epoch timestamp count
1551664576 01:56:16 1764178
1.5 查看别名接口(_cat/aliases): 查看索引别名
[root@es2-6 conf]# curl '192.168.243.155:9200/_cat/aliases/'
[root@es2-6 conf]#
1.6 查看分配资源接口(_cat/allocation)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/allocation
16 5.4gb 23.1gb 26.7gb 49.9gb 46 172.31.182.104 172.31.182.104 es2-6.5-2
15 3.4gb 29.6gb 20.3gb 49.9gb 59 192.168.243.155 192.168.243.155 es2-6.5-1
15 2.3gb 19.1gb 30.7gb 49.9gb 38 172.31.182.134 172.31.182.134 es2-6.5-3
1.7 查看文档个数接口(_cat/count)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/count
1551663506 01:38:26 20137956
1.8 查看字段分配情况接口(_cat/fielddata)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/health
1551663439 01:37:19 master green 3 3 46 23 0 0 0 0 - 100.0%
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/fielddata
IwenSyBeS16QE9Z7WBvnFQ 172.31.182.104 172.31.182.104 es2-6.5-2 kibana_stats.kibana.status 728b
IwenSyBeS16QE9Z7WBvnFQ 172.31.182.104 172.31.182.104 es2-6.5-2 shard.index 1.9kb
IwenSyBeS16QE9Z7WBvnFQ 172.31.182.104 172.31.182.104 es2-6.5-2 shard.node 544b
1.9 查看健康状态接口(_cat/health)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/health
1551663439 01:37:19 master green 3 3 46 23 0 0 0 0 - 100.0%
1.10 查看索引信息接口(_cat/indices)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/indices
green open metricbeat-6.5.4-2019.03.03 vJag6asmS0ifG6Ty-AEj9Q 1 1 270319 0 86.6mb 43.3mb
green open .monitoring-es-6-2019.03.01 eLpZBY0cTk6XOgKuJNgsUQ 1 1 111017 128 133.2mb 67.2mb
1.11 查看master信息接口(_cat/master)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/master
1r7UJOsPS-ufZyLGlaa2vw 192.168.243.155 192.168.243.155 es2-6.5-1
1.12 查看nodes信息接口(_cat/nodes)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/nodes
192.168.243.155 44 57 2 0.05 0.08 0.14 mdi * es2-6.5-1
172.31.182.104 51 98 3 0.02 0.12 0.19 di - es2-6.5-2
172.31.182.134 56 98 1 0.78 0.33 0.24 di - es2-6.5-3
1.13 查看正在挂起的任务接口(_cat/pending_tasks)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/pending_tasks
[root@es2-6 conf]#
1.14查看插件接口(_cat/plugins)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/plugins
[root@es2-6 conf]#
1.15 查看修复状态接口(_cat/recovery)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/recovery
.monitoring-es-6-2019.03.01 0 5.1s peer done 172.31.182.134 es2-6.5-3 192.168.243.155 es2-6.5-1 n/a n/a 32 32 100.0% 32 2285670 2285670 100.0% 2285670 8363 8363 100.0%
.monitoring-es-6-2019.03.01 0 59ms existing_store done n/a n/a 172.31.182.134 es2-6.5-3 n/a n/a 0 0 100.0% 32 0 0 100.0% 2285670 0 0 100.0%
.monitoring-es-6-2019.03.04 0 111ms empty_store done n/a n/a 172.31.182.104 es2-6.5-2 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0%
.monitoring-es-6-2019.03.04 0 257ms peer done 172.31.182.104 es2-6.5-2 192.168.243.155 es2-6.5-1 n/a n/a 1 1 100.0% 1 230 230 100.0% 230 1 1 100.0%
.monitoring-es-6-2019.03.03 0 111ms empty_store done n/a n/a 192.168.243.155 es2-6.5-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0%
.monitoring-es-6-2019.03.03 0 648ms peer done 192.168.243.155 es2-6.5-1 172.31.182.134 es2-6.5-3 n/a n/a 1 1 100.0% 1 230 230 100.0% 230 1 1 100.0%
.monitoring-es-6-2019.03.02 0 261ms peer done 172.31.182.134 es2-6.5-3 172.31.182.104 es2-6.5-2 n/a n/a 1 1 100.0% 1 230 230 100.0% 230 1 1 100.0%
.monitoring-es-6-2019.03.02 0 64ms empty_store done n/a n/a 172.31.182.134 es2-6.5-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0%
.monitoring-es-6-2019.02.28 0 350ms existing_store done n/a n/a 172.31.182.104 es2-6.5-2 n/a n/a 0 0 100.0% 93 0 0 100.0% 293511693 0 0 100.0%
.monitoring-es-6-2019.02.28 0 156ms peer done 172.31.182.104 es2-6.5-2 172.31.182.134 es2-6.5-3 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0%
metricbeat-6.5.4-2019.03.04 0 123ms empty_store done n/a n/a 192.168.243.155 es2-6.5-1 n/a n/a 0 0 0.0% 0 0 0 0.0% 0 0 0 100.0%
1.16查看线城池接口(_cat/thread_pool)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/thread_pool
es2-6.5-1 analyze 0 0 0
es2-6.5-1 ccr 0 0 0
es2-6.5-1 fetch_shard_started 0 0 0
es2-6.5-1 fetch_shard_store 0 0 0
es2-6.5-1 flush 0 0 0
es2-6.5-1 force_merge 0 0 0
es2-6.5-1 generic 0 0 0
es2-6.5-1 get 0 0 0
es2-6.5-1 index 0 0 0
es2-6.5-1 listener 0 0 0
es2-6.5-1 management 1 0 0
es2-6.5-1 ml_autodetect 0 0 0
es2-6.5-1 ml_datafeed 0 0 0
es2-6.5-1 ml_utility 0 0 0
es2-6.5-1 refresh 0 0 0
es2-6.5-1 rollup_indexing 0 0 0
es2-6.5-1 search 0 0 0
es2-6.5-1 search_throttled 0 0 0
es2-6.5-1 security-token-key 0 0 0
es2-6.5-1 snapshot 0 0 0
es2-6.5-1 warmer 0 0 0
es2-6.5-1 watcher 0 0 0
es2-6.5-1 write 0 0 0
es2-6.5-2 analyze 0 0 0
es2-6.5-2 ccr 0 0 0
es2-6.5-2 fetch_shard_started 0 0 0
es2-6.5-2 fetch_shard_store 0 0 0
es2-6.5-2 flush 0 0 0
es2-6.5-2 force_merge 0 0 0
es2-6.5-2 generic 0 0 0
es2-6.5-2 get 0 0 0
es2-6.5-2 index 0 0 0
es2-6.5-2 listener 0 0 0
es2-6.5-2 management 1 0 0
es2-6.5-2 ml_autodetect 0 0 0
es2-6.5-2 ml_datafeed 0 0 0
es2-6.5-2 ml_utility 0 0 0
es2-6.5-2 refresh 0 0 0
es2-6.5-2 rollup_indexing 0 0 0
es2-6.5-2 search 0 0 0
es2-6.5-2 search_throttled 0 0 0
es2-6.5-2 security-token-key 0 0 0
es2-6.5-2 snapshot 0 0 0
es2-6.5-2 warmer 0 0 0
es2-6.5-2 watcher 0 0 0
es2-6.5-2 write 0 0 0
es2-6.5-3 analyze 0 0 0
es2-6.5-3 ccr 0 0 0
es2-6.5-3 fetch_shard_started 0 0 0
es2-6.5-3 fetch_shard_store 0 0 0
es2-6.5-3 flush 0 0 0
es2-6.5-3 force_merge 0 0 0
es2-6.5-3 generic 0 0 0
es2-6.5-3 get 0 0 0
es2-6.5-3 index 0 0 0
es2-6.5-3 listener 0 0 0
es2-6.5-3 management 1 0 0
es2-6.5-3 ml_autodetect 0 0 0
es2-6.5-3 ml_datafeed 0 0 0
es2-6.5-3 ml_utility 0 0 0
es2-6.5-3 refresh 0 0 0
es2-6.5-3 rollup_indexing 0 0 0
es2-6.5-3 search 0 0 0
es2-6.5-3 search_throttled 0 0 0
es2-6.5-3 security-token-key 0 0 0
es2-6.5-3 snapshot 0 0 0
es2-6.5-3 warmer 0 0 0
es2-6.5-3 watcher 0 0 0
es2-6.5-3 write 0 0 0
1.17 查看分片信息接口(_cat/shards)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/shards
metricbeat-6.5.2-2019.03.02 0 p STARTED 5577032 1.4gb 172.31.182.104 es2-6.5-2
metricbeat-6.5.2-2019.03.02 0 r STARTED 5577032 1.5gb 192.168.243.155 es2-6.5-1
.kibana 0 r STARTED 4 19.6kb 172.31.182.104 es2-6.5-2
.kibana 0 p STARTED 4 19.6kb 192.168.243.155 es2-6.5-1
1.18查看lucence的段信息接口(_cat/segments)
[root@es2-6 conf]# curl -XGET http://192.168.243.155:9200/_cat/segments
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _hza 23302 98618 26 56.8mb 91334 true true 7.5.0 false
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jya 25858 3062 26 2.2mb 40476 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jyk 25868 3067 26 2.2mb 40463 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jyu 25878 3077 26 2.2mb 40578 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jz4 25888 3121 0 2.2mb 45620 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jz5 25889 1 0 28.5kb 4198 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jz6 25890 14 28 69.8kb 9941 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jz7 25891 1 0 28.5kb 4198 true true 7.5.0 true
.monitoring-es-6-2019.03.01 0 r 192.168.243.155 _jz8 25892 14 28 69.7kb 9941 true true 7.5.0 true
2 Kibana查询语句
2.1 关于时间戳字段的查询
下图为通过使用kibana的界面进行查询的结果
2.2.1 对于索引megacorp信息过滤的查询
GET /megacorp/employee/_search
{
"query": {
"bool" : {
"filter" : {
"range" : {
"age" : { "gt" : 30}
}
},
"must" :{
"match" :{
"last_name" : "Smith"
}
}
}
}
}
2.2.2 对于megacorp信息的聚合查询
GET /megacorp/employee/_search
{
"query":{
"match": {
"last_name": "smith"
}
},
"aggs":{
"all_interests":{
"terms":{"field":"interests"}
}
}
}
2.2.3 开启megacorp索引的fielddata字段数据
PUT megacorp/_mapping/employee/
{
"properties" : {
"interests": {
"type" : "text",
"fielddata": true
}
}
}
2.2.4 在kibana的dev tools中输入查询语句可以查询某一个索引的的详细信息，如输入以下GET /logstash-2019.03/doc/OfSWRmkB5FvT4MF8kDZg?pretty语句
在该查询结果中hits数组中每个结果都包含_index、_type和文档的_id字段，被加入到_source字段中这意味着在搜索结果中我们将可以直接使用全部文档，每个节点都有一个_score字段，这是相关性得分，它衡量了文档与查询的匹配程度，默认的返回的结果中关联性最大的文档排在首位，这意味着它是按照_score降序排列，这种情况下，我们没有指定任何查询，所以所有文档的相关性是一样的，因此所有结果的_score都是取得一个中间值1，max_score指的是所有文档匹配查询中_score的最大值。
Hits.total:
搜索到的document总条数
Hits.hits:
返回查询到的document的详细数据
_shard：
返回查询到的数据的分片数量
Took:
Took告诉我们整个搜索请求花费的毫秒数
Shards：
_shards节点告诉我们参与查询的分片数（total字段），有多少是成功的（successful字段），有多少是失败的（failed字段），通常我们不希望分片失败，不过这个有可能发生。如果我们遭受一些重大的故障导致主分片和复制分片都故障，那这个分片的数据将无法响应给搜索请求。这种情况下，elasticsearch将报告分片failed，但仍将继续返回剩余分片上的请求。
Timeout:
Time_out值告诉我们查询超时与否。一般的搜索请求不会超时，如果响应速度比完整的结果更重要，你可以定义timeout参数为10或者10ms(10毫秒)，或者1s（1秒）。
2.2.5 在kibana的dev tool编写查询语句如下
GET /logstash-2019.03/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-04T13:44:46.203+0800",
"lt": "2019-03-04T13:44:51.492+0800"
}
}
}
}
}
}
该语句查询index为logstash-2019.03，type为doc类型的文档，搜索范围为2019-03-04T13:44:46.203+0800到2019-03-04T13:44:51.492+0800，输出结果如下图所示
在如上查询结果的基础上如果想查找更具体的信息，如某个时间段内某个主机的信息，可以编写如下的查询语句，查询结果如下所示：
GET /logstash-2019.03/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-04T13:44:46.203+0800",
"lt": "2019-03-04T13:44:51.492+0800"
}
}
},
"must" : {
"match":{
"host.name": "test"
}
}
}
}
}
Term精确过滤
Term主要用于精确匹配哪些值，比如数字、日期、布尔值或者not_analyzed的字符串（未经分析的文本数据类型），查询语句如下所示，查询的是主机名称为ceph03的机器的日志
GET /logstash-2019.03/doc/_search
{
"query": {
"term": {
"host": "test"
}
}
}
Terms
Terms与term类似，但terms允许指定多个匹配条件，如果某个字段指定了多个值，那么文档需要一起去做匹配：
GET /logstash-2019.03/doc/_search
{
"query": {
"terms": {
"tags": ["beats_input_codec_plain_applied"]
}
}
}
Match_all查询所有文档，是没有查询条件下的默认语句
GET /_search
{
"query": {
"match_all": {
}
}
}
Match查询是一个标准查询，不管你需要全文本查询还是精确查询基本上都要用到它
GET /_search
{
"query": {
"match": {
"_id" : "space:default"
}
}
}
Wildcards查询
使用标准的shell通配符查询
GET /logstash-2019.03/doc/_search
{
"query": {
"wildcard": {
"host" : "co?t*"
}
}
}
prefix查询
只要匹配一某个字符或者某个字符串开头的，通过使用prefix可以查询
GET /logstash-2019.03/doc/_search
{
"query": {
"prefix":{
"host": "con*"
}
}
}
短语匹配（phrase matching）
当你需要寻找邻近的几个单词时，可以使用match_phrase查询
GET /logstash-2019.03/doc/_search
{
"query": {
"match_phrase": {
"host" : "test"
}
}
}
2.2.5 可以在kibana首页输入关键字也可以进行简单查询
2.2.6 查询特定机器特定时间范围进程之间的聚合分析
GET /metricbeat-6.5.2-2019.03.13/doc/_search
{
#“size”:0 , 改行脚本的含义是只显示具体的聚合分析的结果，中间的具体数据不需要
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-13T09:11:20.627+0800",
"lt": "2019-03-13T09:11:21.295+0800"
}
}
},
"must":{
"match":{
"host.name":"test"
}
}
}
},
"aggs": {
"cpu_process": {
"avg": {
"field": "system.process.cpu.total.pct"
}
}
}
}
2.2.7 查询固定时间范围内，固定宿主机，固定进程的聚合查询的结果
GET /metricbeat-6.5.2-2019.03.13/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-13T09:11:20.627+0800",
"lt": "2019-03-13T09:11:21.295+0800"
}
}
},
"must":[
{"term":{"host.name":"test"}},
{"term":{"system.process.name": "mysql"}}
]
}
},
"aggs": {
"cpu_process": {
"avg": {
"field": "system.process.cpu.total.pct"
}
}
}
}
2.2.8 查询某一时间段内根据某项指标进行降序排序的结果
GET /metricbeat-6.5.2-2019.03.13/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-13T09:11:20.627+0800",
"lt": "2019-03-13T09:11:21.295+0800"
}
}
},
"must":[
{"term":{"host.name":"test"}}
]
}
},
"aggs": {
"cpu_process": {
"avg": {
"field": "system.process.memory.rss.bytes"
}
}
},
"sort":{
"system.process.memory.rss.bytes":{
"order":"desc"
}
}
}
2.2.9 查询的日志信息只包含需要的字段信息
GET /metricbeat-6.5.2-2019.03.13/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-13T09:11:20.627+0800",
"lt": "2019-03-13T09:11:21.295+0800"
}
}
},
"must":[
{"term":{"host.name":"test"}}
]
}
},
"aggs": {
"cpu_process": {
"avg": {
"field": "system.process.memory.rss.bytes"
}
}
},
"sort":{
"system.process.memory.rss.bytes":{
"order":"desc"
}
},
"_source": {"includes": ["system.process.memory.rss.bytes"]}
}
2.2.10 查询的日志只包含多个字段
GET /metricbeat-6.5.2-2019.03.13/doc/_search
{
"query": {
"bool": {
"filter": {
"range": {
"@timestamp": {
"gte": "2019-03-13T09:11:20.627+0800",
"lt": "2019-03-13T09:11:21.295+0800"
}
}
},
"must":[
{"term":{"host.name":"test"}}
]
}
},
"aggs": {
"cpu_process": {
"avg": {
"field": "system.process.memory.rss.bytes"
}
}
},
"sort":{
"system.process.memory.rss.bytes":{
"order":"desc"
}
},
"_source": {"includes": ["system.process.memory.rss.bytes","system.process.name"]}
}
152
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
