Elasticsearch指标监控说明
指标维度
可以把指标分为6个维度。
下面的表格总结了上一篇文章中提到搜索性能，索引性能，内存性能，网络性能对应的ES API。其中有些性能数据是从多个维度描述的，比如搜索性能在节点维度和索引维度都有提供。
| Metric category | Availability | Description |
|---|---|---|
| Search performance metrics | Node Stats API, Index Stats API | 查询性能指标 |
| Indexing performance metrics | Node Stats API, Index Stats API | 写入性能指标 |
| Memory and garbage collection | Node Stats API, Cluster Stats API | 内存与GC性能指标 |
| Network metrics | Node Stats API | 网络情况 |
| Cluster health and node availability | Cluster Health API | 集群健康和节点的可用性指标 |
| Resource saturation and errors | Node Stats API, Index Stats API, Cluster Stats API, Pending Tasks API | 资源状态和错误信息指标 |
指标分级
Elasticsearch集群监控状态指标分三个级别
集群级别
集群级别的监控主要是针对整个Elasticsearch集群来说，包括集群的健康状况、集群的状态等。
节点级别
节点级别的监控主要是针对每个Elasticsearch实例的监控，其中包括每个实例的查询索引指标和物理资源使用指标。
索引级别
索引级别的监控主要是针对每个索引来说，主要包括每个索引的性能指标。
指标分布如图
API使用示例
主要有五个HTTP REST API可用于采集Elasticsearch监控指标：
- Cluster Health API
- Cluster Stats API
- Node Stats API
- Index Stats API
- Pending Tasks API
集群健康情况
GET /_cluster/health?human&pretty
结果示例
{
"cluster_name": "es-platform-log",
"status": "green",
"timed_out": false,
"number_of_nodes": 9,
"number_of_data_nodes": 6,
"active_primary_shards": 80,
"active_shards": 166,
"relocating_shards": 0,
"initializing_shards": 0,
"unassigned_shards": 0,
"delayed_unassigned_shards": 0,
"number_of_pending_tasks": 0,
"number_of_in_flight_fetch": 0,
"task_max_waiting_in_queue": "0s",
"task_max_waiting_in_queue_millis": 0,
"active_shards_percent": "100.0%",
"active_shards_percent_as_number": 100
}
统计集群信息
GET /_cluster/stats?human&pretty
结果示例
{
"_nodes": {
"total": 9,
"successful": 9,
"failed": 0
},
"cluster_name": "es-platform-log",
"cluster_uuid": "M0qBb884SWCYd8KjQcpFQg",
"timestamp": 1625657783874,
"status": "green",
"indices": {
"count": 34,
"shards": {
"total": 166,
"primaries": 80,
"replication": 1.075,
"index": {
"shards": {
"min": 2,
"max": 20,
"avg": 4.882352941176471
},
"primaries": {
"min": 1,
"max": 5,
"avg": 2.3529411764705883
},
"replication": {
"min": 0,
"max": 3,
"avg": 1
}
}
},
"docs": {
"count": 794178337,
"deleted": 8595881
},
"store": {
"size": "813.1gb",
"size_in_bytes": 873069900311
},
"fielddata": {
"memory_size": "1.7kb",
"memory_size_in_bytes": 1840,
"evictions": 0
},
"query_cache": {
"memory_size": "1gb",
"memory_size_in_bytes": 1099581304,
"total_count": 3012012,
"hit_count": 2854064,
"miss_count": 157948,
"cache_size": 3570,
"cache_count": 5415,
"evictions": 1845
},
"completion": {
"size": "0b",
"size_in_bytes": 0
},
"segments": {
"count": 659,
"memory": "898.8mb",
"memory_in_bytes": 942513495,
"terms_memory": "387.5mb",
"terms_memory_in_bytes": 406388453,
"stored_fields_memory": "483.7mb",
"stored_fields_memory_in_bytes": 507295448,
"term_vectors_memory": "0b",
"term_vectors_memory_in_bytes": 0,
"norms_memory": "18kb",
"norms_memory_in_bytes": 18432,
"points_memory": "22.9mb",
"points_memory_in_bytes": 24036474,
"doc_values_memory": "4.5mb",
"doc_values_memory_in_bytes": 4774688,
"index_writer_memory": "9.7mb",
"index_writer_memory_in_bytes": 10228272,
"version_map_memory": "260b",
"version_map_memory_in_bytes": 260,
"fixed_bit_set": "733.2kb",
"fixed_bit_set_memory_in_bytes": 750848,
"max_unsafe_auto_id_timestamp": 1625616004589,
"file_sizes": {}
}
},
"nodes": {
"count": {
"total": 9,
"data": 6,
"coordinating_only": 0,
"master": 3,
"ingest": 9
},
"versions": [
"6.7.2"
],
"os": {
"available_processors": 360,
"allocated_processors": 360,
"names": [
{
"name": "Linux",
"count": 9
}
],
"pretty_names": [
{
"pretty_name": "CentOS Linux 7 (Core)",
"count": 9
}
],
"mem": {
"total": "1.1tb",
"total_in_bytes": 1211755806720,
"free": "21.7gb",
"free_in_bytes": 23325560832,
"used": "1tb",
"used_in_bytes": 1188430245888,
"free_percent": 2,
"used_percent": 98
}
},
"process": {
"cpu": {
"percent": 0
},
"open_file_descriptors": {
"min": 869,
"max": 1000,
"avg": 948
}
},
"jvm": {
"max_uptime": "297.7d",
"max_uptime_in_millis": 25724732930,
"versions": [
{
"version": "1.8.0_161",
"vm_name": "Java HotSpot(TM) 64-Bit Server VM",
"vm_version": "25.161-b12",
"vm_vendor": "Oracle Corporation",
"count": 9
}
],
"mem": {
"heap_used": "135.7gb",
"heap_used_in_bytes": 145777776144,
"heap_max": "279gb",
"heap_max_in_bytes": 299573968896
},
"threads": 2654
},
"fs": {
"total": "43tb",
"total_in_bytes": 47300324622336,
"free": "42.6tb",
"free_in_bytes": 46859340689408,
"available": "42.1tb",
"available_in_bytes": 46376411246592
},
"plugins": [
{
"name": "analysis-pinyin",
"version": "6.7.2.4",
"elasticsearch_version": "6.7.2",
"java_version": "1.8",
"description": "Pinyin Analysis for Elasticsearch",
"classname": "org.elasticsearch.plugin.analysis.pinyin.AnalysisPinyinPlugin",
"extended_plugins": [],
"has_native_controller": false
},
{
"name": "analysis-dynamic-synonym",
"version": "6.7.2.4",
"elasticsearch_version": "6.7.2",
"java_version": "1.8",
"description": "Analysis-plugin for synonym",
"classname": "com.bellszhu.elasticsearch.plugin.DynamicSynonymPlugin",
"extended_plugins": [],
"has_native_controller": false
},
{
"name": "repository-sdoss",
"version": "6.7.2.4",
"elasticsearch_version": "6.7.2",
"java_version": "1.8",
"description": "The Sdoss repository plugin adds support for SDFS or other sdoss server repositories.",
"classname": "org.elasticsearch.repositories.sdoss.SdossPlugin",
"extended_plugins": [],
"has_native_controller": false
},
{
"name": "analysis-ik",
"version": "6.7.2.4",
"elasticsearch_version": "6.7.2",
"java_version": "1.8",
"description": "IK Analyzer for Elasticsearch",
"classname": "org.elasticsearch.plugin.analysis.ik.AnalysisIkPlugin",
"extended_plugins": [],
"has_native_controller": false
}
],
"network_types": {
"transport_types": {
"netty4": 9
},
"http_types": {
"netty4": 9
}
}
}
}
统计索引信息
GET /_stats
或
GET /index1,index2/_stats
可以在URL中增减参数，进行部分指标的统计 ，如GET /_stats/merge,refresh。
| 参数值 | 说明 |
|---|---|
docs | The number of docs / deleted docs (docs not yet merged out). Note, affected by refreshing the index. |
store | The size of the index. |
indexing | Indexing statistics, can be combined with a comma separated list of types to provide document type level stats. |
get | Get statistics, including missing stats. |
search | Search statistics including suggest statistics. You can include statistics for custom groups by adding an extra groups parameter (search operations can be associated with one or more groups). The groups parameter accepts a comma separated list of group names. Use _all to return statistics for all groups. |
segments | Retrieve the memory use of the open segments. Optionally, setting the include_segment_file_sizes flag, report the aggregated disk usage of each one of the Lucene index files. |
completion | Completion suggest statistics. |
fielddata | Fielddata statistics. |
flush | Flush statistics. |
merge | Merge statistics. |
request_cache | Shard request cache statistics. |
refresh | Refresh statistics. |
warmer | Warmer statistics. |
translog | Translog statistics. |
结果示例
{
"_shards": {
"total": 4,
"successful": 4,
"failed": 0
},
"_all": {
"primaries": {
"docs": {
"count": 791173931,
"deleted": 8580767
},
"store": {
"size_in_bytes": 437042050823
},
"indexing": {
"index_total": 1005140339,
"index_time_in_millis": 234948689,
"index_current": 0,
"index_failed": 0,
"delete_total": 458968359,
"delete_time_in_millis": 90953371,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 527285,
"query_time_in_millis": 38479084,
"query_current": 0,
"fetch_total": 307477,
"fetch_time_in_millis": 7206781,
"fetch_current": 0,
"scroll_total": 51,
"scroll_time_in_millis": 358349299,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 2973208,
"total_time_in_millis": 972619673,
"total_docs": 14744942732,
"total_size_in_bytes": 7712062295680,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 530615641,
"total_auto_throttle_in_bytes": 10485760
},
"refresh": {
"total": 26691087,
"total_time_in_millis": 474628597,
"listeners": 0
},
"flush": {
"total": 3305,
"periodic": 3301,
"total_time_in_millis": 1497051
},
"warmer": {
"current": 0,
"total": 26662293,
"total_time_in_millis": 493330
},
"query_cache": {
"memory_size_in_bytes": 443995504,
"total_count": 2037568,
"hit_count": 1964092,
"miss_count": 73476,
"cache_size": 1436,
"cache_count": 2735,
"evictions": 1299
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 138,
"memory_in_bytes": 467411612,
"terms_memory_in_bytes": 202202784,
"stored_fields_memory_in_bytes": 254003248,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 8832,
"points_memory_in_bytes": 11182860,
"doc_values_memory_in_bytes": 13888,
"index_writer_memory_in_bytes": 665540,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1619778779940,
"file_sizes": {}
},
"translog": {
"operations": 639821,
"size_in_bytes": 1156652496,
"uncommitted_operations": 297497,
"uncommitted_size_in_bytes": 619765088,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 39,
"miss_count": 90
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1331100
}
},
"total": {
"docs": {
"count": 1582347865,
"deleted": 14594676
},
"store": {
"size_in_bytes": 869264426131
},
"indexing": {
"index_total": 2014175604,
"index_time_in_millis": 479642467,
"index_current": 0,
"index_failed": 0,
"delete_total": 754975511,
"delete_time_in_millis": 169701198,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 785988,
"query_time_in_millis": 51228008,
"query_current": 0,
"fetch_total": 466539,
"fetch_time_in_millis": 10669770,
"fetch_current": 0,
"scroll_total": 76,
"scroll_time_in_millis": 530816494,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 6413503,
"total_time_in_millis": 2003883284,
"total_docs": 29718572943,
"total_size_in_bytes": 15853747835184,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 1054369010,
"total_auto_throttle_in_bytes": 20971520
},
"refresh": {
"total": 57623183,
"total_time_in_millis": 915566697,
"listeners": 0
},
"flush": {
"total": 6556,
"periodic": 6552,
"total_time_in_millis": 3071668
},
"warmer": {
"current": 0,
"total": 57573088,
"total_time_in_millis": 1042909
},
"query_cache": {
"memory_size_in_bytes": 1099581304,
"total_count": 3011984,
"hit_count": 2854056,
"miss_count": 157928,
"cache_size": 3570,
"cache_count": 5411,
"evictions": 1841
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 278,
"memory_in_bytes": 932203584,
"terms_memory_in_bytes": 403395911,
"stored_fields_memory_in_bytes": 506429088,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 17792,
"points_memory_in_bytes": 22322197,
"doc_values_memory_in_bytes": 38596,
"index_writer_memory_in_bytes": 1329712,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1619778779940,
"file_sizes": {}
},
"translog": {
"operations": 1287083,
"size_in_bytes": 2326543009,
"uncommitted_operations": 469588,
"uncommitted_size_in_bytes": 984353378,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 39,
"miss_count": 94
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1520232
}
}
},
"indices": {
"index_search_slowlog": {
"uuid": "5Byt_5F4S6yljXQjd50BhQ",
"primaries": {
"docs": {
"count": 791173931,
"deleted": 8580767
},
"store": {
"size_in_bytes": 437042050823
},
"indexing": {
"index_total": 1005140339,
"index_time_in_millis": 234948689,
"index_current": 0,
"index_failed": 0,
"delete_total": 458968359,
"delete_time_in_millis": 90953371,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 527285,
"query_time_in_millis": 38479084,
"query_current": 0,
"fetch_total": 307477,
"fetch_time_in_millis": 7206781,
"fetch_current": 0,
"scroll_total": 51,
"scroll_time_in_millis": 358349299,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 2973208,
"total_time_in_millis": 972619673,
"total_docs": 14744942732,
"total_size_in_bytes": 7712062295680,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 530615641,
"total_auto_throttle_in_bytes": 10485760
},
"refresh": {
"total": 26691087,
"total_time_in_millis": 474628597,
"listeners": 0
},
"flush": {
"total": 3305,
"periodic": 3301,
"total_time_in_millis": 1497051
},
"warmer": {
"current": 0,
"total": 26662293,
"total_time_in_millis": 493330
},
"query_cache": {
"memory_size_in_bytes": 443995504,
"total_count": 2037568,
"hit_count": 1964092,
"miss_count": 73476,
"cache_size": 1436,
"cache_count": 2735,
"evictions": 1299
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 138,
"memory_in_bytes": 467411612,
"terms_memory_in_bytes": 202202784,
"stored_fields_memory_in_bytes": 254003248,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 8832,
"points_memory_in_bytes": 11182860,
"doc_values_memory_in_bytes": 13888,
"index_writer_memory_in_bytes": 665540,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1619778779940,
"file_sizes": {}
},
"translog": {
"operations": 639821,
"size_in_bytes": 1156652496,
"uncommitted_operations": 297497,
"uncommitted_size_in_bytes": 619765088,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 39,
"miss_count": 90
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1331100
}
},
"total": {
"docs": {
"count": 1582347865,
"deleted": 14594676
},
"store": {
"size_in_bytes": 869264426131
},
"indexing": {
"index_total": 2014175604,
"index_time_in_millis": 479642467,
"index_current": 0,
"index_failed": 0,
"delete_total": 754975511,
"delete_time_in_millis": 169701198,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 785988,
"query_time_in_millis": 51228008,
"query_current": 0,
"fetch_total": 466539,
"fetch_time_in_millis": 10669770,
"fetch_current": 0,
"scroll_total": 76,
"scroll_time_in_millis": 530816494,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 6413503,
"total_time_in_millis": 2003883284,
"total_docs": 29718572943,
"total_size_in_bytes": 15853747835184,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 1054369010,
"total_auto_throttle_in_bytes": 20971520
},
"refresh": {
"total": 57623183,
"total_time_in_millis": 915566697,
"listeners": 0
},
"flush": {
"total": 6556,
"periodic": 6552,
"total_time_in_millis": 3071668
},
"warmer": {
"current": 0,
"total": 57573088,
"total_time_in_millis": 1042909
},
"query_cache": {
"memory_size_in_bytes": 1099581304,
"total_count": 3011984,
"hit_count": 2854056,
"miss_count": 157928,
"cache_size": 3570,
"cache_count": 5411,
"evictions": 1841
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 278,
"memory_in_bytes": 932203584,
"terms_memory_in_bytes": 403395911,
"stored_fields_memory_in_bytes": 506429088,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 17792,
"points_memory_in_bytes": 22322197,
"doc_values_memory_in_bytes": 38596,
"index_writer_memory_in_bytes": 1329712,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": 1619778779940,
"file_sizes": {}
},
"translog": {
"operations": 1287083,
"size_in_bytes": 2326543009,
"uncommitted_operations": 469588,
"uncommitted_size_in_bytes": 984353378,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 39,
"miss_count": 94
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1520232
}
}
}
}
}
索引恢复统计
索引恢复API提供了对正在进行的索引分片恢复的深入了解。 可以报告特定索引或群集范围的恢复状态。
GET index1,index2/_recovery?human
统计节点指标信息
按节点分别统计
GET /_nodes/nodeId1,nodeId2/stats
统计全部节点
GET /_nodes/stats
可以使用在URL 增加参数控制返回的指标项，如GET /_nodes/stats/os,process
| 参数值 | 说明 |
|---|---|
indices | Indices stats about size, document count, indexing and deletion times, search times, field cache size, merges and flushes |
fs | File system information, data path, free disk space, read/write stats (see FS information) |
http | HTTP connection information |
jvm | JVM stats, memory pool information, garbage collection, buffer pools, number of loaded/unloaded classes |
os | Operating system stats, load average, mem, swap (see OS statistics) |
process | Process statistics, memory consumption, cpu usage, open file descriptors (see Process statistics) |
thread_pool | Statistics about each thread pool, including current size, queue and rejected tasks |
transport | Transport statistics about sent and received bytes in cluster communication |
breaker | Statistics about the field data circuit breaker |
discovery | Statistics about the discovery |
ingest | Statistics about ingest preprocessing |
adaptive_selection | Statistics about adaptive replica selection. See adaptive selection statistics. |
结果示例
{
"_nodes": {
"total": 3,
"successful": 3,
"failed": 0
},
"cluster_name": "es-platform-log",
"nodes": {
"8L_IVOfFQXWLJ9a3gWNbPA": {
"timestamp": 1625658351545,
"name": "bdes02-platform-log-prd-master",
"transport_address": "192.168.1.1:9501",
"host": "host1",
"ip": "192.168.1.1:9501",
"roles": [
"master",
"ingest"
],
"attributes": {
"xpack.installed": "true"
},
"indices": {
"docs": {
"count": 0,
"deleted": 0
},
"store": {
"size_in_bytes": 0
},
"indexing": {
"index_total": 0,
"index_time_in_millis": 0,
"index_current": 0,
"index_failed": 0,
"delete_total": 0,
"delete_time_in_millis": 0,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 0,
"time_in_millis": 0,
"exists_total": 0,
"exists_time_in_millis": 0,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 0,
"query_time_in_millis": 0,
"query_current": 0,
"fetch_total": 0,
"fetch_time_in_millis": 0,
"fetch_current": 0,
"scroll_total": 0,
"scroll_time_in_millis": 0,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 0,
"total_time_in_millis": 0,
"total_docs": 0,
"total_size_in_bytes": 0,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 0,
"total_auto_throttle_in_bytes": 0
},
"refresh": {
"total": 0,
"total_time_in_millis": 0,
"listeners": 0
},
"flush": {
"total": 0,
"periodic": 0,
"total_time_in_millis": 0
},
"warmer": {
"current": 0,
"total": 0,
"total_time_in_millis": 0
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 0,
"hit_count": 0,
"miss_count": 0,
"cache_size": 0,
"cache_count": 0,
"evictions": 0
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 0,
"memory_in_bytes": 0,
"terms_memory_in_bytes": 0,
"stored_fields_memory_in_bytes": 0,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 0,
"doc_values_memory_in_bytes": 0,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -9223372036854776000,
"file_sizes": {}
},
"translog": {
"operations": 0,
"size_in_bytes": 0,
"uncommitted_operations": 0,
"uncommitted_size_in_bytes": 0,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 0
}
},
"os": {
"timestamp": 1625658351547,
"cpu": {
"percent": 0,
"load_average": {
"1m": 0.47,
"5m": 0.25,
"15m": 0.28
}
},
"mem": {
"total_in_bytes": 134639534080,
"free_in_bytes": 4437078016,
"used_in_bytes": 130202456064,
"free_percent": 3,
"used_percent": 97
},
"swap": {
"total_in_bytes": 10737414144,
"free_in_bytes": 8169005056,
"used_in_bytes": 2568409088
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 49022249544214930
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "125844779008"
}
}
},
"process": {
"timestamp": 1625658351548,
"open_file_descriptors": 869,
"max_file_descriptors": 65536,
"cpu": {
"percent": 0,
"total_in_millis": 400659630
},
"mem": {
"total_virtual_in_bytes": 55508516864
}
},
"jvm": {
"timestamp": 1625658351548,
"uptime_in_millis": 25725228648,
"mem": {
"heap_used_in_bytes": 17088622792,
"heap_used_percent": 51,
"heap_committed_in_bytes": 33285996544,
"heap_max_in_bytes": 33285996544,
"non_heap_used_in_bytes": 143388120,
"non_heap_committed_in_bytes": 154206208,
"pools": {
"young": {
"used_in_bytes": 15468593152,
"max_in_bytes": 0,
"peak_used_in_bytes": 19956498432,
"peak_max_in_bytes": 0
},
"survivor": {
"used_in_bytes": 75497472,
"max_in_bytes": 0,
"peak_used_in_bytes": 226492416,
"peak_max_in_bytes": 0
},
"old": {
"used_in_bytes": 1544532168,
"max_in_bytes": 33285996544,
"peak_used_in_bytes": 1548961320,
"peak_max_in_bytes": 33285996544
}
}
},
"threads": {
"count": 202,
"peak_count": 203
},
"gc": {
"collectors": {
"young": {
"collection_count": 2496,
"collection_time_in_millis": 136048
},
"old": {
"collection_count": 0,
"collection_time_in_millis": 0
}
}
},
"buffer_pools": {
"direct": {
"count": 250,
"used_in_bytes": 1352714432,
"total_capacity_in_bytes": 1352714431
},
"mapped": {
"count": 0,
"used_in_bytes": 0,
"total_capacity_in_bytes": 0
}
},
"classes": {
"current_loaded_count": 14999,
"total_loaded_count": 14999,
"total_unloaded_count": 0
}
},
"thread_pool": {
"analyze": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_started": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_store": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"flush": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 5,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 33438567
},
"get": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"index": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"listener": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 30672080
},
"refresh": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"rollup_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"scroll": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"search": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"search_throttled": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snccr": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"write": {
"threads": 20,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 20,
"completed": 2572364
}
},
"fs": {
"timestamp": 1625658351549,
"total": {
"total_in_bytes": 52710309888,
"free_in_bytes": 51881930752,
"available_in_bytes": 49180798976
},
"data": [
{
"path": "/home/es/masterdata/nodes/0",
"mount": "/home (/dev/mapper/systemvg-homelv)",
"type": "ext4",
"total_in_bytes": 52710309888,
"free_in_bytes": 51881930752,
"available_in_bytes": 49180798976
}
],
"io_stats": {
"devices": [
{
"device_name": "dm-6",
"operations": 12629460,
"read_operations": 30170,
"write_operations": 12599290,
"read_kilobytes": 235072,
"write_kilobytes": 50528812
}
],
"total": {
"operations": 12629460,
"read_operations": 30170,
"write_operations": 12599290,
"read_kilobytes": 235072,
"write_kilobytes": 50528812
}
}
},
"transport": {
"server_open": 94,
"rx_count": 235177654,
"rx_size_in_bytes": 1235206551899,
"tx_count": 235177656,
"tx_size_in_bytes": 782183874119
},
"http": {
"current_open": 2,
"total_opened": 2155604
},
"breakers": {
"request": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 757,
"estimated_size": "757b",
"overhead": 1,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 23300197580,
"limit_size": "21.6gb",
"estimated_size_in_bytes": 757,
"estimated_size": "757b",
"overhead": 1,
"tripped": 0
}
},
"script": {
"compilations": 1,
"cache_evictions": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 1,
"incompatible_diffs": 0,
"compatible_diffs": 24
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_2": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"script": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"rename": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"set": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"gsub": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
]
},
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": []
}
}
},
"adaptive_selection": {}
},
"swBMho--QCiUl2uaoYmtlg": {
"timestamp": 1625658351545,
"name": "bdes02-platform-log-prd-dnode1",
"transport_address": "192.168.1.1:9611",
"host": "host2",
"ip": "192.168.1.1:9611",
"roles": [
"data",
"ingest"
],
"attributes": {
"xpack.installed": "true"
},
"indices": {
"docs": {
"count": 395574024,
"deleted": 1721339
},
"store": {
"size_in_bytes": 215695623629
},
"indexing": {
"index_total": 870256500,
"index_time_in_millis": 216749357,
"index_current": 0,
"index_failed": 0,
"delete_total": 466942043,
"delete_time_in_millis": 91348254,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 1154559,
"time_in_millis": 66722,
"exists_total": 1154008,
"exists_time_in_millis": 66619,
"missing_total": 551,
"missing_time_in_millis": 103,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 5131328,
"query_time_in_millis": 38868703,
"query_current": 0,
"fetch_total": 4886228,
"fetch_time_in_millis": 7345419,
"fetch_current": 0,
"scroll_total": 116,
"scroll_time_in_millis": 359183189,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 2533891,
"total_time_in_millis": 905697740,
"total_docs": 12775267780,
"total_size_in_bytes": 6984542865196,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 472591465,
"total_auto_throttle_in_bytes": 3653648821
},
"refresh": {
"total": 22770706,
"total_time_in_millis": 459299858,
"listeners": 0
},
"flush": {
"total": 3696,
"periodic": 2948,
"total_time_in_millis": 1445893
},
"warmer": {
"current": 0,
"total": 21066048,
"total_time_in_millis": 380306
},
"query_cache": {
"memory_size_in_bytes": 307500752,
"total_count": 2023494,
"hit_count": 1960810,
"miss_count": 62684,
"cache_size": 1017,
"cache_count": 2316,
"evictions": 1299
},
"fielddata": {
"memory_size_in_bytes": 1248,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 72,
"memory_in_bytes": 231242086,
"terms_memory_in_bytes": 99675599,
"stored_fields_memory_in_bytes": 126004008,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 4480,
"points_memory_in_bytes": 5552893,
"doc_values_memory_in_bytes": 5106,
"index_writer_memory_in_bytes": 328188,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 0,
"max_unsafe_auto_id_timestamp": -1,
"file_sizes": {}
},
"translog": {
"operations": 314352,
"size_in_bytes": 567227350,
"uncommitted_operations": 179035,
"uncommitted_size_in_bytes": 365902284,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 849,
"evictions": 0,
"hit_count": 39,
"miss_count": 221
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 408011
}
},
"os": {
"timestamp": 1625658351555,
"cpu": {
"percent": 0,
"load_average": {
"1m": 0.47,
"5m": 0.25,
"15m": 0.28
}
},
"mem": {
"total_in_bytes": 134639534080,
"free_in_bytes": 4437078016,
"used_in_bytes": 130202456064,
"free_percent": 3,
"used_percent": 97
},
"swap": {
"total_in_bytes": 10737414144,
"free_in_bytes": 8169005056,
"used_in_bytes": 2568409088
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 49022249564144610
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "125844779008"
}
}
},
"process": {
"timestamp": 1625658351556,
"open_file_descriptors": 981,
"max_file_descriptors": 65536,
"cpu": {
"percent": 0,
"total_in_millis": 1941138130
},
"mem": {
"total_virtual_in_bytes": 275270959104
}
},
"jvm": {
"timestamp": 1625658351556,
"uptime_in_millis": 25725129013,
"mem": {
"heap_used_in_bytes": 18656575544,
"heap_used_percent": 56,
"heap_committed_in_bytes": 33285996544,
"heap_max_in_bytes": 33285996544,
"non_heap_used_in_bytes": 187997384,
"non_heap_committed_in_bytes": 201261056,
"pools": {
"young": {
"used_in_bytes": 11534336000,
"max_in_bytes": 0,
"peak_used_in_bytes": 19956498432,
"peak_max_in_bytes": 0
},
"survivor": {
"used_in_bytes": 67108864,
"max_in_bytes": 0,
"peak_used_in_bytes": 444596224,
"peak_max_in_bytes": 0
},
"old": {
"used_in_bytes": 7055130680,
"max_in_bytes": 33285996544,
"peak_used_in_bytes": 7061443880,
"peak_max_in_bytes": 33285996544
}
}
},
"threads": {
"count": 351,
"peak_count": 355
},
"gc": {
"collectors": {
"young": {
"collection_count": 11872,
"collection_time_in_millis": 476529
},
"old": {
"collection_count": 0,
"collection_time_in_millis": 0
}
}
},
"buffer_pools": {
"direct": {
"count": 531,
"used_in_bytes": 1355206107,
"total_capacity_in_bytes": 1355206106
},
"mapped": {
"count": 281,
"used_in_bytes": 215629878453,
"total_capacity_in_bytes": 215629878453
}
},
"classes": {
"current_loaded_count": 17036,
"total_loaded_count": 17036,
"total_unloaded_count": 0
}
},
"thread_pool": {
"analyze": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_started": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_store": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 4,
"completed": 606
},
"flush": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 25155
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 6,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 6,
"completed": 49943010
},
"get": {
"threads": 40,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 40,
"completed": 1154559
},
"index": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"listener": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 282
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 887894177
},
"refresh": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 323532500
},
"rollup_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"scroll": {
"threads": 5,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 838721
},
"search": {
"threads": 80,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 80,
"completed": 5441828
},
"search_throttled": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snccr": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 14991780
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"write": {
"threads": 20,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 20,
"completed": 860089180
}
},
"fs": {
"timestamp": 1625658351558,
"total": {
"total_in_bytes": 23623807156224,
"free_in_bytes": 23406986366976,
"available_in_bytes": 23166872211456
},
"data": [
{
"path": "/data06/dnode1/nodes/0",
"mount": "/data06 (/dev/sdh)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3937208426496,
"available_in_bytes": 3897189400576
},
{
"path": "/data07/dnode1/nodes/0",
"mount": "/data07 (/dev/sdi)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3720944369664,
"available_in_bytes": 3680925343744
},
{
"path": "/data08/dnode1/nodes/0",
"mount": "/data08 (/dev/sdj)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3937208426496,
"available_in_bytes": 3897189400576
},
{
"path": "/data09/dnode1/nodes/0",
"mount": "/data09 (/dev/sdk)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3937208352768,
"available_in_bytes": 3897189326848
},
{
"path": "/data10/dnode1/nodes/0",
"mount": "/data10 (/dev/sdl)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3937208344576,
"available_in_bytes": 3897189318656
},
{
"path": "/data11/dnode1/nodes/0",
"mount": "/data11 (/dev/sdm)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3937208446976,
"available_in_bytes": 3897189421056
}
],
"io_stats": {
"devices": [
{
"device_name": "sdh",
"operations": 6061671,
"read_operations": 50806,
"write_operations": 6010865,
"read_kilobytes": 611312,
"write_kilobytes": 64505652
},
{
"device_name": "sdj",
"operations": 8052455,
"read_operations": 21219,
"write_operations": 8031236,
"read_kilobytes": 146196,
"write_kilobytes": 208720928
},
{
"device_name": "sdi",
"operations": 4541924400,
"read_operations": 67084658,
"write_operations": 4474839742,
"read_kilobytes": 859201652,
"write_kilobytes": 38659973616
},
{
"device_name": "sdm",
"operations": 15910770,
"read_operations": 15195,
"write_operations": 15895575,
"read_kilobytes": 123412,
"write_kilobytes": 250193548
},
{
"device_name": "sdk",
"operations": 8906985,
"read_operations": 3401,
"write_operations": 8903584,
"read_kilobytes": 13764,
"write_kilobytes": 88742756
},
{
"device_name": "sdl",
"operations": 7670617,
"read_operations": 22366,
"write_operations": 7648251,
"read_kilobytes": 253316,
"write_kilobytes": 196709232
}
],
"total": {
"operations": 4588526898,
"read_operations": 67197645,
"write_operations": 4521329253,
"read_kilobytes": 860349652,
"write_kilobytes": 39468845732
}
}
},
"transport": {
"server_open": 126,
"rx_count": 2630200220,
"rx_size_in_bytes": 1933150910178,
"tx_count": 2868995384,
"tx_size_in_bytes": 2811416415838
},
"http": {
"current_open": 2,
"total_opened": 692729
},
"breakers": {
"request": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 1248,
"estimated_size": "1.2kb",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 757,
"estimated_size": "757b",
"overhead": 1,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 231241876,
"estimated_size": "220.5mb",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 23300197580,
"limit_size": "21.6gb",
"estimated_size_in_bytes": 231243881,
"estimated_size": "220.5mb",
"overhead": 1,
"tripped": 0
}
},
"script": {
"compilations": 1,
"cache_evictions": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 1,
"incompatible_diffs": 0,
"compatible_diffs": 3472
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_2": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"script": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"rename": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"set": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"gsub": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
]
},
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": []
}
}
},
"adaptive_selection": {
"OY-X1-1aRjm4m92KTY0EPg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 81703134,
"avg_response_time_ns": 548246,
"rank": "0.5"
},
"1RF501M0TY2dt1Ir4aTZtA": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 178832,
"avg_response_time_ns": 799764,
"rank": "0.8"
},
"MKFeShkgQEClpoMYMJ9LoQ": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 765967,
"avg_response_time_ns": 683758,
"rank": "0.7"
},
"ZYXwMegIRjK4Romw625Juw": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 504257,
"avg_response_time_ns": 1074127,
"rank": "1.1"
},
"swBMho--QCiUl2uaoYmtlg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 1477685,
"avg_response_time_ns": 866463,
"rank": "0.9"
},
"PFDIsSN9R7aH1mDJ2f6ntQ": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 387462,
"avg_response_time_ns": 1625918,
"rank": "1.6"
}
}
},
"OY-X1-1aRjm4m92KTY0EPg": {
"timestamp": 1625658351545,
"name": "bdes02-platform-log-prd-dnode0",
"transport_address": "192.168.1.1:9601",
"host": "host3",
"ip": "192.168.1.1:9601",
"roles": [
"data",
"ingest"
],
"attributes": {
"xpack.installed": "true"
},
"indices": {
"docs": {
"count": 2561749,
"deleted": 12944
},
"store": {
"size_in_bytes": 1646778911
},
"indexing": {
"index_total": 656390678,
"index_time_in_millis": 228792708,
"index_current": 0,
"index_failed": 0,
"delete_total": 165433342,
"delete_time_in_millis": 45335974,
"delete_current": 0,
"noop_update_total": 0,
"is_throttled": false,
"throttle_time_in_millis": 0
},
"get": {
"total": 12,
"time_in_millis": 7,
"exists_total": 12,
"exists_time_in_millis": 7,
"missing_total": 0,
"missing_time_in_millis": 0,
"current": 0
},
"search": {
"open_contexts": 0,
"query_total": 230252,
"query_time_in_millis": 13807156,
"query_current": 0,
"fetch_total": 142095,
"fetch_time_in_millis": 2376137,
"fetch_current": 0,
"scroll_total": 60,
"scroll_time_in_millis": 126866030,
"scroll_current": 0,
"suggest_total": 0,
"suggest_time_in_millis": 0,
"suggest_current": 0
},
"merges": {
"current": 0,
"current_docs": 0,
"current_size_in_bytes": 0,
"total": 2562264,
"total_time_in_millis": 883360461,
"total_docs": 9459187666,
"total_size_in_bytes": 6193035313754,
"total_stopped_time_in_millis": 0,
"total_throttled_time_in_millis": 209239910,
"total_auto_throttle_in_bytes": 7477197639
},
"refresh": {
"total": 23443127,
"total_time_in_millis": 454258615,
"listeners": 0
},
"flush": {
"total": 2174,
"periodic": 1786,
"total_time_in_millis": 1051179
},
"warmer": {
"current": 0,
"total": 369945,
"total_time_in_millis": 80369
},
"query_cache": {
"memory_size_in_bytes": 0,
"total_count": 28,
"hit_count": 8,
"miss_count": 20,
"cache_size": 0,
"cache_count": 4,
"evictions": 4
},
"fielddata": {
"memory_size_in_bytes": 0,
"evictions": 0
},
"completion": {
"size_in_bytes": 0
},
"segments": {
"count": 117,
"memory_in_bytes": 4193343,
"terms_memory_in_bytes": 1147825,
"stored_fields_memory_in_bytes": 360912,
"term_vectors_memory_in_bytes": 0,
"norms_memory_in_bytes": 0,
"points_memory_in_bytes": 736370,
"doc_values_memory_in_bytes": 1948236,
"index_writer_memory_in_bytes": 0,
"version_map_memory_in_bytes": 0,
"fixed_bit_set_memory_in_bytes": 325552,
"max_unsafe_auto_id_timestamp": 1600659492685,
"file_sizes": {}
},
"translog": {
"operations": 653876,
"size_in_bytes": 681964368,
"uncommitted_operations": 383092,
"uncommitted_size_in_bytes": 399616918,
"earliest_last_modified_age": 0
},
"request_cache": {
"memory_size_in_bytes": 0,
"evictions": 0,
"hit_count": 0,
"miss_count": 0
},
"recovery": {
"current_as_source": 0,
"current_as_target": 0,
"throttle_time_in_millis": 1944663
}
},
"os": {
"timestamp": 1625658351555,
"cpu": {
"percent": 0,
"load_average": {
"1m": 0.47,
"5m": 0.25,
"15m": 0.28
}
},
"mem": {
"total_in_bytes": 134639534080,
"free_in_bytes": 4437078016,
"used_in_bytes": 130202456064,
"free_percent": 3,
"used_percent": 97
},
"swap": {
"total_in_bytes": 10737414144,
"free_in_bytes": 8169005056,
"used_in_bytes": 2568409088
},
"cgroup": {
"cpuacct": {
"control_group": "/",
"usage_nanos": 49022249566149550
},
"cpu": {
"control_group": "/",
"cfs_period_micros": 100000,
"cfs_quota_micros": -1,
"stat": {
"number_of_elapsed_periods": 0,
"number_of_times_throttled": 0,
"time_throttled_nanos": 0
}
},
"memory": {
"control_group": "/",
"limit_in_bytes": "9223372036854771712",
"usage_in_bytes": "125844779008"
}
}
},
"process": {
"timestamp": 1625658351556,
"open_file_descriptors": 985,
"max_file_descriptors": 65536,
"cpu": {
"percent": 0,
"total_in_millis": 2029369310
},
"mem": {
"total_virtual_in_bytes": 61236346880
}
},
"jvm": {
"timestamp": 1625658351557,
"uptime_in_millis": 25725141339,
"mem": {
"heap_used_in_bytes": 21190002472,
"heap_used_percent": 63,
"heap_committed_in_bytes": 33285996544,
"heap_max_in_bytes": 33285996544,
"non_heap_used_in_bytes": 192070824,
"non_heap_committed_in_bytes": 205127680,
"pools": {
"young": {
"used_in_bytes": 12532580352,
"max_in_bytes": 0,
"peak_used_in_bytes": 19956498432,
"peak_max_in_bytes": 0
},
"survivor": {
"used_in_bytes": 16777216,
"max_in_bytes": 0,
"peak_used_in_bytes": 419430400,
"peak_max_in_bytes": 0
},
"old": {
"used_in_bytes": 8640644904,
"max_in_bytes": 33285996544,
"peak_used_in_bytes": 8643342144,
"peak_max_in_bytes": 33285996544
}
}
},
"threads": {
"count": 321,
"peak_count": 323
},
"gc": {
"collectors": {
"young": {
"collection_count": 27478,
"collection_time_in_millis": 971570
},
"old": {
"collection_count": 0,
"collection_time_in_millis": 0
}
}
},
"buffer_pools": {
"direct": {
"count": 431,
"used_in_bytes": 1354353935,
"total_capacity_in_bytes": 1354353934
},
"mapped": {
"count": 245,
"used_in_bytes": 1628673550,
"total_capacity_in_bytes": 1628673550
}
},
"classes": {
"current_loaded_count": 17109,
"total_loaded_count": 17109,
"total_unloaded_count": 0
}
},
"thread_pool": {
"analyze": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_started": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"fetch_shard_store": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 606
},
"flush": {
"threads": 1,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 15574
},
"force_merge": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"generic": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 53050971
},
"get": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"index": {
"threads": 12,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 12,
"completed": 12
},
"listener": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"management": {
"threads": 5,
"queue": 0,
"active": 1,
"rejected": 0,
"largest": 5,
"completed": 440282886
},
"refresh": {
"threads": 10,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 10,
"completed": 401224161
},
"rollup_indexing": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"scroll": {
"threads": 5,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 283778
},
"search": {
"threads": 80,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 80,
"completed": 336975
},
"search_throttled": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snapshot": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"snccr": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"warmer": {
"threads": 5,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 5,
"completed": 123720972
},
"watcher": {
"threads": 0,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 0,
"completed": 0
},
"write": {
"threads": 20,
"queue": 0,
"active": 0,
"rejected": 0,
"largest": 20,
"completed": 401247127
}
},
"fs": {
"timestamp": 1625658351558,
"total": {
"total_in_bytes": 23623807156224,
"free_in_bytes": 23620919427072,
"available_in_bytes": 23380805271552
},
"data": [
{
"path": "/data00/dnode0/nodes/0",
"mount": "/data00 (/dev/sdb)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936956600320,
"available_in_bytes": 3896937574400
},
{
"path": "/data01/dnode0/nodes/0",
"mount": "/data01 (/dev/sdc)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936957739008,
"available_in_bytes": 3896938713088
},
{
"path": "/data02/dnode0/nodes/0",
"mount": "/data02 (/dev/sdd)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936706592768,
"available_in_bytes": 3896687566848
},
{
"path": "/data03/dnode0/nodes/0",
"mount": "/data03 (/dev/sde)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936877350912,
"available_in_bytes": 3896858324992
},
{
"path": "/data04/dnode0/nodes/0",
"mount": "/data04 (/dev/sdf)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936467005440,
"available_in_bytes": 3896447979520
},
{
"path": "/data05/dnode0/nodes/0",
"mount": "/data05 (/dev/sdg)",
"type": "ext4",
"total_in_bytes": 3937301192704,
"free_in_bytes": 3936954138624,
"available_in_bytes": 3896935112704
}
],
"io_stats": {
"devices": [
{
"device_name": "sdb",
"operations": 36266081,
"read_operations": 104017,
"write_operations": 36162064,
"read_kilobytes": 890376,
"write_kilobytes": 891363836
},
{
"device_name": "sdc",
"operations": 2036108914,
"read_operations": 40337626,
"write_operations": 1995771288,
"read_kilobytes": 622571048,
"write_kilobytes": 17856679036
},
{
"device_name": "sdd",
"operations": 36105318,
"read_operations": 68790,
"write_operations": 36036528,
"read_kilobytes": 722840,
"write_kilobytes": 805683512
},
{
"device_name": "sdg",
"operations": 30281741,
"read_operations": 95617,
"write_operations": 30186124,
"read_kilobytes": 807140,
"write_kilobytes": 750739504
},
{
"device_name": "sde",
"operations": 32830267,
"read_operations": 86226,
"write_operations": 32744041,
"read_kilobytes": 724844,
"write_kilobytes": 820100480
},
{
"device_name": "sdf",
"operations": 80159260,
"read_operations": 159637,
"write_operations": 79999623,
"read_kilobytes": 1212576,
"write_kilobytes": 1245826068
}
],
"total": {
"operations": 2251751581,
"read_operations": 40851913,
"write_operations": 2210899668,
"read_kilobytes": 626928824,
"write_kilobytes": 22370392436
}
}
},
"transport": {
"server_open": 126,
"rx_count": 1385372179,
"rx_size_in_bytes": 1083785580289,
"tx_count": 1638025356,
"tx_size_in_bytes": 2538203254681
},
"http": {
"current_open": 4,
"total_opened": 702692
},
"breakers": {
"request": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1,
"tripped": 0
},
"fielddata": {
"limit_size_in_bytes": 19971597926,
"limit_size": "18.5gb",
"estimated_size_in_bytes": 0,
"estimated_size": "0b",
"overhead": 1.03,
"tripped": 0
},
"in_flight_requests": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 757,
"estimated_size": "757b",
"overhead": 1,
"tripped": 0
},
"accounting": {
"limit_size_in_bytes": 33285996544,
"limit_size": "31gb",
"estimated_size_in_bytes": 4193343,
"estimated_size": "3.9mb",
"overhead": 1,
"tripped": 0
},
"parent": {
"limit_size_in_bytes": 23300197580,
"limit_size": "21.6gb",
"estimated_size_in_bytes": 4194100,
"estimated_size": "3.9mb",
"overhead": 1,
"tripped": 0
}
},
"script": {
"compilations": 1,
"cache_evictions": 0
},
"discovery": {
"cluster_state_queue": {
"total": 0,
"pending": 0,
"committed": 0
},
"published_cluster_states": {
"full_states": 1,
"incompatible_diffs": 0,
"compatible_diffs": 3473
}
},
"ingest": {
"total": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
},
"pipelines": {
"xpack_monitoring_2": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": [
{
"script": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"rename": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"set": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
},
{
"gsub": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0
}
}
]
},
"xpack_monitoring_6": {
"count": 0,
"time_in_millis": 0,
"current": 0,
"failed": 0,
"processors": []
}
}
},
"adaptive_selection": {
"OY-X1-1aRjm4m92KTY0EPg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 6410578969,
"avg_response_time_ns": 8599169143,
"rank": "8599.2"
},
"1RF501M0TY2dt1Ir4aTZtA": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 10787942065,
"avg_response_time_ns": 13417173793,
"rank": "13417.2"
},
"MKFeShkgQEClpoMYMJ9LoQ": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 3841891969,
"avg_response_time_ns": 2398008100,
"rank": "2398.0"
},
"ZYXwMegIRjK4Romw625Juw": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 103849,
"avg_response_time_ns": 1032595,
"rank": "1.0"
},
"swBMho--QCiUl2uaoYmtlg": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 400185,
"avg_response_time_ns": 2755711380,
"rank": "2755.7"
},
"PFDIsSN9R7aH1mDJ2f6ntQ": {
"outgoing_searches": 0,
"avg_queue_size": 0,
"avg_service_time_ns": 454081,
"avg_response_time_ns": 13951899480,
"rank": "13951.9"
}
}
}
}
}
统计分片信息
- 集群上所有分片情况
GET _cat/shards?v
结果示例
index shard prirep state docs store ip node
cm_test_20210412 1 p STARTED 0 283b 192.168.1.1 bdes01-platform-log-prd-dnode0
cm_test_20210412 1 r STARTED 0 283b 192.168.1.1 bdes02-platform-log-prd-dnode1
cm_test_20210412 0 p STARTED 0 283b 192.168.1.1 bdes01-platform-log-prd-dnode1
cm_test_20210412 0 r STARTED 0 283b 192.168.1.1 bdes03-platform-log-prd-dnode1
index_search_slowlog 1 r STARTED 395574907 201.1gb 192.168.1.1 bdes03-platform-log-prd-dnode0
index_search_slowlog 1 p STARTED 395574907 200.8gb 192.168.1.1 bdes02-platform-log-prd-dnode1
index_search_slowlog 0 p STARTED 395602882 206.1gb 192.168.1.1 bdes01-platform-log-prd-dnode1
index_search_slowlog 0 r STARTED 395602882 201.3gb 192.168.1.1 bdes03-platform-log-prd-dnode1
lwd_test_0225 1 r STARTED 0 283b 192.168.1.1 bdes03-platform-log-prd-dnode0
lwd_test_0225 1 p STARTED 0 283b 192.168.1.1 bdes01-platform-log-prd-dnode0
lwd_test_0225 0 r STARTED 0 283b 192.168.1.1 bdes02-platform-log-prd-dnode0
lwd_test_0225 0 p STARTED 0 283b 192.168.1.1 bdes03-platform-log-prd-dnode1
slowsearchlogindex 1 r STARTED 0 316b 192.168.1.1 bdes02-platform-log-prd-dnode0
slowsearchlogindex 1 p STARTED 0 316b 192.168.1.1 bdes01-platform-log-prd-dnode1
slowsearchlogindex 0 p STARTED 0 316b 192.168.1.1 bdes03-platform-log-prd-dnode0
slowsearchlogindex 0 r STARTED 0 316b 192.168.1.1 bdes03-platform-log-prd-dnode1
统计任务堆积情况
- 集群层面查询等待的任务
简单格式
GET /_cat/pending_tasks?v
结果示例
insertOrder timeInQueue priority source
1685 855ms HIGH update-mapping [foo][t]
1686 843ms HIGH update-mapping [foo][t]
1693 753ms HIGH refresh-mapping [foo][[t]]
1688 816ms HIGH update-mapping [foo][t]
1689 802ms HIGH update-mapping [foo][t]
1690 787ms HIGH update-mapping [foo][t]
1691 773ms HIGH update-mapping [foo][t]
json完整格式
GET /_cluster/pending_tasks
结果示例
{
"tasks": [
{
"insert_order": 101,
"priority": "URGENT",
"source": "create-index [foo_9], cause [api]",
"time_in_queue_millis": 86,
"time_in_queue": "86ms"
},
{
"insert_order": 46,
"priority": "HIGH",
"source": "shard-started ([foo_2][1], node[tMTocMvQQgGCkj7QDHl3OA], [P], s[INITIALIZING]), reason [after recovery from shard_store]",
"time_in_queue_millis": 842,
"time_in_queue": "842ms"
},
{
"insert_order": 45,
"priority": "HIGH",
"source": "shard-started ([foo_2][0], node[tMTocMvQQgGCkj7QDHl3OA], [P], s[INITIALIZING]), reason [after recovery from shard_store]",
"time_in_queue_millis": 858,
"time_in_queue": "858ms"
}
]
}
637
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
