es报错no_shard_available_action_exception

一、问题描述
机房迁移之后,发现ES连接报错,查询某个索引数据

GET  http://IP:PORT/XXX_XXX_XXX/_search

报错

{
    "error": {
        "root_cause": [
            {
                "type": "no_shard_available_action_exception",
                "reason": null,
                "index_uuid": "xxxxx-xxxx-ddddd",
                "shard": "0",
                "index": "XXX_XXX_XXX"
            }
        ],
        "type": "search_phase_execution_exception",
        "reason": "all shards failed",
        "phase": "query",
        "grouped": true,
        "failed_shards": [
            {
                "shard": 0,
                "index": "XXX_XXX_XXX",
                "node": null,
                "reason": {
                    "type": "no_shard_available_action_exception",
                    "reason": null,
                    "index_uuid": "xxxxx-xxxx-ddddd",
                    "shard": "0",
                    "index": "XXX_XXX_XXX"
                }
            }
        ]
    },
    "status": 503
}

ES日志

{"type": "server", "timestamp": "2024-07-05T09:46:33,177Z", "level": "WARN", "component": "r.suppressed", "cluster.name": "docker-cluster", "node.name": "esXXXX-XXXXXXXXXXXXXX-XXXX", "message": "path: /XXX_XXX_XXX/_search, params: {index=XXX_XXX_XXX}", "cluster.uuid": "_VxVNRD-QB-DDDDDDDDDDD", "node.id": "KdIfLm7ERZGankl-XSWWWWWWWWWWW" , 
"stacktrace": ["org.elasticsearch.action.search.SearchPhaseExecutionException: all shards failed",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseFailure(AbstractSearchAsyncAction.java:663) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executeNextPhase(AbstractSearchAsyncAction.java:384) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onPhaseDone(AbstractSearchAsyncAction.java:696) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:467) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.performPhaseOnShard(AbstractSearchAsyncAction.java:292) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.run(AbstractSearchAsyncAction.java:226) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.executePhase(AbstractSearchAsyncAction.java:424) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.start(AbstractSearchAsyncAction.java:184) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.executeSearch(TransportSearchAction.java:668) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.executeLocalSearch(TransportSearchAction.java:489) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.lambda$executeRequest$3(TransportSearchAction.java:283) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:117) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:103) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.index.query.Rewriteable.rewriteAndFetch(Rewriteable.java:76) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.executeRequest(TransportSearchAction.java:324) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:212) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.TransportSearchAction.doExecute(TransportSearchAction.java:91) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:173) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.support.ActionFilter$Simple.apply(ActionFilter.java:42) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:171) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:149) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:77) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:86) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.action.RestCancellableNodeClient.doExecute(RestCancellableNodeClient.java:80) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:402) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.action.search.RestSearchAction.lambda$prepareRequest$2(RestSearchAction.java:118) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.BaseRestHandler.handleRequest(BaseRestHandler.java:106) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:247) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.RestController.tryAllHandlers(RestController.java:329) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.rest.RestController.dispatchRequest(RestController.java:180) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.http.AbstractHttpServerTransport.dispatchRequest(AbstractHttpServerTransport.java:325) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.http.AbstractHttpServerTransport.handleIncomingRequest(AbstractHttpServerTransport.java:390) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.http.AbstractHttpServerTransport.incomingRequest(AbstractHttpServerTransport.java:307) [elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:31) [transport-netty4-client-7.12.1.jar:7.12.1]",
"at org.elasticsearch.http.netty4.Netty4HttpRequestHandler.channelRead0(Netty4HttpRequestHandler.java:17) [transport-netty4-client-7.12.1.jar:7.12.1]",
"at io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at org.elasticsearch.http.netty4.Netty4HttpPipeliningHandler.channelRead(Netty4HttpPipeliningHandler.java:47) [transport-netty4-client-7.12.1.jar:7.12.1]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageCodec.channelRead(MessageToMessageCodec.java:111) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:324) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:296) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:286) [netty-handler-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.handler.codec.MessageToMessageDecoder.channelRead(MessageToMessageDecoder.java:103) [netty-codec-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:357) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:379) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:365) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) [netty-transport-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [netty-common-4.1.49.Final.jar:4.1.49.Final]",
"at java.lang.Thread.run(Thread.java:831) [?:?]",
"Caused by: org.elasticsearch.action.NoShardAvailableActionException",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:500) ~[elasticsearch-7.12.1.jar:7.12.1]",
"at org.elasticsearch.action.search.AbstractSearchAsyncAction.onShardFailure(AbstractSearchAsyncAction.java:449) [elasticsearch-7.12.1.jar:7.12.1]",
"... 81 more"] }

二、问题排查
第一步:查看分片状态

GET http://IPPORT/_cat/shards?pretty

返回值

wcm                   0 p STARTED     0   208b xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
test                  0 p STARTED    21 37.3kb xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
XXX_XXX_XXX           0 p UNASSIGNED                          
cluster               0 p STARTED     2  5.7kb xxx.xxx.xxx.xxx esuuu-iiiiiiiiiiiiiii-hwwwwww
cluster               0 r UNASSIGNED         
  

XXX_XXX_XXX分片状态是UNASSIGNED,不正常

第二步:查询分片状态unassigned的原因

http://IPPORT/_cluster/allocation/explain?pretty  GET

结果

{
    "index": "XXX_XXX_XXX",
    "shard": 0,
    "primary": true,
    "current_state": "unassigned",
    "unassigned_info": {
        "reason": "CLUSTER_RECOVERED",
        "at": "2024-07-05T09:39:01.951Z",
        "last_allocation_status": "no_valid_shard_copy"
    },
    "can_allocate": "no_valid_shard_copy",
    "allocate_explanation": "cannot allocate because all found copies of the shard are either stale or corrupt",
    "node_allocation_decisions": [
        {
            "node_id": "KdIfLm7ERZGankl-7b_jtw",
            "node_name": "esuuu-iiiiiiiiiiiiiii-hwwwwww",
            "transport_address": "xxx.xxx.xxx.xxx:9300",
            "node_attributes": {
                "ml.machine_memory": "2147483648",
                "xpack.installed": "true",
                "transform.node": "true",
                "ml.max_open_jobs": "20",
                "ml.max_jvm_size": "1073741824"
            },
            "node_decision": "no",
            "store": {
                "in_sync": true,
                "allocation_id": "ahagagagagagaga",
                "store_exception": {
                    "type": "corrupt_index_exception",
                    "reason": "failed engine (reason: [search execution corruption failure]) (resource=preexisting_corruption)",
                    "caused_by": {
                        "type": "i_o_exception",
                        "reason": "failed engine (reason: [search execution corruption failure])",
                        "caused_by": {
                            "type": "corrupt_index_exception",
                            "reason": "Corrupted: docID=147, docBase=15, chunkDocs=11, numDocs=556 (resource=NIOFSIndexInput(path=\"/usr/share/elasticsearch/data/nodes/0/indices/xhxhxhxxhxhxhxxhxhxx-pMxOzkdhdw/0/index/_b.fdt\"))"
                        }
                    }
                }
            }
        }
    ]
} 

三、原因分析
节点长时间掉线后重新加入集群,引入了脏数据。

四、解决办法
我尝试了重新分配分片、将一个空分片分配为主分片、重启ES等方法,结果是没有效果,且返回值显示找不到索引对应的数据。可能数据已经丢失。单独查询索引XXX_XXX_XXX,发现索引还在。
最终,我删除了索引,重新同步了一份数据到ES中。

五、参考
https://www.cnblogs.com/qcloud1001/p/13755469.html
https://blog.csdn.net/vinegar93/article/details/135617013

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值