could not acquire a semaphore for execution

环境：
spring-boot-starter-* 1.5.2.RELEASE
spring-cloud-starter-eureka-* 1.2.6.RELEASE.jar
spring-cloud-starter-hystrix-* 1.2.6.RELEASE
spring-cloud-starter-feign 1.2.6.RELEASE
spring-cloud-starter-ribbon 1.2.6.RELEASE
spring-cloud-starter-zuul 1.2.6.RELEASE

访问链路: nginx->zuul网关->微服务A

现象：jmeter 600个线程压测请求爆红，错误率45%以上，查看对应请求的zuul网关，部分日志如下：

Caused by: java.lang.RuntimeException: could not acquire a semaphore for execution
        at com.netflix.hystrix.AbstractCommand.handleSemaphoreRejectionViaFallback(AbstractCommand.java:949) ~[hystrix-core-1.5.6.
        at com.netflix.hystrix.AbstractCommand.applyHystrixSemantics(AbstractCommand.java:540) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand.access$100(AbstractCommand.java:59) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand$4.call(AbstractCommand.java:405) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand$4.call(AbstractCommand.java:402) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:46) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:35) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeMap.call(OnSubscribeMap.java:33) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:41) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:41) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach.call(OnSubscribeDoOnEach.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.unsafeSubscribe(Observable.java:10211) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:51) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDefer.call(OnSubscribeDefer.java:35) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:48) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeLift.call(OnSubscribeLift.java:30) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.subscribe(Observable.java:10307) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.Observable.subscribe(Observable.java:10274) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.BlockingOperatorToFuture.toFuture(BlockingOperatorToFuture.java:51) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.observables.BlockingObservable.toFuture(BlockingObservable.java:412) ~[rxjava-1.1.10.jar!/:1.1.10]
        at com.netflix.hystrix.HystrixCommand.queue(HystrixCommand.java:377) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.HystrixCommand.execute(HystrixCommand.java:343) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.forward(RibbonRoutingFilter.java:139) ~[spring
        ... 67 more
2020-09-01 15:54:07,361:WARN org.springframework.cloud.netflix.zuul.filters.post.SendErrorFilter http-nio-6100-exec-70 (SendErrorF
com.netflix.zuul.exception.ZuulException: Forwarding error
        at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.handleException(RibbonRoutingFilter.java:170)
        at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.forward(RibbonRoutingFilter.java:145) ~[spring
        at org.springframework.cloud.netflix.zuul.filters.route.RibbonRoutingFilter.run(RibbonRoutingFilter.java:88) ~[spring-clou
        at com.netflix.zuul.ZuulFilter.runFilter(ZuulFilter.java:112) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.FilterProcessor.processZuulFilter(FilterProcessor.java:193) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.FilterProcessor.runFilters(FilterProcessor.java:157) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.FilterProcessor.route(FilterProcessor.java:118) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.ZuulRunner.route(ZuulRunner.java:96) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.http.ZuulServlet.route(ZuulServlet.java:116) [zuul-core-1.3.0.jar!/:1.3.0]
        at com.netflix.zuul.http.ZuulServlet.service(ZuulServlet.java:81) [zuul-core-1.3.0.jar!/:1.3.0]
        at org.springframework.web.servlet.mvc.ServletWrappingController.handleRequestInternal(ServletWrappingController.java:157)
        at org.springframework.cloud.netflix.zuul.web.ZuulController.handleRequest(ZuulController.java:44) [spring-cloud-netflix-c
        at org.springframework.web.servlet.mvc.SimpleControllerHandlerAdapter.handle(SimpleControllerHandlerAdapter.java:50) [spri
        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:963) [spring-webmvc-4.3.7.RELEASE.j
        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:897) [spring-webmvc-4.3.7.RELEASE.ja
        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:970) [spring-webmvc-4.3.7.RELEASE
        at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:861) [spring-webmvc-4.3.7.RELEASE.jar!/:4.
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:622) [tomcat-embed-core-8.5.11.jar!/:8.5.11]
        at org.springframework.web.servlet.FrameworkServlet.service(FrameworkServlet.java:846) [spring-webmvc-4.3.7.RELEASE.jar!/:
        at javax.servlet.http.HttpServlet.service(HttpServlet.java:729) [tomcat-embed-core-8.5.11.jar!/:8.5.11]

查看hystrix配置：

hystrix:
  command:
    default:
      execution:
        isolation:
          thread:
            timeoutInMilliseconds: 10000

从hystrix-core包查看
com.netflix.hystrix.config.HystrixThreadPoolConfiguration
com.netflix.hystrix.HystrixThreadPoolProperties
可以知道
coreSize、maxQueueSize、queueSizeRejectionThreshold
的配置方式

hystrix.threadpool.default.coreSize=100	   // 线程池大小，默认10
hystrix.threadpool.default.maxQueueSize=2000   // 请求队列大小，默认-1 ，表示不开启队列（该数值无法动态更改，因此使用'queueSizeRejectionThreshold'人为地限制和拒绝）
hystrix.threadpool.default.queueSizeRejectionThreshold=2000		// 队列中，默认5，

关键代码如下

public abstract class HystrixThreadPoolProperties {
    /* defaults */
    private Integer default_coreSize = 10; // size of thread pool
     ...
    private Integer default_maxQueueSize = -1; // size of queue (this can't be dynamically changed so we use 'queueSizeRejectionThreshold' to artificially limit and reject)
                                               // -1 turns if off and makes us use SynchronousQueue
    private Integer default_queueSizeRejectionThreshold = 5; // number of items in queue 
     ...

protected HystrixThreadPoolProperties(HystrixThreadPoolKey key, Setter builder, String propertyPrefix) {
        this.corePoolSize = getProperty(propertyPrefix, key, "coreSize", builder.getCoreSize(), default_coreSize);
        ...
        this.maxQueueSize = getProperty(propertyPrefix, key, "maxQueueSize", builder.getMaxQueueSize(), default_maxQueueSize);
        this.queueSizeRejectionThreshold = getProperty(propertyPrefix, key, "queueSizeRejectionThreshold", builder.getQueueSizeRejectionThreshold(), default_queueSizeRejectionThreshold);
        ...
    }

    private static HystrixProperty<Integer> getProperty(String propertyPrefix, HystrixThreadPoolKey key, String instanceProperty, Integer builderOverrideValue, Integer defaultValue) {
        return forInteger()
                .add(propertyPrefix + ".threadpool." + key.name() + "." + instanceProperty, builderOverrideValue)
                .add(propertyPrefix + ".threadpool.default." + instanceProperty, defaultValue)
                .build();
    }
}
...

一顿操作猛如虎，然而令人郁闷的是单独配置hystrix然并卵，最终通过zuul.semaphore.max-semaphores配置信号量，压测2000线程，请求错误率0.2%（微服务A访问超时）

hystrix:
  command:
    default:
      execution:
        isolation:
          thread:
            timeoutInMilliseconds: 10000
  threadpool:
    default:
      coreSize: 100
      maxQueueSize: 2000
      queueSizeRejectionThreshold: 2000

zuul:
  semaphore:
    max-semaphores: 5000

如果出现以下报错，则需要注意ribbon的超时配置

Caused by: java.util.concurrent.TimeoutException
        at com.netflix.hystrix.AbstractCommand.handleTimeoutViaFallback(AbstractCommand.java:980) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand.access$500(AbstractCommand.java:59) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand$12.call(AbstractCommand.java:595) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand$12.call(AbstractCommand.java:587) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at rx.internal.operators.OperatorOnErrorResumeNextViaFunction$4.onError(OperatorOnErrorResumeNextViaFunction.java:140) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach$DoOnEachSubscriber.onError(OnSubscribeDoOnEach.java:87) ~[rxjava-1.1.10.jar!/:1.1.10]
        at rx.internal.operators.OnSubscribeDoOnEach$DoOnEachSubscriber.onError(OnSubscribeDoOnEach.java:87) ~[rxjava-1.1.10.jar!/:1.1.10]
        at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$1.run(AbstractCommand.java:1121) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:41) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable$1.call(HystrixContextRunnable.java:37) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.strategy.concurrency.HystrixContextRunnable.run(HystrixContextRunnable.java:57) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.AbstractCommand$HystrixObservableTimeoutOperator$2.tick(AbstractCommand.java:1138) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at com.netflix.hystrix.util.HystrixTimer$1.run(HystrixTimer.java:99) ~[hystrix-core-1.5.6.jar!/:1.5.6]
        at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) ~[?:1.8.0_241]
        at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) ~[?:1.8.0_241]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180) ~[?:1.8.0_241]
        at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) ~[?:1.8.0_241]
        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) ~[?:1.8.0_241]
        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) ~[?:1.8.0_241]

原来

ribbon:
  ConnectTimeout: 3000
  ReadTimeout: 60000

改为

ribbon:
  ConnectTimeout: 10000
  ReadTimeout: 60000
  MaxConnectionsPerHost: 3000
  MaxTotalConnections: 3000

参考
https://www.cnblogs.com/java-spring/p/13042335.html
https://www.cnblogs.com/killerqi/p/10906392.html
https://blog.csdn.net/weixin_41231928/article/details/105238389
https://blog.csdn.net/wang20y8/article/details/102523265