k8s之istio实现服务请求熔断(1)

测试yaml

# 模拟后端应用
[root@k8s-master-1 example-v1]# cat deployment.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: circuit-break
---
apiVersion: v1
kind: Service
metadata:
  name: busybox-httpd
  namespace: circuit-break
spec:
  ports:
  - name: http
    port: 80
    targetPort: 80
  selector:
    app: httpd
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: busybox-httpd
  namespace: circuit-break
spec:
  replicas: 1
  selector:
    matchLabels:
      app: httpd
  template:
    metadata:
      labels:
        app: httpd
    spec:
      containers:
      - name: busybox-httpd
        image: busybox:1.28
        imagePullPolicy: IfNotPresent
        ports: 
        - containerPort: 80
        command: ["/bin/sh","-c","echo 'this is busybox-httpd' > /var/www/index.html;httpd -f -h /var/www"]

# 创建熔断器
[root@k8s-master-1 example-v1]# cat circuit.yaml 
apiVersion: networking.istio.io/v1beta1
kind: DestinationRule
metadata:
  name: busybox-httpd
  namespace: circuit-break
spec:
  host: busybox-httpd.circuit-break.svc.cluster.local
  trafficPolicy:
    connectionPool: # 连接池(TCP|HTTP)配置,例如:连接数 并发请求等
      http:
        http1MaxPendingRequests: 1  # 连接到目标主机的最大挂起请求数,也就是待处理请求数,这里的目标指的是virtualservice路由规则中配置的 destination 。
        maxRequestsPerConnection: 1 # 连接池中每个连接最多处理1个请求后就关闭，并根据需要重新创建连接池中的连接
      tcp:
        maxConnections: 1           # TCP连接池中的最大连接请求数,当超过这个值会返回503代码,如两个请求过来,就会有一个请求返回503
    outlierDetection:               # 异常检测配置，传统意义上的熔断配置，即对规定时间内服务错误数的监测
      baseEjectionTime: 3m          # 基本驱逐时间3分钟，实际驱逐时间为baseEjectionTime* 驱逐次数
      consecutive5xxErrors: 1       # 连续错误数1 ，即连续返回502 504状态码的Http请求错误数
      interval: 1s                  # 错误异常的扫描间隔1s,即在interval 1s内连续发生consecutiveGatewayErrors 1错误则触发服务熔断
      maxEjectionPercent: 100       # 最大驱逐百分比 100%

# 创建模拟请求客户端
[root@k8s-master-1 example-v1]# cat fortio-deploy.yaml 
apiVersion: v1
kind: Service
metadata:
  name: fortio
  namespace: circuit-break
  labels:
    app: fortio
    service: fortio
spec:
  ports:
  - port: 8080
    name: http
  selector:
    app: fortio
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: fortio-deploy
  namespace: circuit-break
spec:
  replicas: 1
  selector:
    matchLabels:
      app: fortio
  template:
    metadata:
      annotations:
        # This annotation causes Envoy to serve cluster.outbound statistics via 15000/stats
        # in addition to the stats normally served by Istio. The Circuit Breaking example task
        # gives an example of inspecting Envoy stats via proxy config.
        proxy.istio.io/config: |-
          proxyStatsMatcher:
            inclusionPrefixes:
            - "cluster.outbound"
            - "cluster_manager"
            - "listener_manager"
            - "server"
            - "cluster.xds-grpc"
      labels:
        app: fortio
    spec:
      containers:
      - name: fortio
        image: fortio/fortio:latest_release
        imagePullPolicy: Always
        ports:
        - containerPort: 8080
          name: http-fortio
        - containerPort: 8079
          name: grpc-ping

部署

[root@k8s-master-1 example-v1]# istioctl kube-inject -f deployment.yaml | kubectl apply -f -
namespace/circuit-break created
service/busybox-httpd created
deployment.apps/busybox-httpd created

[root@k8s-master-1 example-v1]# kubectl apply -f circuit.yaml 
destinationrule.networking.istio.io/busybox-httpd created

[root@k8s-master-1 example-v1]# istioctl kube-inject -f fortio-deploy.yaml | kubectl apply -f -
service/fortio configured
deployment.apps/fortio-deploy configured

模拟请求

在 DestinationRule 配置中，您定义了 maxConnections: 1 和 http1MaxPendingRequests: 1。这些规则意味着，如果并发的连接和请求数超过一个，在 istio-proxy 进行进一步的请求和连接时，后续请求或连接将被阻止

# 发送并发数为 2 的连接（-c 2），请求 20 次（-n 20）,可见有部分连接被拒绝了
[root@k8s-master-1 example-v1]# kubectl exec $(kubectl get pods -l app=fortio -n circuit-break -o 'jsonpath={.items[0].metadata.name}') -n circuit-break -c fortio -- /usr/bin/fortio load -c 2 -qps 0 -n 20 -loglevel Warning  http://busybox-httpd.circuit-break.svc.cluster.local
............................................................
Sockets used: 9 (for perfect keepalive, would be 2)
Jitter: false
Code 200 : 12 (60.0 %)
Code 503 : 8 (40.0 %)
Response Header Sizes : count 20 avg 130.8 +/- 106.8 min 0 max 218 sum 2616
Response Body/Total Sizes : count 20 avg 240.4 +/- 0.4899 min 240 max 241 sum 4808
All done 20 calls (plus 0 warmup) 4.064 ms avg, 442.0 qps

# 查看熔断请求数,可见有47个请求已被熔断
[root@k8s-master-1 example-v1]# kubectl exec $(kubectl get pods -l app=fortio -n circuit-break -o 'jsonpath={.items[0].metadata.name}') -n circuit-break -c istio-proxy -- pilot-agent request GET stats | grep busybox-httpd.circuit-break.svc.cluster.local | grep pending
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.circuit_breakers.default.remaining_pending: 1
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.circuit_breakers.default.rq_pending_open: 0
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.circuit_breakers.high.rq_pending_open: 0
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.upstream_rq_pending_active: 0
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.upstream_rq_pending_failure_eject: 0
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.upstream_rq_pending_overflow: 47
cluster.outbound|80||busybox-httpd.circuit-break.svc.cluster.local.upstream_rq_pending_total: 57