OPA 防止服务重定向

服务重定向

1.创建一个prod namespace

kubectl create ns prod

2.创建应用
demo.yaml

##################################################################################################
# Details service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: details
  labels:
    app: details
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: details
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: details-v1
spec:
  replicas: 1
  template:
      
      
      
    metadata:
      labels:
        app: details
        version: v1
    spec:
      containers:
      - name: details
        image: openpolicyagent/demo-bookinfo-details-v1:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Ratings service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: ratings
  labels:
    app: ratings
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: ratings
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: ratings-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: ratings
        version: v1
    spec:
      containers:
      - name: ratings
        image: openpolicyagent/demo-bookinfo-ratings-v1:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Reviews service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: reviews
  labels:
    app: reviews
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: reviews
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: reviews-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: reviews
        version: v1
    spec:
      containers:
      - name: reviews
        image: openpolicyagent/demo-bookinfo-reviews-v2:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Productpage services
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: productpage
  labels:
    app: productpage
spec:
  type: NodePort
  ports:
  - port: 80
    targetPort: 9080
    name: http
    nodePort: 30899
  selector:
    app: productpage
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: productpage-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: productpage
        version: v1
    spec:
      containers:
      - name: productpage
        image: openpolicyagent/demo-bookinfo-productpage-v1:v1
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# OPA
##################################################################################################
kind: Service
apiVersion: v1
metadata:
  name: opa
  labels:
    app: opa
spec:
  selector:
    app: opa
  ports:
  - port: 8181
    targetPort: 8181
    name: http
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: opa
  name: opa
spec:
  replicas: 1
  selector:
    matchLabels:
      app: opa
  template:
    metadata:
      labels:
        app: opa
      name: opa
    spec:
      containers:
       - args:
         - run
         - --server
         - "/policies/inject.rego"
         image: openpolicyagent/opa:0.10.1
         imagePullPolicy: IfNotPresent
         name: opa
         ports:
         - containerPort: 8181
         volumeMounts:
            - readOnly: true
              mountPath: /policies
              name: inject-policy
      volumes:
        - name: inject-policy
          configMap:
            name: inject-policy
---
##################################################################################################
# OPA policies
##################################################################################################
apiVersion: v1
kind: ConfigMap
metadata:
  name: inject-policy
data:
  inject.rego: |
    package example

    default allow = true
---
##################################################################################################
# Ingress resource
##################################################################################################
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: productpage-ingress
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - http:
      paths:
      - path: /
        backend:
          serviceName: productpage
          servicePort: 80
---

kubectl apply -f demo.yaml -n prod

3.访问服务

kubectl get svc -n prod
NAME          TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)          AGE
...
productpage   NodePort   10.101.65.203    <none>        80:30889/TCP     50m

浏览器访问http://39.97.42.32:30889/bob
4.部署另一个服务
demo_alice.yaml:

##################################################################################################
# Details service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: details
  labels:
    app: details
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: details
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: details-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: details
        version: v1
    spec:
      containers:
      - name: details
        image: openpolicyagent/demo-bookinfo-details-v1:v2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Ratings service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: ratings
  labels:
    app: ratings
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: ratings
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: ratings-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: ratings
        version: v1
    spec:
      containers:
      - name: ratings
        image: openpolicyagent/demo-bookinfo-ratings-v1:v2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Reviews service
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: reviews
  labels:
    app: reviews
spec:
  ports:
  - port: 9080
    name: http
  selector:
    app: reviews
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: reviews-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: reviews
        version: v1
    spec:
      containers:
      - name: reviews
        image: openpolicyagent/demo-bookinfo-reviews-v2:v2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# Productpage services
##################################################################################################
apiVersion: v1
kind: Service
metadata:
  name: productpage
  labels:
    app: productpage
spec:
  ports:
  - port: 80
    targetPort: 9080
    name: http
  selector:
    app: productpage
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: productpage-v1
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: productpage
        version: v1
    spec:
      containers:
      - name: productpage
        image: openpolicyagent/demo-bookinfo-productpage-v1:v2
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 9080
---
##################################################################################################
# OPA
##################################################################################################
kind: Service
apiVersion: v1
metadata:
  name: opa
  labels:
    app: opa
spec:
  selector:
    app: opa
  ports:
  - port: 8181
    targetPort: 8181
    name: http
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: opa
  name: opa
spec:
  replicas: 1
  selector:
    matchLabels:
      app: opa
  template:
    metadata:
      labels:
        app: opa
      name: opa
    spec:
      containers:
       - args:
         - run
         - --server
         - "/policies/inject.rego"
         image: openpolicyagent/opa:0.10.1
         imagePullPolicy: IfNotPresent
         name: opa
         ports:
         - containerPort: 8181
         volumeMounts:
            - readOnly: true
              mountPath: /policies
              name: inject-policy
      volumes:
        - name: inject-policy
          configMap:
            name: inject-policy
---
##################################################################################################
# OPA policies
##################################################################################################
apiVersion: v1
kind: ConfigMap
metadata:
  name: inject-policy
data:
  inject.rego: |
    package example

    default allow = true
---
##################################################################################################
# Ingress resource
##################################################################################################
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: productpage-ingress
  annotations:
    kubernetes.io/ingress.class: "nginx"
spec:
  rules:
  - http:
      paths:
      - path: /bob
        backend:
          serviceName: productpage
          servicePort: 80
---

kubectl create ns dev
kubectl apply -f demo_alice.yaml -n dev

此时刷新浏览器会发现，之前的界面已有所变化。
说明：demo应用实际上是在“prod“ namespace 为主机 * 创建了路径为 “/” 的URL，而demo_alice 则是在”dev“ namespace 为主机 * 创建了路径为”/bob“ 的URL。那么demo_alice将服务由路径 “/” 重定向到 ”/bob“ ，并由该应用处理流量。
5.OPA策略防御
使用OPA，您可以实施准入控制规则，在创建，更新和删除操作期间验证Kubernetes资源，并实时实施策略，而无需重新编译或重新配置Kubernetes API服务器。
将OPA 部署为Admission Controller
opa-admission-controller.yaml:

## Grant OPA/kube-mgmt read-only access to resources. This lets kube-mgmt
# replicate resources into OPA so they can be used in policies.
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: opa-viewer
roleRef:
  kind: ClusterRole
  name: view
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: Group
  name: system:serviceaccounts:opa
  apiGroup: rbac.authorization.k8s.io
---
# Define role for OPA/kube-mgmt to update configmaps with policy status.
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: opa
  name: configmap-modifier
rules:
- apiGroups: [""]
  resources: ["configmaps"]
  verbs: ["update", "patch"]
---
# Grant OPA/kube-mgmt role defined above.
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: opa
  name: opa-configmap-modifier
roleRef:
  kind: Role
  name: configmap-modifier
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: Group
  name: system:serviceaccounts:opa
  apiGroup: rbac.authorization.k8s.io
---
kind: Service
apiVersion: v1
metadata:
  name: opa
  namespace: opa
spec:
  selector:
    app: opa
  ports:
  - name: https
    protocol: TCP
    port: 443
    targetPort: 443
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  labels:
    app: opa
  namespace: opa
  name: opa
spec:
  replicas: 1
  selector:
    matchLabels:
      app: opa
  template:
    metadata:
      labels:
        app: opa
      name: opa
    spec:
      containers:
        # WARNING: OPA is NOT running with an authorization policy configured. This
        # means that clients can read and write policies in OPA. If you are
        # deploying OPA in an insecure environment, be sure to configure
        # authentication and authorization on the daemon. See the Security page for
        # details: https://www.openpolicyagent.org/docs/security.html.
        - name: opa
          image: openpolicyagent/opa:0.12.0
          args:
            - "run"
            - "--server"
            - "--tls-cert-file=/certs/tls.crt"
            - "--tls-private-key-file=/certs/tls.key"
            - "--addr=0.0.0.0:443"
            - "--addr=http://127.0.0.1:8181"
          volumeMounts:
            - readOnly: true
              mountPath: /certs
              name: opa-server
        - name: kube-mgmt
          image: openpolicyagent/kube-mgmt:0.8
          args:
            - "--replicate-cluster=v1/namespaces"
            - "--replicate=extensions/v1beta1/ingresses"
      volumes:
        - name: opa-server
          secret:
            secretName: opa-server
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: opa-default-system-main
  namespace: opa
data:
  main: |
    package system

    import data.kubernetes.admission

    main = {
      "apiVersion": "admission.k8s.io/v1beta1",
      "kind": "AdmissionReview",
      "response": response,
    }

    default response = {"allowed": true}

    response = {
        "allowed": false,
        "status": {
            "reason": reason,
        },
    } {
        reason = concat(", ", admission.deny)
        reason != ""
    }

生成TLS认证证书
opa-ac.sh:

#!/bin/bash

# create opa namespace
kubectl create ns opa

# create tls secret
rm -rf ./secret
mkdir ./secret
cd ./secret

openssl genrsa -out ca.key 2048
openssl req -x509 -new -nodes -key ca.key -days 100000 -out ca.crt -subj "/CN=admission_ca"
cat >server.conf <<EOF
[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name
[req_distinguished_name]
[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = clientAuth, serverAuth
EOF
openssl genrsa -out server.key 2048
openssl req -new -key server.key -out server.csr -subj "/CN=opa.opa.svc" -config server.conf
openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out server.crt -days 100000 -extensions v3_req -extfile server.conf
cd ..

# create k8s opa secret
kubectl -n opa create secret tls opa-server --cert=./secret/server.crt --key=./secret/server.key

# deploy kubernetes-policy-controller
kubectl apply -n opa -f opa-admission-controller.yaml

# deploy webhooks
cat > ./secret/webhook-configuration.yaml <<EOF
kind: ValidatingWebhookConfiguration
apiVersion: admissionregistration.k8s.io/v1beta1
metadata:
  name: opa-validating-webhook
webhooks:
  - name: validating-webhook.openpolicyagent.org
    rules:
      - operations: ["CREATE", "UPDATE"]
        apiGroups: ["*"]
        apiVersions: ["*"]
        resources: ["*"]
    clientConfig:
      caBundle: $(cat ./secret/ca.crt | base64 | tr -d '\n')
      service:
        namespace: opa
        name: opa
EOF

kubectl -n opa apply -f ./secret/webhook-configuration.yaml

执行opa-ac.sh。
编辑ingress-conflicts.ego

package kubernetes.admission

import data.kubernetes.ingresses

# Policy 1: Ingress hostnames must be unique across Namespaces.
deny[msg] {
    input.request.kind.kind = "Ingress"                # Resource kind
    input.request.operation = "CREATE"                 # Resource Operation
    host = input.request.object.spec.rules[_].host     # Host making the request
    ingress = ingresses[other_ns][other_ingress]       # Iterate over ingresses
    other_ns != input.request.namespace
    ingress.spec.rules[_].host == host                 # Check if same host in the ingress rule
    msg := sprintf("invalid ingress host %q (conflicts with %v/%v)", [host, other_ns, other_ingress])
}

# Policy 2: "host" field not present in the Ingress rule.This means the Ingress rule applies for all inbound traffic.
# So the existence of an Ingress rule in any other namspace would result in a conflict.
deny[msg] {
    input.request.kind.kind = "Ingress"
    input.request.operation = "CREATE"
    x := input.request.object.spec.rules[_]
    not x.host
    ingress = ingresses[other_ns][other_ingress]
    other_ns != input.request.namespace
    count(ingress.spec.rules) > 0
    msg := sprintf("invalid ingress host (conflicts with %v/%v)", [other_ns, other_ingress])
}

该策略将拒绝同一host中不同namespace的Ingress流量。
6.使策略生效

kubectl create configmap -n opa ingress-conflicts --from-file=ingress-conflicts.rego

查看是否生效

kubectl get configmap -n opa ingress-conflicts -ojsonpath='{.metadata.annotations}'
map[openpolicyagent.org/policy-status:{"status":"ok"}]

7.重新部署demo_alice 应用

kubectl delete -f demo_alice.yaml -n dev

kubectl apply -f demo_alice.yaml -n dev
"validating-webhook.openpolicyagent.org" denied the request: invalid ingress host (conflicts with prod/productpage-ingress)

那么这次该应用就不能在同一个cluster中另一个namespace创建出ingress，无法在重定向服务了。