第10章 K8s进阶篇-高级调度计划任务，污点和容忍和Affinity

dluhehe

已于 2023-03-03 15:17:09 修改

阅读量782

点赞数 1

分类专栏： K8s入门教程文章标签： kubernetes 容器云原生

于 2022-12-10 00:11:58 首次发布

本文链接：https://blog.csdn.net/dluhehe/article/details/128259987

版权

K8s入门教程专栏收录该内容

9 篇文章 3 订阅

订阅专栏

10.1 什么是Job？

job常用作初始化数据和基本的创建操作。

job创建成功后不会立即执行容器命令，只有suspend=true，才会执行。

10.2 Job使用入门

[root@k8s-master01 10st]# cat job.yaml 
apiVersion: batch/v1
kind: Job
metadata:
  labels:
    job-name: echo
  name: echo
  namespace: default
spec:
#suspend: true # 1.21+
#  ttlSecondsAfterFinished: 100
  backoffLimit: 4
  completions: 1
  parallelism: 1
  template:
    spec:
      containers:
      - command:
        - echo
        - Hello,Job
        image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
        name: echo
        resources: {}
      restartPolicy: Never
[root@k8s-master01 10st]#

kubectl create  -f job.yaml

job.yaml文件不可以修改后replace/apply,只能kubectl delete -f job,yaml删除后,修改后重新create。job一般是配合Helm进行部署。

[root@k8s-master01 10st]# cat job.yaml 
apiVersion: batch/v1
kind: Job
metadata:
  labels:
    job-name: echo
  name: echo
  namespace: default
spec:
#suspend: true # 1.21+
#  ttlSecondsAfterFinished: 100
  backoffLimit: 4
  completions: 5
  parallelism: 3
  template:
    spec:
      containers:
      - command:
        - echo
        - Hello,Job
        image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
        name: echo
        resources: {}
      restartPolicy: Never
[root@k8s-master01 10st]#

10.3 更强大的计划任务CronJob

10.4 CronJob使用

[root@k8s-master01 10st]# cat cronjob.yaml 
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  labels:
    run: hello
  name: hello
  namespace: default
spec:
  concurrencyPolicy: Allow
  failedJobsHistoryLimit: 1
  jobTemplate:
    metadata:
    spec:
      template:
        metadata:
          labels:
            run: hello
        spec:
          containers:
          - args:
            - /bin/sh
            - -c
            - date;echo Hello from the Kubernetes cluster
            image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
            imagePullPolicy: Always
            name: hello
            resources: {}
          restartPolicy: OnFailure
          securityContext: {}
  schedule: '*/1 * * * *'
  successfulJobsHistoryLimit: 3
  suspend: false
您在 /var/spool/mail/root 中有新邮件
[root@k8s-master01 10st]#

kubectl create -f cronjob.yaml

本案例中cronjob创建成功后会创建一个job,job会创建一个pod执行我们指令。

[root@k8s-master01 10st]# cat cronjob.yaml 
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  labels:
    run: hello
  name: hello
  namespace: default
spec:
  concurrencyPolicy: Allow
  failedJobsHistoryLimit: 1
  jobTemplate:
    metadata:
    spec:
      template:
        metadata:
          labels:
            run: hello
        spec:
          containers:
          - args:
            - /bin/sh
            - -c
            - date;echo Hello from the Kubernetes cluster
            image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
            imagePullPolicy: Always
            name: hello
            resources: {}
          restartPolicy: OnFailure
          securityContext: {}
  schedule: '*/1 * * * *'
  successfulJobsHistoryLimit: 3
  suspend: true
[root@k8s-master01 10st]#

10.5 InitContainer用途

不要以root身份运行业务容器，可以运行Init容器；

10.6 初始化容器和普通容器、PostStart

当pod里有多个Init容器时，只有第一个Init容器运行完成（以成功状态退出）后才会运行第二个，如果第一个Init运行失败，pod会一直运行第一个Init容器。比如pod里有4个Init容器（Init1-Init4），只有4个都执行成功，pod才会执行主程序的容器。

10.7 初始化容器配置解析

10.8 初始化容器使用示例

[root@k8s-master01 10st]# cat init.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: test-init
  name: test-init
  namespace: kube-public
spec:
  replicas: 1
  selector:
    matchLabels:
      app: test-init
  template:
    metadata:
      labels:
        app: test-init
    spec:
      volumes:
      - name: data
        emptyDir: {}
      initContainers:
      - command:
        - sh
        - -c
        - touch /mnt/test-init.txt
        image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
        name: init-touch
        volumeMounts:
        - name: data
          mountPath: /mnt
      containers:
      - image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
       # command: ["/bin/bash", "-ce", "tail -f /dev/null"]
       #加上如下这句，不然容器启动后因为没有常驻进程导致退出，状态:CrashLoopBackOff
        command: ["/bin/sh","-ce","sleep 3600"]
        name: test-init
        volumeMounts:
        - name: data
          mountPath: /mnt
[root@k8s-master01 10st]#

kubectl  create  -f init.yaml


#通过deploy扩容，观察先启动Init容器，后启动主进程
kubectl scale deploy test-init  --replicas=6   -n kube-public

测试多个Init容器

[root@k8s-master01 10st]# cat init.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: test-init
  name: test-init
  namespace: kube-public
spec:
  replicas: 3
  selector:
    matchLabels:
      app: test-init
  template:
    metadata:
      labels:
        app: test-init
    spec:
      volumes:
      - name: data
        emptyDir: {}
      initContainers:
      - command:
        - sh
        - -c
        - touch /mnt/test-init.txt
        image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
        name: init-touch
        volumeMounts:
        - name: data
          mountPath: /mnt
      - command:
        - sh
        - -c
        - for i in `seq 1 10`;do echo $i;sleep 1;done
        image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
        name: echo 
        volumeMounts:
        - name: data
          mountPath: /mnt
      containers:
      - image: registry.cn-beijing.aliyuncs.com/dotbalo/busybox
        imagePullPolicy: IfNotPresent
       # command: ["/bin/bash", "-ce", "tail -f /dev/null"]
        command: ["/bin/sh","-ce","sleep 3600"]
        name: test-init
        volumeMounts:
        - name: data
          mountPath: /mnt
[root@k8s-master01 10st]#

kubectl create  -f init.yaml

10.9 为什么要用临时容器？

临时容器:具有工具包+root权限，帮助排查pod里其他容器的问题，排查容器的容器。

10.10 使用临时容器在线debug

5节点都需要操作的步骤：master01-master03,node01-node02

vi /usr/lib/systemd/system/kube-proxy.service
--feature-gates=EphemeralContainers=true \

vi /etc/kubernetes/kubelet-conf.yml
featureGates:
  EphemeralContainers: true

master01-master03需要操作的步骤

vi /usr/lib/systemd/system/kube-apiserver.service
--feature-gates=EphemeralContainers=true \

vi /usr/lib/systemd/system/kube-controller-manager.service
--feature-gates=EphemeralContainers=true \

vi /usr/lib/systemd/system/kube-scheduler.service
--feature-gates=EphemeralContainers=true \

5个节点操作

systemctl  daemon-reload

systemctl  restart kube-apiserver    kube-scheduler  kube-controller-manager kubelet kube-proxy

直接在你的pod上注入一个容器，并打开控制台；

进入临时容器方式1

kubectl debug   metrics-server-595f65d8d5-zmc88    -ti --image=registry.cn-beijing.aliyuncs.com/dotbalo/debug-tools   -n kube-system

进入临时容器方式2

kubectl attach metrics-server-595f65d8d5-zmc88 -c debugger-b9s9j -i -t -n kube-system

进入临时容器方式3

 kubectl get po metrics-server-595f65d8d5-zmc88 -n kube-system -oyaml

10.11 Taint和Toleration设计理念

Taint（污点--锁）作用在节点（node）上，Toleration（容忍--钥匙）是作用在pod上。

10.12 污点和容忍配置解析

如果希望pod部署到指定节点，需要通过nodeselector或者Affinity，因为容忍和污点没有强制依赖

遇到才会生效。含有容忍点的pod遇到了（被scheduler 调度）对应的污点能容忍它，遇不到就是遇不到。类比回家开门，有很多门有的上锁（污点node），有的没上锁（无污点的node）。碰到没上锁的门直接进去，有锁的才需要钥匙（容忍点）。如果要pod强制走带锁的门,可以使用nodeselector或者Affinity去限制pod调度。

10.13 污点和容忍配置示例

kubectl taint nodes k8s-node01 ssd=true:NoSchedule

驱除pod测试

kubectl taint nodes k8s-node01 ssd=true:NoExecute

kubectl label  node k8s-node01  ssd=true

[root@k8s-master01 10st]#  cat toleration.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    env: test
spec:
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  nodeSelector:
    ssd: "true"
  tolerations:
  - key: "ssd"
    operator: "Exists"
[root@k8s-master01 10st]#

 kubectl create  -f toleration.yaml

注释掉容忍点测试

[root@k8s-master01 10st]# cat toleration.yaml 
apiVersion: v1
kind: Pod
metadata:
  name: nginx
  labels:
    env: test
spec:
  containers:
  - name: nginx
    image: nginx
    imagePullPolicy: IfNotPresent
  nodeSelector:
    ssd: "true"
  #tolerations:
  #- key: "ssd"
  #  operator: "Exists"
[root@k8s-master01 10st]#

pending原因查看

10.14 内置污点

deployment无容忍配置，创建pod时k8s会自动给pod创建容忍的配置；

10.15 节点宕机秒级恢复应用

[root@k8s-master01 10st]# cat toleration_seconds.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  creationTimestamp: null
  labels:
    app: tolerations-second
  name: tolerations-second
spec:
  replicas: 1
  selector:
    matchLabels:
      app: tolerations-second
  strategy: {}
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: tolerations-second
    spec:
      containers:
      - image: registry.cn-beijing.aliyuncs.com/dotbalo/nginx
        name: nginx
        resources:
          requests:
            cpu: 10m
      nodeSelector:
        ssd: "true"
      tolerations:
      - key: ssd
        operator: Equal
        value: "true"
      - effect: NoExecute
        key: node.kubernetes.io/unreachable
        operator: Exists
        tolerationSeconds: 10
      - effect: NoExecute
        key: node.kubernetes.io/notready
        operator: Exists
        tolerationSeconds: 10
[root@k8s-master01 10st]#

kubectl create -f toleration_seconds.yaml

此时将node01节点关机（vm控制台操作），40s后node01状态变为notready，观察pod是否漂移。

因为配置了nodeselector选项只选择node01，所以pod不会调度到其他节点。其他没有配置pod的状态上300s后才变为非running。

10.16 Taint命令入门

用的时候：key名,value和EFFECT一样，则判断为同一个taint;

配得时候，key名和effecf一样，则筛选为同一个taint修改；

查看某一个节点的污点

kubectl describe node k8s-node01 | grep Taints  -A 10

完全匹配

 kubectl taint  node k8s-node01 ssd=true:NoExecute-

污点已删除

污点重新添加上，继续测试

 kubectl taint  node k8s-node01 ssd=true:NoExecute

key名-

修改污点

kubectl taint  node k8s-node01 ssd=fasle:NoExecute  --overwrite

查看帮助

kubectl taint node -h

10.17 生产环境依旧存在的高可用率问题

10.18 Affinity分类

10.19 可用率保障-部署至不同宿主机

10.20 可用率保障-部署至不同机房或区

10.21 降低故障范围-不把鸡蛋放在同一个篮子里

10.22 节点亲和力配置详解

10.23 Pod亲和力和反亲和力配置详解

labselector是通过pod标签选择和哪个pod匹配；nodeslectors是通过节点（node）标签选择节点（node）。

10.24 实现同一个应用分布在不同的宿主机

[root@k8s-master01 10st]# cat pod-mulitinodes.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: must-be-diff-nodes
  name: must-be-diff-nodes
  namespace: kube-public
spec:
  replicas: 3
  selector:
    matchLabels:
      app: must-be-diff-nodes
      #project: multi
  template:
    metadata:
      labels:
        app: must-be-diff-nodes
       # project: multi
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - must-be-diff-nodes
            topologyKey: kubernetes.io/hostname
      containers:
      - image: nginx
        imagePullPolicy: IfNotPresent
        name: must-be-diff-nodes
[root@k8s-master01 10st]#

kubectl create  -f pod-mulitinodes.yaml

10.25 尽量将应用部署至高配置服务器

[root@k8s-master01 10st]# cat nodeAffinitySSD.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: prefer-ssd
  name: prefer-ssd
  namespace: kube-public
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prefer-ssd
  template:
    metadata:
      creationTimestamp: null
      labels:
        app: prefer-ssd
    spec:
      affinity:
        nodeAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - preference:
              matchExpressions:
              - key: ssd
                operator: In
                values:
                - "true"
              - key: gpu
                operator: NotIn
                values:
                - "true"
            weight: 100
          - preference:
              matchExpressions:
              - key: type
                operator: In
                values:
                - physical
            weight: 10
      containers:
      - image: nginx
        imagePullPolicy: IfNotPresent
        name: prefer-ssd-1
[root@k8s-master01 10st]#

kubectl create  -f nodeAffinitySSD.yaml

继续测试

kubectl label  nodes k8s-node01 ssd-

因为node02打了type=physical标签，所以被调度。

10.26 Topology的重要性

可以打区域标签，subnet网络区域标签；

打不同机房机柜标签，分到不同机房的不同机柜；逻辑上的划分，分为三个不同的域；

10.27 应用多地区多机房部署

实现同一应用多区域多机房部署

在一个域内，只能存在一个标签app=must-be-diff-zone的pod

[root@k8s-master01 10st]# cat must-be-diff-zone.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: must-be-diff-zone
  name: must-be-diff-zone
  namespace: kube-public
spec:
  replicas: 3
  selector:
    matchLabels:
      app: must-be-diff-zone
  template:
    metadata:
      labels:
        app: must-be-diff-zone
    spec:
      affinity:
        podAntiAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
          - labelSelector:
              matchExpressions:
              - key: app
                operator: In
                values:
                - must-be-diff-zone
            topologyKey: region
      containers:
      - image: nginx
        imagePullPolicy: IfNotPresent
        name: must-be-diff-zone
[root@k8s-master01 10st]#

kubectl create  -f must-be-diff-zone.yaml

副本改为4继续测试

---------------教程来源：51cto 杜宽老师k8s课程的学习笔记 -------------