jenkins 使用 k8s 动态创建 jenkins slave pod

环境：centos7.6 、Docker version 19.03.5、Jenkins 2.249.3（jackson = 2.11.2、docker plugin = 1.2.1）、 kubernetes cli plugin 1.9.0、kubernetes plugin 1.27.7
主要思想： 使用 kubenetes cli plugin 配置一个动态的 pod jenkins jnlp slave，pod 中包含 mvn、docker、kubectl 等容器。创建多分支流水线，Jenkinsfile 使用前面配置好的 k8s 。在该 jenkins slave pod 中执行 maven 构建、docker build、jave 、kubectl apply 等操作。
完整项目见 github 项目
1、k8s 集群创建 kube-ops namespace 以及 jenkins2 serviceaccout 并绑定权限

kubect create ns kube-ops

[root@k8s-master ~]# cat clusterrolebinding.yaml 
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins2
  namespace: kube-ops

---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins2
rules:
  - apiGroups: ["extensions", "apps"]
    resources: ["deployments"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get","list","watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: jenkins2
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jenkins2
subjects:
  - kind: ServiceAccount
    name: jenkins2
    namespace: kube-ops

kubectl apply -f clusterrolebinding.yaml 

2、配置 kubernetes
“系统管理“ – ”节点管理“ – “configure clouds” – “add a new cloud” – “kubernetes”
配置： Kubernetes 地址、Kubernetes 命名空间、凭据、Jenkins 地址、Jenkins 通道、Pod Labels、
配置： Pod Templates 中 名称、命名空间、标签列表、容器列表（名称、docker 镜像、工作目录）、卷（这里选择 hostpath，挂载 /var/run/docker.sock 和 /root/.kube 目录）

service accout 选择前面创建的 jenkins2

3、创建多分支流水线
完整项目见 github 项目

Jenkinsfile 如下：

def label = "slave-${UUID.randomUUID().toString()}"

podTemplate(label: label, cloud: 'k8s', serviceAccount: 'jenkins2', containers: [
  containerTemplate(name: 'maven', image: 'maven:3.6-alpine', command: 'cat', ttyEnabled: true),
  containerTemplate(name: 'docker', image: 'docker:19.03.5', command: 'cat', ttyEnabled: true),
  containerTemplate(name: 'kubectl', image: 'cnych/kubectl', command: 'cat', ttyEnabled: true),
  containerTemplate(name: 'jnlp', image: 'cnych/jenkins:jnlp6',ttyEnabled: true)
], volumes: [
  hostPathVolume(mountPath: '/root/.m2', hostPath: '/var/run/m2'),
  hostPathVolume(mountPath: '/home/jenkins/.kube', hostPath: '/root/.kube'),
  hostPathVolume(mountPath: '/var/run/docker.sock', hostPath: '/var/run/docker.sock')
]) {
  node(label) {
    def APP_NAME = "k8sjnekinsslave"
    def APP_PORT = "8081"
    def NODE_PORT_DEV = "30050"
    def NODE_PORT_PRO = "32050"
    def REPLICAS = "1"
    def cicd_admin = "mapleaves"
    def myRepo = checkout scm
    def gitCommit = myRepo.GIT_COMMIT
    def gitBranch = myRepo.GIT_BRANCH
    def imageTag = sh(script: "git rev-parse --short HEAD", returnStdout: true).trim()
    def imageEndpoint = "mapleaves/k8sjnekinsslave-${gitBranch}"
    def IMAGE = "${imageEndpoint}:${imageTag}" 
    if (gitBranch != 'dev' && gitBranch != 'master'){
      echo "${gitBranch} 分支不参与执行，开始退出，如有疑问，请联系运维人员$cicd_admin"
      return     
    }    
    stage('单元测试') {
      echo "============================== 1.测试阶段 =============================="
      echo "branch name is ${gitBranch}"
    }
    stage('代码编译打包') {
      echo "============================== 2.代码编译打包阶段 =============================="
      try {
        container('maven') {
          sh "ls"
          sh "mvn clean package -s settings.xml -Dmaven.test.skip=true"
          sh "ls"
          sh "ls target"
        }
      } catch (exc) {
        println "构建失败 - ${currentBuild.fullDisplayName}"
        throw(exc)
      }
    }
    stage('构建 Docker 镜像') {
      echo "============================== 3.构建 Docker 镜像阶段  =============================="

      withCredentials([[$class: 'UsernamePasswordMultiBinding',
        credentialsId: 'mydockerhub',
        usernameVariable: 'dockerHubUser',
        passwordVariable: 'dockerHubPassword']]) { 
          container('docker') {   
            sh """
            sed -i 's/<APP_PORT>/${APP_PORT}/g' Dockerfile
            docker login -u ${dockerHubUser} -p ${dockerHubPassword}
            docker build -t ${IMAGE} .
            docker push ${IMAGE}
            """ 
          } 
      }
    }
    stage('部署 $APP_NAME  到 k8s') {
      echo "============================== 4.部署 $APP_NAME ${gitBranch} 分支到 k8s =============================="
      if (gitBranch == 'master') {      
        input "确认要部署到生产环境吗？"
        NAMESPACE = "pro"
        NODE_PORT = "${NODE_PORT_PRO}"
      }
      if (gitBranch == 'dev') {
        NAMESPACE = "dev"
        NODE_PORT = "${NODE_PORT_DEV}"
      }    
      withKubeConfig([credentialsId: 'k8s',contextName: 'kubernetes-admin@kubernetes',]) {
        container('kubectl') {    
        sh """
          sed -i 's/<APP_NAME>/${APP_NAME}/g' k8s.yaml
          sed -i 's/<APP_PORT>/${APP_PORT}/g' k8s.yaml
          sed -i 's/<NODE_PORT>/${NODE_PORT}/g' k8s.yaml
          sed -i 's/<REPLICAS>/${REPLICAS}/g' k8s.yaml
          sed -i 's?<IMAGE>?${IMAGE}?g' k8s.yaml
          sed -i 's/<NAMESPACE>/${NAMESPACE}/g' k8s.yaml
          """       
        sh "kubectl apply -f k8s.yaml --record"
        }          
      }   
    }
  }
}

Dockerfile 如下：

FROM openjdk:8-jdk-alpine

ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
ENV TZ=Asia/Shanghai

RUN mkdir /app

WORKDIR /app

COPY target/javawebdemo-1.0-SNAPSHOT.jar  /app/javawebdemo.jar

EXPOSE <APP_PORT>

ENTRYPOINT ["java", "-Djava.security.egd=file:/dev/./urandom", "-jar", "javawebdemo.jar", "--server.port=<APP_PORT>"]

k8s.yaml 如下：

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: <APP_NAME>
  namespace: <NAMESPACE>
  labels:
    app: <APP_NAME> 
spec:
  replicas: <REPLICAS>
  strategy:
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 1
    type: RollingUpdate
  template:
    metadata:
      labels:
        app: <APP_NAME>
    spec:
      restartPolicy: Always
      hostAliases:
      - ip: "10.2.7.1"
        hostnames:
        - "kafka01"
      - ip: "10.2.7.9"
        hostnames:
        - "kafka02"
      containers:
      - image: <IMAGE>
        name: <APP_NAME>
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: <APP_PORT>
          name: api
        resources:
          limits:
            cpu: 800m
            memory: 1200Mi
          requests:
            cpu: 50m
            memory: 600Mi          

---

kind: Service
apiVersion: v1
metadata:
  name: <APP_NAME>
  namespace: <NAMESPACE>
spec:
  selector:
    app: <APP_NAME>
  type:  NodePort
  ports:
  - name: api-port
    port: 8080
    targetPort:  api
    nodePort: <NODE_PORT>

点击立即构建或者更新 github 仓库代码，即可看到 k8s 集群中有 jenkins slave pod 创建，构建完成后该 pod 会被销毁。

[root@k8s-master ~]# kubectl get po -n kube-ops
NAME                                                     READY   STATUS    RESTARTS   AGE
slave-9af30823-2ecd-41c1-9662-6a86b594048d-l4792-4sbmr   4/4     Running   0          3m2s