jenkins部署K8S应用

 

通过独立的jenkins部署K8S应用

PS：这种方式是jenkins独立，然后slave已容器方式运行，master是独立出来的，减少运维成本，还能提高效率。

如果使用master部署需要把master加入k8s集群

jenkins安装不赘述

创建证书秘钥

在k8s-master节点上生成对应的证书

#1、查看kubernetes的config文件
cat ~/.kube/config
#2、根据配置文件生成证书.替换引号内部的信息为config内相关value
echo "certificate-authority-data" | base64 -d > ca.crt
echo "client-certificate-data" | base64 -d > client.crt
echo "client-key-data" | base64 -d > client.key
#3、生成jenkins使用的cert.pfx，此处需要设置一个4位数以上的密码
openssl pkcs12 -export -out cert.pfx -inkey client.key -in client.crt -certfile ca.crt
ls
ca.crt cert.pfx client.key client.crt

添加凭据

在jenkins里添加凭据

凭据类型选Certificate

上传cert.pfx，输入刚才设置的密码即可

安装插件

jenkins的地址为http://192.168.1.22:8080/

安装完毕后安装kubernetes插件重启

打开代理

manage jenkins ->configure global security -> 代理

制作jenkins-slave镜像

FROM jenkins/jnlp-slave:latest-jdk11
USER root

#修改镜像时区
RUN ln -sf /usr/share/zoneinfo/Asia/ShangHai /etc/localtime && \
    echo "Asia/Shanghai" > /etc/timezone && \
    dpkg-reconfigure -f noninteractive tzdata    #重新配置tzdata软件包,使得时区设置生效

#修改debia源
RUN echo "deb http://mirrors.163.com/debian/ buster main" > /etc/apt/sources.list && \
    echo "deb http://mirrors.163.com/debian/ buster-updates main non-free contrib" >> /etc/apt/sources.list && \
    echo "deb http://mirrors.163.com/debian-security/ buster/updates main non-free contrib" >> /etc/apt/sources.list
#安装docker相关依赖
RUN apt-get update && \
    apt-get -y install apt-transport-https ca-certificates curl gnupg2 apt-utils lsb-release software-properties-common && \
    curl -fsSL http://mirrors.aliyun.com/docker-ce/linux/debian/gpg | apt-key add - && \
    add-apt-repository "deb [arch=amd64] http://mirrors.aliyun.com/docker-ce/linux/debian $(lsb_release -cs) stable" && \
    apt-get -y update && \
    apt-get -y install docker-ce
#安装kubectl
RUN apt-get update && apt-get install -y apt-transport-https && \
    curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add - && \
    echo "deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list && \
    apt-get update && \
    apt-get install kubectl

docker build -t harbor.test.com/jenkins/jenkins-slave:v3 .

docker push harbor.test.com/jenkins/jenkins-slave:v3

配置K8S集群信息

重启后点击manage jenkins->manage nodes and clouds->configure clouds -> add a new cloud

或者直接http://192.168.1.22:8080/manage/configureClouds/

填写集群信息

填写k8s集群api-server的url，kubernetes服务证书为刚刚生成的ca.crt，凭据为刚刚添加的，填写Jenkins地址，点击连接测试验证

连接测试

填写jenkins信息

配置jenkins地址

添加POD模板

添加 Pod 模板------>Kubernetes Pod Template--->按如下配置

往下拉有个sa信息，填写刚才的sa

添加卷

添加容器模板

添加容器------>Container Template------>按如下配置------>

容器填写刚才制作的镜像

完成。

---

通过容器版jenkins部署K8S

创建名称空间

kubectl create namespace jenkins-k8s

创建 pv

apiVersion: v1
kind: PersistentVolume
metadata:
 name: jenkins-k8s-pv
spec:
 capacity:
 storage: 10Gi
 accessModes:
 - ReadWriteMany
 nfs:
 server: 192.168.1.63
 path: /data/v1

kubectl apply -f pv.yaml

创建 pvc

kind: PersistentVolumeClaim
apiVersion: v1
metadata:
 name: jenkins-k8s-pvc
 namespace: jenkins-k8s
spec:
 resources:
 requests:
 storage: 10Gi
 accessModes:
 - ReadWriteMany

kubectl apply -f pvc.yaml

查看 pvc 和 pv 绑定是否成功

kubectl get pvc -n jenkins-k8s

创建一个 sa 账号

kubectl create sa jenkins-k8s-sa -n jenkins-k8s

把上面的 sa 账号做 rbac 授权

kubectl create clusterrolebinding jenkins-k8s-sa-cluster --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa

通过 deployment 部署 jenkins

docker pull jenkins/jenkins:latest

kind: Deployment
apiVersion: apps/v1
metadata:
  name: jenkins
  namespace: jenkins-k8s
spec:
  replicas: 1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccount: jenkins-k8s-sa
      containers:
      - name: jenkins
        image:  jenkins/jenkins:latest
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          name: web
          protocol: TCP
        - containerPort: 50000
          name: agent
          protocol: TCP
        resources:
          limits:
            cpu: 1000m
            memory: 1Gi
          requests:
            cpu: 500m
            memory: 512Mi
        livenessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        readinessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        volumeMounts:
        - name: jenkins-volume
          subPath: jenkins-home
          mountPath: /var/jenkins_home
      volumes:
      - name: jenkins-volume
        persistentVolumeClaim:
          claimName: jenkins-k8s-pvc

kubectl apply -f jenkins-deployment.yaml

kubectl get pods -n jenkins-k8s

注：如果看到CrashLoopBackOff，解决方法如下：

#查看 jenkins-675b586fdb-5mzmk 日志

[root@xuegod63 ~]# kubectl logs jenkins-675b586fdb-5mzmk -n jenkins-k8s

touch: cannot touch '/var/jenkins_home/copy_reference_file.log': Permission denied

#上面问题是因为/data/v1 目录权限问题，按如下方法解决：

chown -R 1000.1000 /data/v1/

重新部署deployment即可

把 jenkins 前端加上 service，提供外部网络访问

apiVersion: v1
kind: Service
metadata:
  name: jenkins-service
  namespace: jenkins-k8s
  labels:
    app: jenkins
spec:
  selector:
    app: jenkins
  type: NodePort
  ports:
  - name: web
    port: 8080
    targetPort: web
    nodePort: 30002
  - name: agent
    port: 50000
    targetPort: agent

kubectl apply -f jenkins-service.yaml

kubectl get svc -n jenkins-k8s

配置 Jenkins

在浏览器访问 jenkins 的 web 界面：

http://192.168.1.5:30002/

初始化不赘述。

安装kubernetes插件

配置 jenkins 连接到我们存在的 k8s 集群

jenkins地址填http://jenkins-service.jenkins-k8s.svc.cluster.local:8080/

POD和容器配置和上半部分一样即可。

---

流水线脚本

通用

node('testhan') {
    stage('Clone') {
        // sh "git config --global user.name 'root'"
        // sh "git config --global user.email 'admin@example.com'"

        // echo "1.Clone Stage"
        // git url: "git@192.168.1.20:test/jenkins-test.git"
        // script {
        //     build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
        // }
        deleteDir()
        git branch: 'main', credentialsId: 'gitlab', url: 'http://192.168.1.20/test/jenkins-test.git'
    }
    stage('Test') {
        echo "2.Test Stage"

    }
    stage('Build') {
        echo "3.Build Docker Image Stage"
        sh "docker build -t harbor.test.com/test-image/jenkins-demo:${build_tag} ."
    }
    stage('Push') {
        echo "4.Push Docker Image Stage"
        withCredentials([usernamePassword(credentialsId: 'dockerhub', passwordVariable: 'dockerHubPassword', usernameVariable: 'dockerHubUser')]) {
            sh "docker login harbor.test.com -u ${dockerHubUser} -p ${dockerHubPassword}"
            sh "docker push harbor.test.com/test-image/jenkins-demo:${build_tag}"
        }
    }
    stage('Deploy to dev') {
        echo "5. Deploy DEV"
        sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-dev.yaml"
        sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-dev.yaml"
        //        sh "bash running-devlopment.sh"
        sh "kubectl apply -f k8s-dev.yaml  --validate=false"
    }	
    stage('Promote to qa') {	
        def userInput = input(
            id: 'userInput',

            message: 'Promote to qa?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-qa.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-qa.yaml"
            //            sh "bash running-qa.sh"
            sh "kubectl apply -f k8s-qa.yaml --validate=false"
            sh "sleep 6"
            sh "kubectl get pods -n qatest"
        } else {
            //exit
        }
  }
    stage('Promote to pro') {	
        def userInput = input(

            id: 'userInput',
            message: 'Promote to pro?',
            parameters: [
                [
                    $class: 'ChoiceParameterDefinition',
                    choices: "YES\nNO",
                    name: 'Env'
                ]
            ]
        )
        echo "This is a deploy step to ${userInput}"
        if (userInput == "YES") {
            sh "sed -i 's/<BUILD_TAG>/${build_tag}/' k8s-prod.yaml"
            sh "sed -i 's/<BRANCH_NAME>/${env.BRANCH_NAME}/' k8s-prod.yaml"
            //            sh "bash running-production.sh"
            sh "cat k8s-prod.yaml"
            sh "kubectl apply -f k8s-prod.yaml --record --validate=false"
        }
    }
}