基于Kubernetes的jenkins上线

1、基于helm 部署jenkins

要求：当前集群配置了storageClass，并已指定默认的storageClass，一般情况下，创建的storageClass即为默认类 指定默认storageClass的方式

# 如果是新创建默认类：
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
  annotations:
    # 这里注解说明了这个是默认的storageclass
    storageclass.kubernetes.io/is-default-class: "true"
provisioner: fuseim.pri/ifs 
parameters:
  archiveOnDelete: "true"
​
# 如果是修改为默认类
kubectl patch storageclass managed-nfs-storage -p '{"metadata": {"annotations":{"storageclass.kubernetes.io/is-default-class":"true"}}}'
​
# 检查,可以看到类后面增加了(default)
[root@kube-master1 ~]# kubectl get sc
NAME                            PROVISIONER                    RECLAIMPOLICY   VOLUMEBINDINGMODE      ALLOWVOLUMEEXPANSION   AGE
managed-nfs-storage (default)   gxf-nfs-storage                Retain          Immediate              false                  41d

1.1 部署helm

[root@kube-master ~]# wget https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz
[root@kube-master ~]# tar zxvf helm-v3.2.4-linux-amd64.tar.gz 
[root@kube-master ~]# mv linux-amd64/helm /usr/bin/

1.2 部署jenkins

# 1、添加jenkins仓库
[root@kube-master ~]# helm repo add jenkinsci https://charts.jenkins.io && helm repo update
​
# 2、安装jenkins
## 2.1 如果需要自定义配置，需要先将chart下载下来，如果无需修改配置，直接install即可
[root@kube-master ~]# helm pull jenkinsci/jenkins
[root@kube-master ~]# tar xf jenkins-4.8.4.tgz # 修改目录中的values.yml
[root@kube-master ~]# helm install jenkins . -f jenkins-values.yaml
## 2.2 直接安装
[root@kube-master ~]# helm install jenkins jenkinsci/jenkins

1.3 检查 jenkins

# helm list
NAME    NAMESPACE       REVISION        UPDATED                                 STATUS          CHART           APP VERSION
jenkins default         1               2023-12-17 14:19:50.59739362 +0800 CST  deployed        jenkins-4.8.4   2.426.1 
​
[root@kube-master1 ~]# kubectl get pod
NAME                                      READY   STATUS    RESTARTS       AGE
jenkins-0                                 2/2     Running   0              2m30s
nfs-client-provisioner-66bc5457d6-vbzfj   1/1     Running   16 (20m ago)   38d
​
# 需要查看登录密码，可以使用
[root@kube-master1 ~]# helm get notes jenkins
NOTES:
1. Get your 'admin' user password by running:
  kubectl exec --namespace default -it svc/jenkins -c jenkins -- /bin/cat /run/secrets/additional/chart-admin-password && echo
2. Get the Jenkins URL to visit by running these commands in the same shell:
  echo http://127.0.0.1:8080
  kubectl --namespace default port-forward svc/jenkins 8080:8080
​
3. Login with the password from step 1 and the username: admin
4. Configure security realm and authorization strategy
5. Use Jenkins Configuration as Code by specifying configScripts in your values.yaml file, see documentation: http://127.0.0.1:8080/configuration-as-code and examples: https://github.com/jenkinsci/configuration-as-code-plugin/tree/master/demos
​
For more information on running Jenkins on Kubernetes, visit:
https://cloud.google.com/solutions/jenkins-on-container-engine
​
For more information about Jenkins Configuration as Code, visit:
https://jenkins.io/projects/jcasc/
​
​
NOTE: Consider using a custom image with pre-installed plugins

1.4 配置访问

# 这是当前jenkins的服务，将他改为nodeport，或通过ingress代理
[root@kube-master1 ~]# kubectl get svc jenkins
NAME      TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE
jenkins   ClusterIP   10.99.36.63   <none>        8080/TCP   32h
​
[root@kube-master1 ~]# vim ingress.yml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  labels:
    name: myingress
  name: myingress
  namespace: default
spec:
  ingressClassName: nginx
  rules:
  - host: qf.jenkins.vip
    http:
      paths:
      - backend:
          service:
            name: jenkins
            port:
              number: 8080
        path: /
        pathType: Prefix

2、安装插件

插件列表：
将准备好的插件包，替换掉对应的pv数据卷中的plugins目录

3、准备工作

3.1 准备ruoyi数据

[root@kube-master1 ~]# git clone https://gitea.beyourself.org.cn/newrain001/RuoYi-Vue-cloud.git
[root@kube-master1 ~]# cd RuoYi-Vue-cloud/
[root@kube-master1 RuoYi-Vue-cloud]# ls
bin  cloud  doc  LICENSE  pom.xml  README.md  ruoyi-admin  ruoyi-common  ruoyi-framework  ruoyi-generator  ruoyi-quartz  ruoyi-system  ruoyi-ui  ry.bat  ry.sh  sql
[root@kube-master1 RuoYi-Vue-cloud]# cd sql/
[root@kube-master1 sql]# ls
quartz.sql  ry_20230223.sql
[root@kube-master1 sql]# kubectl create configmap ruoyi-init-data --from-file=.

3.2 准备k8s证书

[root@kube-master1 ~]# kubectl create secret generic kubeconfig --from-file=/root/.kube/config
​

3.3 准备maven配置文件

[root@kube-master1 ~]# vim settings.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.2.0"
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.2.0 https://maven.apache.org/xsd/settings-1.2.0.xsd">
  <pluginGroups>
  </pluginGroups>
  <proxies>
  </proxies>
  <servers>
  </servers>
  <mirrors>
    <mirror>
        <id>aliyunmaven</id>
        <mirrorOf>*</mirrorOf>
        <name>阿里云公共仓库</name>
        <url>https://maven.aliyun.com/repository/public</url>
    </mirror>
  </mirrors>
  <profiles>
  </profiles>
</settings>
[root@kube-master1 ~]# kubectl create configmap maven-repo --from-file=settings.xml

3.4 配置钉钉插件

在系统管理的下方有未归类的插件

4、项目配置

4.1 创建pipeline项目

4.2 配置参数

# 参数内容:
字符参数：
PROJECT_NAME 项目名称
凭据参数：
DOCKER_REPOSITORY_CREDENTIAL_ID 需要推送的docker镜像仓库凭据id，可指定多个，然后加介绍
选项参数：
HARBOR_HOST 需要推送的镜像仓库，可以指定多个，需要与凭据对应
选项参数：
NAMESPACE_NAME 需要推送的命名空间
字符参数：后端镜像的tag
JAVA_TAG
字符参数：前端镜像的tag
NODE_TAG
字符参数：
JAVA_REPLICAS 后端副本数
字符参数：
NODE_REPLICAS 前端副本数
字符参数：
JENKINS_URL 当前jenkins的url，通知时使用，一般不变，可以设置默认值 

5、构建项目

6、cloud 目录内的文件解析

6.1 cloud/Jenkinsfile

pipeline {
    // 定义流水线中使用的环境变量
    environment {
        PROJECT_NAME = "${PROJECT_NAME}"  // 项目名称
        DOCKER_REPOSITORY_CREDENTIAL_ID = "${DOCKER_REPOSITORY_CREDENTIAL_ID}"  // Docker仓库的凭证ID
        HARBOR_HOST = "${HARBOR_HOST}"  // Docker Harbor主机
        NAMESPACE_NAME = "${NAMESPACE_NAME}"  // Kubernetes命名空间
        JAVA_REPOSITORY_NAME = "ruoyi-java"  // Java应用的仓库名称
        NODE_REPOSITORY_NAME = "ruoyi-node"  // Node.js应用的仓库名称
        JAVA_TAG = "${JAVA_TAG}"  // Java应用的Docker标签
        NODE_TAG = "${NODE_TAG}"  // Node.js应用的Docker标签
        JAVA_REPLICAS = "${JAVA_REPLICAS}"  // Java应用的副本数
        NODE_REPLICAS = "${NODE_REPLICAS}"  // Node.js应用的副本数
        YAML_PATH = "cloud/deploy/"  // Kubernetes YAML文件的路径
        JAVA_DEPLOYMENT_NAME = "ruoyi-java.yaml"  // Java部署的YAML文件名称
        NODE_DEPLOYMENT_NAME = "ruoyi-nginx.yaml"  // Node.js部署的YAML文件名称
        STATUS_URL = "${JENKINS_URL}/job/ruoyi/${BUILD_NUMBER}"  // 查看构建状态的URL
        CONSOLE_URL = "${JENKINS_URL}/job/ruoyi/${BUILD_NUMBER}/console"  // 查看构建控制台输出的URL
    }

    // 使用Kubernetes定义代理配置
    agent {
        kubernetes {
            cloud "kubernetes"
            yaml """
            // 定义Kubernetes Pod的规格
            apiVersion: v1
            kind: Pod
            spec:
              // 定义Pod中的容器
              containers:
              - name: jnlp
                image: jenkins/inbound-agent:3107.v665000b_51092-15
                args: ['\$(JENKINS_SECRET)', '\$(JENKINS_NAME)']
                imagePullPolicy: IfNotPresent
                volumeMounts:
                - mountPath: "/etc/localtime"
                  name: "volume-2"
                  readOnly: false
              - name: maven
                image: maven:3.6.3-jdk-11
                command: ['cat']
                tty: true
                volumeMount:
                - mountPath: "/usr/share/maven/ref/"
                  name: "maven-repo"
                  subPath: "settings.xml"
                  readOnly: true
                - mountPath: "/root/.m2"
                  name: "maven-data"
              - name: nodejs
                image: node:14
                command: ['cat']
                tty: true
              - name: kubectl
                image: kubesphere/kubectl:v1.22.0
                imagePullPolicy: IfNotPresent
                tty: true
                command: ["cat"]
                volumeMounts:
                - mountPath: "/etc/localtime"
                  name: "volume-2"
                  readOnly: false
                - mountPath: "/var/run/docker.sock"
                  name: "volume-docker"
                  readOnly: false
                - mountPath: "/root/.kube/config"
                  subPath: config
                  name: "kubeconfig"
                  readOnly: false
              - name: docker
                image: docker:19.03.15-git
                command: ['cat']
                tty: true
                volumeMounts:
                - mountPath: "/var/run/docker.sock"
                  name: "volume-docker"
                  readOnly: false
              // 定义Pod使用的卷
              volumes:
              - name: volume-2
                hostPath:
                  path: "/usr/share/zoneinfo/Asia/Shanghai"
              - name: volume-docker
                hostPath:
                  path: "/var/run/docker.sock"
              - name: kubeconfig
                secret:
                  secretName: kubeconfig
                  items:
                  - key: config
                    path: config
              - name: maven-repo
                configMap:
                  name: maven-repo
              - name: maven-data
                hostPath: 
                  path: "/opt/data/m2"
            """
        }
    }

    // 定义流水线的各个阶段
    stages {
        // 第一阶段：Maven打包
        stage('Maven 打包') {
            steps {
                // 使用Maven容器来构建Java应用
                container('maven') {
                    sh """
                       mvn clean package -Dmaven.test.skip=true && cp ruoyi-admin/target/ruoyi-admin.jar cloud/ruoyi-java
                    """
                }
            }
            post {
                // Maven打包成功或失败后的通知
                success {
                    // Maven打包成功时的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
                failure {
                    // Maven打包失败时的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
            }
        }

        // 第二阶段：Node.js打包
        stage('Node.js 打包') {
            steps {
                container('nodejs') {
                    sh """
                       cd ruoyi-ui && npm install --registry http://registry.npmmirror.com && npm run build:prod && cp -r dist ../cloud/ruoyi-nginx
                    """
                }
            }
            post {
                success {
                    // Node.js打包成功的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
                failure {
                    // Node.js打包失败的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
            }
        }

        // 第三阶段：构建镜像
        stage('构建镜像') {
            steps {
                withCredentials([usernamePassword(credentialsId: env.DOCKER_REPOSITORY_CREDENTIAL_ID, passwordVariable: 'PASSWORD', usernameVariable: 'USERNAME')]) {
                    container('docker') {
                        sh """
                          docker build -t ${env.HARBOR_HOST}/${env.NAMESPACE_NAME}/${env.JAVA_REPOSITORY_NAME}:${env.JAVA_TAG} cloud/ruoyi-java
                          docker build -t ${env.HARBOR_HOST}/${env.NAMESPACE_NAME}/${env.NODE_REPOSITORY_NAME}:${env.NODE_TAG} cloud/ruoyi-nginx
                          docker login ${env.HARBOR_HOST} --username ${env.USERNAME} --password ${env.PASSWORD}
                          docker push ${env.HARBOR_HOST}/${env.NAMESPACE_NAME}/${env.JAVA_REPOSITORY_NAME}:${env.JAVA_TAG}
                          docker push ${env.HARBOR_HOST}/${env.NAMESPACE_NAME}/${env.NODE_REPOSITORY_NAME}:${env.NODE_TAG}
                        """
                    }
                }
            }
            post {
                success {
                    // 构建镜像成功的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
                failure {
                    // 构建镜像失败的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
            }
        }

        // 第四阶段：部署到kubernetes
        stage('部署到kubernetes') {
            steps {
                container('kubectl') {
                    sh """
                      sed -i "s/REPLICAS/${env.JAVA_REPLICAS}/;s/HARBOR_HOST/${env.HARBOR_HOST}/;s/NAMESPACE_NAME/${env.NAMESPACE_NAME}/;s/REPOSITORY_NAME/${env.JAVA_REPOSITORY_NAME}/;s/TAG/${env.JAVA_TAG}/" ${env.YAML_PATH}${env.JAVA_DEPLOYMENT_NAME}
                      sed -i "s/REPLICAS/${env.NODE_REPLICAS}/;s/HARBOR_HOST/${env.HARBOR_HOST}/;s/NAMESPACE_NAME/${env.NAMESPACE_NAME}/;s/REPOSITORY_NAME/${env.NODE_REPOSITORY_NAME}/;s/TAG/${env.NODE_TAG}/" ${env.YAML_PATH}${env.NODE_DEPLOYMENT_NAME}
                      kubectl apply -f ${env.YAML_PATH} --record
                    """
                }
            }
            post {
                success {
                    // 部署成功的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
                failure {
                    // 部署失败的钉钉消息配置
                    dingtalk (
                        // 其他配置...
                    )
                }
            }
        }
    }
}

6.2 cloud/deploy/ruoyi-nginx.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: ruoyi-nginx
spec:
  replicas: REPLICAS
  selector:
    matchLabels:
      app: ruoyi-nginx
  template:
    metadata:
      labels:
        app: ruoyi-nginx
    spec:
      containers:
      - name: ruoyi-nginx
        image: HARBOR_HOST/NAMESPACE_NAME/REPOSITORY_NAME:TAG
        imagePullPolicy: Always
        ports:
        - containerPort: 80

---
apiVersion: v1
kind: Service
metadata:
  name: ruoyi-nginx-service
spec:
  selector:
    app: ruoyi-nginx
  ports:
  - port: 80
    targetPort: 80

---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: ruoyi-nginx-ingress
  labels:
    name: ruoyi-nginx-ingress
spec:
  ingressClassName: nginx
  rules:
  - host: ruoyi.nginx.com
    http:
      paths:
      - pathType: Prefix
        path: "/"
        backend:
          service:
            name: ruoyi-nginx-service
            port: 
              number: 80

6.3 cloud/deploy/ruoyi-java.yamll

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ruoyi-java
spec:
  replicas: REPLICAS
  selector:
    matchLabels:
      app: ruoyi-java
  template:
    metadata:
      labels:
        app: ruoyi-java
    spec:
      containers:
      - name: ruoyi-java
        image: HARBOR_HOST/NAMESPACE_NAME/REPOSITORY_NAME:TAG
        imagePullPolicy: Always
        ports:
        - containerPort: 8080

---
apiVersion: v1
kind: Service
metadata:
  name: ruoyi-java-service
spec:
  selector:
    app: ruoyi-java
  ports:
  - port: 8080
    targetPort: 8080

6.4 cloud/deploy/mysql.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: database
spec:
  selector:
    matchLabels:
      app: database
  template:
    metadata:
      labels:
        app: database
    spec:
      containers:
      - name: database
        image: daocloud.io/library/mysql:5.7
        env:
        - name: MYSQL_ROOT_PASSWORD
          value: "123456"
        - name: MYSQL_DATABASE
          value: "ry"
        volumeMounts:
          - mountPath: /docker-entrypoint-initdb.d
            name: ruoyi-data
        resources:
          limits:
            memory: "500Mi"
            cpu: "500m"
        ports:
        - containerPort: 3306
      volumes:
      - name: ruoyi-data
        configMap:
          name: ruoyi-init-data

---
apiVersion: v1
kind: Service
metadata:
  name: ruoyi-db
spec:
  selector:
    app: database
  ports:
  - port: 3306
    targetPort: 3306

6.5 cloud/deploy/redis.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: redis
spec:
  selector:
    matchLabels:
      app: redis
  template:
    metadata:
      labels:
        app: redis
    spec:
      containers:
      - name: redis
        image: redis
        resources:
          limits:
            memory: "128Mi"
            cpu: "500m"
        ports:
        - containerPort: 6379

---
apiVersion: v1
kind: Service
metadata:
  name: ruoyi-redis
spec:
  selector:
    app: redis
  ports:
  - port: 6379
    targetPort: 6379

6.6 cloud/ruoyi-java/Dockerfile

FROM buildo/java8-wkhtmltopdf:latest

COPY ruoyi-admin.jar /opt

EXPOSE 8080

CMD ["java", "-jar", "/opt/ruoyi-admin.jar"]

6.7 cloud/ruoyi-nginx/Dockerfile