在k8s中部署jenkins并通过pipeline发布项目

背景需求：

由于目前现有Jenkins上的项目多、杂，不仅有开发、测试、灰度、生产，而且还有一些其他的脚本实现的操作，尽管有账号权限划分，但还是显得很臃肿，其次就是经常出现多个项目同时打包导致jinkens卡死现象。因此需要将生产上的jenkens与其他环境分开，进行单独部署。

前置条件：

基于k8s集群1.18.0

step1：准备相关文件

1. 下载jdk-8u201安装包，见java环境（jdk1.8）安装-linux&windows
2. 下载maven-3.6.3安装包，见maven环境配置
3. k8s 1.18.0的kubectl命令，本地拷贝即可
4. maven的setting文件，此文件需要自己适配
5. k8s 集群的admin.conf认证配置文件，本地拷贝即可
   

step2：编写jenkins的dockerfile文件

cat Dockerfile

# 官方镜像
FROM jenkins/jenkins:lts
# root权限，需要操作k8s、打包
USER root
# 删除自带的jdk
RUN rm -rf /opt/java/openjdk/bin/java
# 自建jdk
ADD jdk-8u201-linux-x64.tar.gz /usr/local
ENV JAVA_HOME  /usr/local/jdk1.8.0_201
ENV CLASSPATH $JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
ENV PATH $PATH:$JAVA_HOME/bin
# 自建maven,settings.xml需配置好
# 这里我把仓库路径配置为：/var/jenkins_home/repository
ADD apache-maven-3.6.3-bin.tar.gz /usr/local
RUN mkdir -p /var/jenkins_home/repository && rm -rf /usr/local/apache-maven-3.6.3/conf/settings.xml
ADD settings.xml /usr/local/apache-maven-3.6.3/conf/
ENV MAVEN_HOME /usr/local/apache-maven-3.6.3
ENV PATH $MAVEN_HOME/bin:$PATH
# 安装kubeclt工具
ADD kubectl /usr/local/bin/kubectl
RUN ln -s /usr/local/bin/kubectl /usr/bin/kubectl
# 拷贝k8s的配置（master节点上的/etc/kubernetes/admin.conf）
RUN mkdir -p /root/.kube
ADD admin.conf /root/.kube/config
RUN chown $(id -u):$(id -g) /root/.kube/config
# 更改jenkins镜像的时区；添加hosts配置访问k8s的apiserver、访问内网maven私服、加速拉去镜像、
RUN echo 'Asia/Shanghai' >/etc/timezone \
    && echo '10.7.54.134 apiserver.my.cn' >> /etc/hosts \
    && echo '172.16.89.181  nexus.my.cn' >> /etc/hosts\
    && echo '172.16.18.64 harbor.my.cn' >> /etc/hosts

step3：编写jenkins服务的rbac文件

cat jenkins-service-account.yml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins-admin
secrets:
- name: jenkins-admin-token
---
kind: Role
#rbac的版本为官方定义的版本，可去官网查询
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: jenkins-admin
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]
- apiGroups: ["","extensions", "apps"]
  resources: ["deployments"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
- apiGroups: [""]
  resources: ["", "services", "endpoints"]
  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: jenkins-admin
roleRef: 
  kind: Role
  name: jenkins-admin
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: jenkins-admin

step4：编写jenkins的资源清单文件（deployment和service）

cat jenkins-dep.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  labels: 
    app: jenkins
spec:
  replicas: 1                #副本数为1
  selector:
    matchLabels:
      app: jenkins
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      serviceAccountName: jenkins-admin
      imagePullSecrets:
        - name: jenkins-admin-token
      containers:
      - name: dd-jenkins
        image: harbor.****.cn/k8s_basic_images/dd-jenkins:lts
        imagePullPolicy: Always
        securityContext:
          runAsUser: 0   #直接使用root权限启动，可以减少很多权限带来的问题
        ports:
        - containerPort: 8080
        volumeMounts:
        - mountPath: /var/jenkins_home  #为jenkins的数据目录
          name: jenkins-home 
        - name: docker   #docker的挂载不能少，否则在运行脚本时会找不到docker命令
          mountPath: /usr/bin/docker
        - name: docker-sock
          mountPath: /var/run/docker.sock
        - name: date-config
          mountPath: /etc/localtime
        - name: kube-config
          mountPath: /root/.kube
      volumes:
      - name: jenkins-home
        hostPath: 
          path: /data/jenkins_home
      - name: docker
        hostPath:
          path: /usr/bin/docker
      - name: docker-sock
        hostPath:
          path: /var/run/docker.sock
      - name: date-config
        hostPath:
          path: /etc/localtime
      - name: kube-config
        hostPath:
          path: /root/.kube
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  type: NodePort
  ports:
  - name: jenkins
    port: 8080 
    targetPort: 8080
    nodePort: 30011        #开启nodeport
  - name: jenkins-agent
    port: 50000 
    targetPort: 50000
    nodePort: 30010
  selector:
    app: jenkins

step5：创建jenkins服务

kubectl apply -f jenkins-service-account.yml
kubectl apply -f jenkins-dep.yaml

step6：验证

1、新建项目–流水线

2、创建并编写pipeline

3、运行流水线

step7：流水线代码

// harbor镜像仓库地址和密码信息
def registry = "harbor.my.cn" 
def registry_se = "5221ab7b-ecdssda-45fa-814a-063bce38c1df"
// gitlab地址和密码信息
def gitlab_url = "http://gitlab.my.cn/wkx/backend/xxxxxx.git"
def gitlab_se = "bb763521-0cf5-4099-86e5-130fddas0269ac9d"
// k8s 凭据信息
def k8s_auth = "4f2a66c9-2446-4bb6-8088-93a8xasxac9bddab5"
// harbor仓库的项目前缀信息
def prefix = "blue-k8s"
// 部署应用的服务名称
def app_name = "access-ultra-vires"
// 部署服务所在的命名空间、副本数、容器暴露的端口、svc的端口
def ns = "cool"
def rc = 2
def cport = 30111    #容器端口
def cluport = 30111  #cluster的port
// 构建编译的环境参数
def env = "blue"    #maven编译的环境参数

node(){
  stage("1.pull git code"){
	checkout([$class: 'GitSCM', branches: [[name: '*/master']], userRemoteConfigs: [[credentialsId: "${gitlab_se}", url: "${gitlab_url}"]]]) 	
		script {
            build_tag = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
            TM = sh(script: 'date +%Y%m%d%H%M')
        }
		image_name_build = "${registry}/${prefix}/${app_name}:${BUILD_NUMBER}"
		image_name_latest = "${registry}/${prefix}/${app_name}:latest"
		
  }

  stage("2.build code"){
	  sh "mvn clean package -P${env} -DskipTests"
  }
  stage("3.build docker and push to harbor") {
        withCredentials([usernamePassword(credentialsId: "${registry_se}", passwordVariable: 'dockerHubPassword', usernameVariable: 'dockerHubUser')]) {
            sh "docker login -u ${dockerHubUser} -p ${dockerHubPassword} ${registry}"
            sh "docker build -t ${image_name_build}  -f Dockerfile .  --no-cache"
            sh "docker tag ${image_name_build} ${image_name_latest}"
            sh "docker push ${image_name_build}"
            sh "docker push ${image_name_latest}"
        }
    }
  stage("4.deploy to k8s") {
        sh "/usr/bin/ossutil cp -uf oss://prod-***/api/blue/templete-v1.yaml templete-v1.yaml"
        sh "cp templete-v1.yaml ${app_name}-dep.yaml"
		sh "sed -i 's#SVC_NAME#${app_name}#g'  ${app_name}-dep.yaml"
		sh "sed -i 's#NS_NAME#${ns}#g'  ${app_name}-dep.yaml"
		sh "sed -i 's#RC_NUM#${rc}#g'  ${app_name}-dep.yaml"
		sh "sed -i 's#IMAGE_URL#${image_name_latest}#g'  ${app_name}-dep.yaml"
		sh "sed -i 's#CON_PORT#${cport}#g'  ${app_name}-dep.yaml"
		sh "sed -i 's#CLU_PORT#${cluport}#g'  ${app_name}-dep.yaml"
        sh "kubectl apply -f  ${app_name}-dep.yaml"
		echo "deploy success"
    }
}

cat templete-v1.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  labels:
    app: SVC_NAME
  name: SVC_NAME
  namespace: NS_NAME
spec:
  replicas: RC_NUM
  selector:
    matchLabels:
      app: SVC_NAME
  template:
    metadata:
      labels:
        app: SVC_NAME
    spec:
      containers:
      - image: IMAGE_URL
        imagePullPolicy: Always
        lifecycle:
          preStop:
            exec:
              command:
              - sleep
              - "30"
        livenessProbe:
          failureThreshold: 3
          initialDelaySeconds: 300
          periodSeconds: 10
          successThreshold: 1
          tcpSocket:
            port: CON_PORT
          timeoutSeconds: 10
        readinessProbe:
          failureThreshold: 3
          initialDelaySeconds: 30
          periodSeconds: 10
          successThreshold: 1
          tcpSocket:
            port: CON_PORT
          timeoutSeconds: 10
        name: SVC_NAME
      affinity:
        podAntiAffinity:
          preferredDuringSchedulingIgnoredDuringExecution:
          - weight: 100
            podAffinityTerm:
              labelSelector:
                matchExpressions:
                - key: app
                  operator: In
                  values:
                  - SVC_NAME
              topologyKey: "kubernetes.io/hostname"
      imagePullSecrets:
        - name: admin-token-cool
      restartPolicy: Always
      terminationGracePeriodSeconds: 60
---
apiVersion: v1
kind: Service
metadata:
  labels:
    app: SVC_NAME
  name: SVC_NAME
  namespace: NS_NAME
spec:
  ports:
  - port: CLU_PORT
    protocol: TCP
    targetPort: CON_PORT
  selector:
    app: SVC_NAME
  type: ClusterIP #NodePort/ClusterIP

step8：优化点

1、将pipeline写成Jenkinsfile放在项目代码中，更方便控制
2、将templete.yaml文件放在oss中是因为对默认文件的只需要一次修改就可以全部生效，如果每个项目的模板文件无法统一就单独存放到项目中或者oss多个不同文件中
3、配合jenkins的参数化构建，可以通过传参去实现pod副本数灵活部署