kubernetes使用jenkinsci/kubernetes-plugin完成持续构建与发布

github:https://github.com/jenkinsci/kubernetes-plugin

参考:https://blog.csdn.net/aixiaoyang168/article/details/79767649

一、部署jenkins master

首先从github的说明可以看到jenkins的配置文件在src/main/kubernetes路径里面

分别是jenkins.ymlservice-account.yml

github是使用StatefulSet的方式部署jenkins,并且指定了ingress策略。这里我也使用StatefulSet的方式部署,由于本人使用的是kubernates v1.16.0,版本较新,所以需要修改github提供的配置文件才行

注:前提是已经做好storageClass动态申请存储卷,这里我是基于nfs做的动态存储卷

jenkins.yml

# jenkins

---
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  serviceName: jenkins
  replicas: 1
  selector:
    matchLabels:
      name: jenkins
  updateStrategy:
    type: RollingUpdate
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccount: jenkins
      # serviceAccount: nfs-provisioner # 和创建的资源访问角色(nfs-provisioner)保持一致
      containers:
        - name: jenkins
          # image: jenkins/jenkins:lts-alpine
          image: jenkins/jenkins:lts-alpine
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 8080
            name: jenkins-http
            protocol: TCP
          - containerPort: 50000
            name: jenkins-agent
            protocol: TCP
          resources:
            limits:
              cpu: 2048m
              memory: 2Gi
            requests:
              cpu: 512m
              memory: 512Mi
          env:
          - name: LIMITS_MEMORY
            valueFrom:
              resourceFieldRef:
                resource: limits.memory
                divisor: 1Mi
          - name: JAVA_OPTS
          # value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
            value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          volumeMounts:
          - name: jenkins-home
            mountPath: /var/jenkins_home
          livenessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12 # ~2 minutes
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12 # ~2 minutes
      #securityContext:
        # runAsUser: 1000
        # fsGroup: 1000
  volumeClaimTemplates:
  - metadata:
      name: jenkins-home
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: nfs # 和定义的storageclass的名称保持一致
      resources:
        requests:
          storage: 1Gi

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
spec:
  # type: LoadBalancer
  selector:
    name: jenkins
  # ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
  #externalTrafficPolicy: Local
  ports:
  - name: jenkins-http
    port: 8080
    targetPort: 8080
    protocol: TCP
  - name: jenkins-agent
    port: 50000
    targetPort: 50000
    protocol: TCP

---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: jenkins
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    kubernetes.io/tls-acme: "true"
    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
    nginx.ingress.kubernetes.io/proxy-body-size: 50m
    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
    # For nginx-ingress controller < 0.9.0.beta-18
    ingress.kubernetes.io/ssl-redirect: "true"
    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
    ingress.kubernetes.io/proxy-body-size: 50m
    ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
  rules:
  - http:
      paths:
      - path: /
        backend:
          serviceName: jenkins
          servicePort: 8080 # 与service的端口对应
    host: jenkins.example.com
  tls:
  - hosts:
    - jenkins.example.com
    secretName: tls-jenkins

如果不想部署statefulset的jenkins,也可以修改一下yml,部署deployment的jenkins,如下

# jenkins
 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: jenkins
  labels:
    name: jenkins
spec:
  replicas: 1
  selector:
    matchLabels:
      name: jenkins
  template:
    metadata:
      name: jenkins
      labels:
        name: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccount: jenkins
      # serviceAccount: nfs-provisioner # 和创建的资源访问角色(nfs-provisioner)保持一致
      containers:
        - name: jenkins
          # image: jenkins/jenkins:lts-alpine
          image: jenkins/jenkins:lts-alpine
          imagePullPolicy: IfNotPresent
          ports:
          - containerPort: 8080
            name: jenkins-http
            protocol: TCP
          - containerPort: 50000
            name: jenkins-agent
            protocol: TCP
          resources:
            limits:
              cpu: 2048m
              memory: 2Gi
            requests:
              cpu: 512m
              memory: 512Mi
          env:
          - name: LIMITS_MEMORY
            valueFrom:
              resourceFieldRef:
                resource: limits.memory
                divisor: 1Mi
          - name: JAVA_OPTS
          # value: -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMFraction=1 -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
            value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85
          volumeMounts:
          - name: jenkins-home
            mountPath: /var/jenkins_home
          livenessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12 # ~2 minutes
          readinessProbe:
            httpGet:
              path: /login
              port: 8080
            initialDelaySeconds: 60
            timeoutSeconds: 5
            failureThreshold: 12 # ~2 minutes
      #securityContext:
        # runAsUser: 1000
        # fsGroup: 1000
      volumes:
      - name: jenkins-home
        persistentVolumeClaim:
          claimName: pvc-jenkins-home
 
---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
spec:
  # type: LoadBalancer
  selector:
    name: jenkins
  # ensure the client ip is propagated to avoid the invalid crumb issue when using LoadBalancer (k8s >=1.7)
  #externalTrafficPolicy: Local
  ports:
  - name: jenkins-http
    port: 8080
    targetPort: 8080
    protocol: TCP
  - name: jenkins-agent
    port: 50000
    targetPort: 50000
    protocol: TCP
 
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: jenkins
  annotations:
    nginx.ingress.kubernetes.io/ssl-redirect: "true"
    kubernetes.io/tls-acme: "true"
    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
    nginx.ingress.kubernetes.io/proxy-body-size: 50m
    nginx.ingress.kubernetes.io/proxy-request-buffering: "off"
    # For nginx-ingress controller < 0.9.0.beta-18
    ingress.kubernetes.io/ssl-redirect: "true"
    # "413 Request Entity Too Large" uploading plugins, increase client_max_body_size
    ingress.kubernetes.io/proxy-body-size: 50m
    ingress.kubernetes.io/proxy-request-buffering: "off"
spec:
  rules:
  - http:
      paths:
      - path: /
        backend:
          serviceName: jenkins
          servicePort: 8080 # 与service的端口对应
    host: jenkins.example.com
  tls:
  - hosts:
    - jenkins.example.com
    secretName: tls-jenkins

service-account.yml

# In GKE need to get RBAC permissions first with
# kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin [--user=<user-name>|--group=<group-name>]

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins
rules:
- apiGroups: [""]
  resources: ["pods"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/exec"]
  verbs: ["create","delete","get","list","patch","update","watch"]
- apiGroups: [""]
  resources: ["pods/log"]
  verbs: ["get","list","watch"]
- apiGroups: [""]
  resources: ["secrets"]
  verbs: ["get"]

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: jenkins
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: jenkins
subjects:
- kind: ServiceAccount
  name: jenkins

 

kubectl describe pods jenkins-0

报错提示

pod has unbound immediate PersistentVolumeClaims

查看pvc列表发现有个jenkins-home-jenkins-0的pvc,AGE竟然是45h,回想起来我之前在没部署好nfs-

[root@k8s-master ~/docker-about/image-and-yml/jenkins]# kubectl get pvc
NAME                     STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
jenkins-home-jenkins-0   Pending                                                                                       45h
pvc-mysql-data           Bound     pv-mysql-data                              20Gi       RWO            nfs            5d22h
pvc-redis-data           Bound     pv-redis-data                              5Gi        RWO            nfs            4d6h
test-web-0               Bound     pvc-d8cb8112-86f5-4b56-a830-04574412e165   1Gi        RWO            nfs            26h
test-web-1               Bound     pvc-f2b7dbaf-6478-407d-9cd1-3d080efcfffa   1Gi        RWO            nfs            26h

 

注意: 初始化过程中,让输入 /var/jenkins_home/secret/initialAdminPassword 初始密码时,因为我们设置的 emptyDir: {}没有挂载到外部路径,可以进入到容器内部进行获取。

 

安装完插件,创建帐号后,就能看到界面

 

二、安装插件Kubernetes

管理员账户登录 Jenkins Master 页面,点击 “系统管理” —> “管理插件” —> “可选插件” —> “Kubernetes” 勾选安装即可。

`注:没看到有Kubernetes plugin插件,只有kubernetes,原来是插件名字变了`

 

点击安装,发现它一共安装两个插件kubernetes、kubernetes-client-api,由于我在线安装失败,所以到官网下载插件后,通过上传方式安装

下载的插件

通过上传方式安装,“系统管理” —> “管理插件” —> “可选插件” —> “高级”

安装后查看“已安装的插件”发现kubernetes plugin已经安装

 

 

三、Jenkins 配置 Kubernetes Plugin

安装完毕后,

点击 “系统管理” —> “系统设置” —> “新增一个云” —> 选择 “Kubernetes”,然后填写 Kubernetes 和 Jenkins 配置信息。

说明一下:

先查看自己的svc

kubectl get svc

NAME         TYPE        CLUSTER-IP     EXTERNAL-IP   PORT(S)              AGE
jenkins      ClusterIP   10.1.106.26    <none>        8080/TCP,50000/TCP   4h22m
kubernetes   ClusterIP   10.1.0.1       <none>        443/TCP              11d

kubernetes的svc名称是kubernetes,端口443

jenkins的svc名称是jenkins,端口8080

1、Name 处默认为 kubernetes,也可以修改为其他名称,如果这里修改了,下边在执行 Job 时指定 podTemplate() 参数 cloud 为其对应名称,否则会找不到,cloud 默认值取:kubernetes


2、Kubernetes URL 格式为

https://<svc_name>.<namespace_name>

https://<svc_name>.<namespace_name>.svc.cluster.local

https://<ClusterIP>:<Ports>

我这里填写为https://kubernetes.default,命名空间就是default

 

3、Jenkins URL 格式跟上面类似

我填写为:https://jenkins.default:8080

配置完毕,可以点击 “测试连接” 按钮测试是否能够连接的到 Kubernetes,如果显示 Connection test successful 则表示连接成功,配置没有问题。

 

4、创建一个 Pipeline 并随便命名为 my-k8s-jenkins-pipeline,然后在 Pipeline 脚本处填写一个简单的测试脚本如下

//运行的pod的名字
def label = "mypod-${UUID.randomUUID().toString()}"
//使用pod模板运行
podTemplate(label: label, cloud: 'kubernetes') {
    node(label) {
        stage('Run shell') {
            sh 'sleep 130s'
            sh 'echo hello world.'
        }
    }
}

注:使用默认的jenkins-jnlp-slave,尝试部署流水线,k8s会部署一个pod用于执行jenkins给的任务,执行完毕后,pod会自动销毁

四、使用自定义镜像执行流水线

如果想在jenkins的slave的pod运行时,做一系列工作,如使用maven对代码做构建工作。由于jenkins-jnlp-slave里面没有maven,可以用过dockerfile的方式把maven打进去镜像里面

1、创建自定义的jenkins-jnlp-slave镜像,并push到docker私库,我用的是docker register

# 指定基础镜像
FROM jenkins/jnlp-slave:latest

MAINTAINER my-jenkins

LABEL Description="This is a extend image base from jenkins/jnlp-slave which install maven in it."

# 切换到 root 账户进行操作
USER root

# 安装 maven-3.6.2
RUN wget http://mirrors.tuna.tsinghua.edu.cn/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
    tar -zxf apache-maven-3.6.3-bin.tar.gz && \
    mv apache-maven-3.6.3 /usr/local && \
    rm -f apache-maven-3.6.3-bin.tar.gz && \
    ln -s /usr/local/apache-maven-3.6.3/bin/mvn /usr/bin/mvn && \
    ln -s /usr/local/apache-maven-3.6.3 /usr/local/apache-maven

# 安装 node-v10
RUN  wget https://nodejs.org/download/release/v10.17.0/node-v10.17.0-linux-x64.tar.gz && \
     tar -xzf node-v10.17.0-linux-x64.tar.gz && \
     mv node-v10.17.0-linux-x64 /usr/local && \
     rm -r node-v10.17.0-linux-x64.tar.gz && \
     ln -s /usr/local/node-v10.17.0-linux-x64/bin/node /usr/bin/node && \
     ln -s /usr/local/node-v10.17.0-linux-x64/bin/npm /usr/bin/npm && \
     ln -s /usr/local/node-v10.17.0-linux-x64 /usr/local/node-v10.17.0

USER jenkins

build镜像

docker build -t 192.168.200.207:5000/my-jenkins-jnlp-slave:latest -f ./my-dockerfile .

创建一个 Pipeline 类型 Job 并命名为 my-k8s-jenkins-container-custom,然后在 Pipeline 脚本处填写一个简单的测试脚本如下:

def label = "mypod-${UUID.randomUUID().toString()}"
podTemplate(label: label, cloud: 'kubernetes',containers: [
    containerTemplate(
        name: 'jnlp', 
        image: '192.168.200.207:5000/my-jenkins-jnlp-slave:latest', 
        alwaysPullImage: false, 
        args: '${computer.jnlpmac} ${computer.name}'),
  ]) {

    node(label) {
        stage('stage1') {
            stage('Show Maven version') {
                sh 'mvn -version'
                sh 'sleep 60s'
            }
        }
    }
}

说明一下:这里 containerTemplate 的 name 属性必须叫 jnlp,Kubernetes 才能用自定义 images 指定的镜像替换默认的 jenkinsci/jnlp-slave 镜像。此外,args 参数传递两个 jenkins-slave 运行需要的参数。还有一点就是这里并不需要指定 container('jnlp'){...} 了,因为它被 Kubernetes 指定了要被执行的容器,所以直接执行 Stage 就可以了。
 

3、在jenkins的pipeline中,可以使用SSH Pipeline Steps插件做远程操作

jenkins-》系统管理-》插件管理-》可选插件,安装SSH Pipeline Steps

官网:https://jenkins.io/doc/pipeline/steps/ssh-steps/

 

评论
添加红包

请填写红包祝福语或标题

红包个数最小为10个

红包金额最低5元

当前余额3.43前往充值 >
需支付:10.00
成就一亿技术人!
领取后你会自动成为博主和红包主的粉丝 规则
hope_wisdom
发出的红包
实付
使用余额支付
点击重新获取
扫码支付
钱包余额 0

抵扣说明:

1.余额是钱包充值的虚拟货币,按照1:1的比例进行支付金额的抵扣。
2.余额无法直接购买下载,可以购买VIP、付费专栏及课程。

余额充值