一起来学k8s 32.二进制k8s集群部署jenkins

二进制k8s集群部署jenkins

环境准备

##/etc/hosts
192.168.48.101 master01
192.168.48.102 master02
192.168.48.103 master03
192.168.48.201 node01
192.168.48.202 node02
192.168.48.54  nfs

## keepalived的vip
192.168.48.66

IP	Hostname	CPU	Memory
192.168.48.101	master01	2	4G
192.168.48.102	master02	2	4G
192.168.48.103	master03	2	4G
192.168.48.201	node01	2	4G
192.168.48.202	node02	2	4G
192.168.48.54	nfs	2	4G

软件	版本
kubernetes	1.15.2
docker-ce	19.03
calico	3.8
etcd	3.3.13
CNI	0.8.1
coredns	1.4.0
metrics-server	0.3.3
ingress-controller	0.25.0
dashboard	1.10.1
Weave Scope	1.11.4
nfs	v4
helm	2.14.3
harbor	1.1.1

安装

编写jenkins.yaml

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: jenkins-pvc
  namespace: kube-ops
  annotations:
    volume.beta.kubernetes.io/storage-class: "k8s-nfs-storage"
spec:
  accessModes:
  - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: jenkins
  namespace: kube-ops
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: jenkins
rules:
  - apiGroups: ["extensions", "apps"]
    resources: ["deployments"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["services"]
    verbs: ["create", "delete", "get", "list", "watch", "patch", "update"]
  - apiGroups: [""]
    resources: ["pods"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/exec"]
    verbs: ["create","delete","get","list","patch","update","watch"]
  - apiGroups: [""]
    resources: ["pods/log"]
    verbs: ["get","list","watch"]
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get"]
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: jenkins
  namespace: kube-ops
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: jenkins
subjects:
  - kind: ServiceAccount
    name: jenkins
    namespace: kube-ops

---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: jenkins
  namespace: kube-ops
spec:
  template:
    metadata:
      labels:
        app: jenkins
    spec:
      terminationGracePeriodSeconds: 10
      serviceAccount: jenkins
      containers:
      - name: jenkins
        image: jenkins/jenkins:lts
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 8080
          name: web
          protocol: TCP
        - containerPort: 50000
          name: agent
          protocol: TCP
        resources:
          limits:
            cpu: 1000m
            memory: 1Gi
          requests:
            cpu: 500m
            memory: 512Mi
        livenessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        readinessProbe:
          httpGet:
            path: /login
            port: 8080
          initialDelaySeconds: 60
          timeoutSeconds: 5
          failureThreshold: 12
        volumeMounts:
        - name: jenkinshome
          subPath: jenkins
          mountPath: /var/jenkins_home
        env:
        - name: LIMITS_MEMORY
          valueFrom:
            resourceFieldRef:
              resource: limits.memory
              divisor: 1Mi
        - name: JAVA_OPTS
          value: -Xmx$(LIMITS_MEMORY)m -XshowSettings:vm -Dhudson.slaves.NodeProvisioner.initialDelay=0 -Dhudson.slaves.NodeProvisioner.MARGIN=50 -Dhudson.slaves.NodeProvisioner.MARGIN0=0.85 -Duser.timezone=Asia/Shanghai
      securityContext:
        fsGroup: 1000
      volumes:
      - name: jenkinshome
        persistentVolumeClaim:
          claimName: jenkins-pvc

---
apiVersion: v1
kind: Service
metadata:
  name: jenkins
  namespace: kube-ops
  labels:
    app: jenkins
spec:
  selector:
    app: jenkins
  type: NodePort
  ports:
  - name: web
    port: 8080
    targetPort: web
    nodePort: 32002
  - name: agent
    port: 50000
    targetPort: agent
---
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: jenkins
  namespace: kube-ops
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/proxy-body-size: "50m"
spec:
  rules:
  - host: jenkins.tk8s.com
    http:
      paths:
      - backend:
          serviceName: jenkins
          servicePort: web

镜像下载

jenkins/jenkins:lts 
cnych/jenkins:jnlp6

链接: https://pan.baidu.com/s/1E5G0ItE6meN0N_d0qyPxPg 提取码: ti2e

docker load -i  jenkins2.176.tar.gz

创建jenkins

kubectl apply -f jenkins.yaml

[root@master01 ~]# kubectl get pod -n kube-ops -o wide
NAME                          READY   STATUS              RESTARTS   AGE     IP               NODE     NOMINATED NODE   READINESS GATES
jenkins-7578f9d747-j5jv2      1/1     Running             2          3h15m   10.244.196.134   node01   <none>           <none>

访问

[root@master01 ~]# kubectl exec -n kube-ops jenkins-7578f9d747-j5jv2 -it -- /bin/sh
$ cat /var/jenkins_home/secrets/initialAdminPassword
12b9e9b483374716a72c6c73d875ed33

插件安装

选择第一个推荐安装

设置管理员账号

配置

接下来我们就需要来配置 Jenkins，让他能够动态的生成 Slave 的 Pod。

第1步. 我们需要安装**kubernetes， 点击 Manage Jenkins -> Manage Plugins -> Available -> Kubernetes 勾选安装即可。

第2步. 安装完毕后，点击 Manage Jenkins —> Configure System —> (拖到最下方)Add a new cloud —> 选择 Kubernetes，然后填写 Kubernetes 和 Jenkins 配置信息

第3步. 配置 Pod Template，其实就是配置 Jenkins Slave 运行的 Pod 模板，命名空间我们同样是用 kube-ops，Labels 这里也非常重要，对于后面执行 Job 的时候需要用到该值，然后我们这里使用的是 cnych/jenkins:jnlp6 这个镜像，这个镜像是在官方的 jnlp 镜像基础上定制的，加入了 kubectl 等一些实用的工具。

测试

在 Jenkins 首页点击create new jobs，创建一个测试的任务，输入任务名称，然后我们选择 Freestyle project 类型的任务：

注意在下面的 Label Expression 这里要填入haimaxy-jnlp，就是前面我们配置的 Slave Pod 中的 Label，这两个地方必须保持一致

然后往下拉，在 Build 区域选择Execute shell

echo "测试 Kubernetes 动态生成 jenkins slave"
echo "==============docker in docker==========="
docker info

echo "=============kubectl============="
kubectl get pods

现在我们直接在页面点击做成的 Build now 触发构建即可

查看到对应的控制台信息：