Kubernetes配置Jenkins Slave
部署在kubernetes集群内
1、自定义Jenkins镜像
# 可直接使用registry.cn-hangzhou.aliyuncs.com/xumeng03/jenkinsci:2.462
# Build: docker build --no-cache --force-rm -t jenkinsci:2.462 ./
# Run: docker run -d -u root -v jenkinsci:/root/.jenkins --name jenkinsci -p 10000:8080 jenkinsci:2.462
# Into: docker exec -it jenkinsci /bin/sh
# Access: http://your_ip:10000/
FROM registry.cn-hangzhou.aliyuncs.com/xumeng03/alpine:3.17
# 在https://mirrors.aliyun.com/jenkins/war/下载自己所需版本jenkins,这里使用的是jenkins-2.462
COPY jenkins.war jenkins.war
ENV TZ Asia/Shanghai
RUN echo -e "http://mirrors.tuna.tsinghua.edu.cn/alpine/v3.17/main\nhttp://mirrors.tuna.tsinghua.edu.cn/alpine/v3.17/community" > /etc/apk/repositories && \
apk update && \
apk add --no-cache tzdata && \
apk add openjdk17 && \
apk add openjdk17-jre fontconfig ttf-dejavu && \
apk add git
ENTRYPOINT ["/bin/sh", "-c", "java -jar jenkins.war --httpPort=8080"]
2、部署jenkins
1、命名空间
apiVersion: v1
kind: Namespace
metadata:
name: jenkinsci
labels:
app: jenkinsci
2、Deployment
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkinsci
namespace: jenkinsci
labels:
app: jenkinsci
spec:
replicas: 1
selector:
matchLabels:
app: jenkinsci
template:
metadata:
labels:
app: jenkinsci
spec:
# 增加污点容忍,可以在具有NoSchedule污点节点上部署
tolerations:
- key: node-role.kubernetes.io/master
operator: Exists
effect: NoSchedule
- key: node-role.kubernetes.io/control-plane
operator: Exists
effect: NoSchedule
# 选择具有jenkinsci标签的节点部署
nodeSelector:
node-role: jenkinsci
containers:
- name: jenkinsci
image: registry.cn-hangzhou.aliyuncs.com/ialso/jenkinsci:2.462
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
- containerPort: 50000
name: jnlp
volumeMounts:
- name: jenkinsci-data
mountPath: /root/.jenkins
volumes:
# jenkins数据存放位置
- name: jenkinsci-data
hostPath:
path: /root/.jenkins
3、service
apiVersion: v1
kind: Service
metadata:
name: jenkinsci
namespace: jenkinsci
spec:
selector:
app: jenkinsci
ports:
- name: web
port: 80
targetPort: 8080
- name: jnlp
port: 50000
targetPort: 50000
type: ClusterIP
4、配置外部访问
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: ingress-jenkinsci
namespace: jenkinsci
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
# 转发规则
rules:
- host: jenkins.ialso.cn
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: jenkinsci
port:
number: 80
5、自定义jenkins
需要安装下列插件
- Git
- Pipeline
- Email Extension Template
- Build With Parameters
- Kubernetes
- Kubernetes CLI
3、密钥配置
Dashboard->Manage Jenkins->Credentials->System->Global credentials
4、配置Agent
Dashboard->Manage Jenkins->Configure Global Security->Agents
将模式修改为Fixed,端口为50000
5、配置Jenkins Location
6、配置kubernetes集群
Dashboard->Manage Jenkins->Manage Nodes and Clouds->Configure Clouds
选择上面步骤创建的kubernetes凭证,随后测试连接
7、镜像准备
接下来需要在jenkins slave里面进行项目打包、编译、镜像构建、集群中部署,因此需要两个中间镜像:docker、kubectl
1、idocker
# 可直接使用registry.cn-hangzhou.aliyuncs.com/xumeng03/idocker:20.10.24
# Build: docker build --no-cache --force-rm -t idocker:20.10.24 ./
# Run: docker run -d --name idocker -v /var/run/docker.sock:/var/run/docker.sock idocker:20.10.24
# Into: docker exec -it idocker /bin/sh
FROM registry.cn-hangzhou.aliyuncs.com/xumeng03/alpine:3.17
# 在https://mirrors.aliyun.com/alpine/v3.17/community/x86_64/下载自己所需的版本docker-cli
COPY docker-cli-20.10.24-r2.apk docker-cli-20.10.24-r2.apk
RUN echo -e "http://mirrors.tuna.tsinghua.edu.cn/alpine/v3.17/main\nhttp://mirrors.tuna.tsinghua.edu.cn/alpine/v3.17/community" > /etc/apk/repositories && \
apk update && \
apk --allow-untrusted add docker-cli-20.10.24-r2.apk
ENTRYPOINT ["/bin/sh", "-c", "sleep 3600"]
2、ikubectl
# 可直接使用registry.cn-hangzhou.aliyuncs.com/xumeng03/ikubectl:1.27.4
# Build: docker build --no-cache --force-rm -t ikubectl:1.27.4 ./
# Run: docker run -d --name ikubectl ikubectl:1.27.4
# Into: docker exec -it ikubectl /bin/sh
FROM registry.cn-hangzhou.aliyuncs.com/xumeng03/alpine:3.17
# 在https://github.com/kubernetes/kubernetes/tree/master/CHANGELOG下载自己集群对应版本的kubectl(kubernetes-client-linux-amd64.tar.gz)
COPY kubectl /usr/local/bin/kubectl
ENTRYPOINT ["/bin/sh", "-c", "sleep 3600"]
8、配置slave
1、项目配置
需在项目中将Jenkinsfile配置为Pipeline script from SCM
, 并配置相关信息
2、Jenkinsfile
此文件需放置在项目根目录
// Jenkinsfile
pipeline {
// 配置代理
agent {
// kubernetes代理信息
kubernetes {
// 这里要填写Jenkins configureClouds中配置的kubernetes信息
cloud 'kubernetes'
label "jenkinsci"
// 超时时间
slaveConnectTimeout 1200
// podTemplate
yamlFile 'PodTemplate.yaml'
}
}
stages {
stage('Pull') {
steps {
checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 'gitee', url: 'https://gitee.com/xumeng03/demo.git']])
}
}
stage('Build && Push') {
steps {
container(name: 'docker'){
// 测试kubectl命令
sh """
docker images
"""
}
}
}
stage('Deploy') {
environment {
KUBE_CONFIG = credentials('kubernetes-config')
}
steps {
container(name: 'kuberctl'){
// 测试kubectl命令
sh """
kubectl --kubeconfig ${KUBE_CONFIG} get nodes
"""
}
}
}
}
}
3、podTemplate
此文件需放置在yamlFile能对应到的位置,这里是项目根目录
apiVersion: v1
kind: Pod
metadata:
name: jenkinsci
namespace: devops
spec:
containers:
- name: jnlp
image: jenkins/jnlp-slave:4.9-1-jdk11
imagePullPolicy: IfNotPresent
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
image: registry.cn-hangzhou.aliyuncs.com/ialso/idocker:1.0
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
mountPath: /var/run/docker.sock
- name: kuberctl
image: registry.cn-hangzhou.aliyuncs.com/ialso/ikubectl:1.0
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: docker
hostPath:
path: /var/run/docker.sock
8、示例
0、镜像拉取密钥配置
注意secret具有命名空间隔离性!!!
kubectl create secret -n name-space \
docker-registry <name> \
--docker-server=DOCKER_REGISTRY_SERVER \
--docker-username=DOCKER_USER \
--docker-password=DOCKER_PASSWORD
1、Vue
项目地址:https://gitee.com/xumeng03/vue-demo
Dockerfile
FROM node:16.20.0-alpine as builder
COPY ./ /dest
WORKDIR /dest
RUN npm config set registry https://registry.npm.taobao.org/ && \
npm install && \
npm run build
FROM nginx
COPY --from=builder /dest/dist /usr/share/nginx/html
EXPOSE 80
PodTemplate.yaml
apiVersion: v1
kind: Pod
metadata:
name: jenkinsci
namespace: devops
spec:
containers:
- name: jnlp
# 这里的jnlp中jdk版本一定要与jenkins master中的一致
image: jenkins/jnlp-slave:4.9-1-jdk11
imagePullPolicy: IfNotPresent
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
image: registry.cn-hangzhou.aliyuncs.com/ialso/idocker:1.0
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
mountPath: /var/run/docker.sock
- name: kuberctl
image: registry.cn-hangzhou.aliyuncs.com/ialso/ikubectl:1.0
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: docker
hostPath:
path: /var/run/docker.sock
deployment.tql
# DEPLOY_NAMESPACE:部署命名空间
# DEPLOY_NAME:部署名称
# APPLICATION_REPLICAS:应用部署份数
# APPLICATION_NAME:应用名称
# IMAGE_SECRET:镜像下载密钥
# HUB_ADDRESS:镜像仓库地址
# HUB_REGISTRY:镜像仓库
# IMAGE_NAME:镜像名称
# IMAGE_TAG:镜像标签
# APPLICATION_PORT:应用端口
apiVersion: apps/v1
kind: Deployment
metadata:
name: {DEPLOY_NAME}
namespace: {DEPLOY_NAMESPACE}
labels:
app: {DEPLOY_NAME}
spec:
replicas: {APPLICATION_REPLICAS}
selector:
matchLabels:
app: {APPLICATION_NAME}
template:
metadata:
name: {APPLICATION_NAME}
labels:
app: {APPLICATION_NAME}
spec:
imagePullSecrets:
- name: {IMAGE_SECRET}
containers:
- name: {APPLICATION_NAME}
image: {HUB_ADDRESS}/{HUB_REGISTRY}/{IMAGE_NAME}:{IMAGE_TAG}-{IMAGE_TAG_EXTEND}
imagePullPolicy: IfNotPresent
ports:
- containerPort: {APPLICATION_PORT}
volumeMounts:
- name: timezone
mountPath: /etc/localtime
restartPolicy: Always
volumes:
# 时间处理
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
---
apiVersion: v1
kind: Service
metadata:
name: {APPLICATION_NAME}-service
namespace: {DEPLOY_NAMESPACE}
spec:
selector:
app: {APPLICATION_NAME}
ports:
- protocol: TCP
port: 80
targetPort: {APPLICATION_PORT}
type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {APPLICATION_NAME}
namespace: {DEPLOY_NAMESPACE}
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules:
- host: ialso.cn
http:
paths:
- pathType: Prefix
path: /
backend:
service:
name: {APPLICATION_NAME}-service
port:
number: 80
email.html
待补
Jenkinsfile
// Jenkinsfile
pipeline {
// 配置代理
agent {
// kubernetes代理信息
kubernetes {
// 这里要填写Jenkins configureClouds中配置的kubernetes信息
cloud 'kubernetes'
// Jenkins slave pod前缀
label "jenkinsci"
// 超时时间,单位秒
slaveConnectTimeout 600
// podTemplate
yamlFile 'PodTemplate.yaml'
}
}
environment {
// 项目地址
PROJECT_URL = "https://gitee.com/xumeng03/vue-demo.git"
// 镜像名称
IMAGE_NAME = "vue-demo"
// 获取构建时间作为镜像tag
IMAGE_TAG = sh(script: "date --date='0 days ago' +%Y%m%d%H%M%S", returnStdout: true).trim()
// 获取commit id(short形式)作为镜像tag extend
IMAGE_TAG_EXTEND = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
// 镜像仓库地址
HUB_ADDRESS = "registry.cn-hangzhou.aliyuncs.com"
// 镜像仓库
HUB_REGISTRY = "ialso"
// 账户信息
HUB_USER = credentials('aliyun-hub')
// 部署命名空间
DEPLOY_NAMESPACE = "ialso"
// 部署名称
DEPLOY_NAME = "vue-demo"
// 应用部署份数
APPLICATION_REPLICAS = 1
// 应用名称
APPLICATION_NAME = "vue-demo"
// 应用镜像下载密钥
IMAGE_SECRET = "aliyun-hub"
// 应用端口
APPLICATION_PORT = 80
}
stages {
stage('Pull') {
steps {
checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 'gitee', url: "${PROJECT_URL}"]])
}
}
stage('Build && Push') {
steps {
container(name: 'docker'){
sh """
docker build --no-cache -t ${HUB_ADDRESS}/${HUB_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}-${IMAGE_TAG_EXTEND} .
docker login -u ${HUB_USER_USR} -p ${HUB_USER_PSW} ${HUB_ADDRESS}
docker push ${HUB_ADDRESS}/${HUB_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}-${IMAGE_TAG_EXTEND}
"""
}
}
}
stage('Deploy') {
environment {
KUBE_CONFIG = credentials('kubernetes-config')
}
steps {
container(name: 'kuberctl'){
sh """
sed -e 's#{DEPLOY_NAMESPACE}#${DEPLOY_NAMESPACE}#g;s#{DEPLOY_NAME}#${DEPLOY_NAME}#g;s#{APPLICATION_REPLICAS}#${APPLICATION_REPLICAS}#g;s#{APPLICATION_NAME}#${APPLICATION_NAME}#g;s#{IMAGE_SECRET}#${IMAGE_SECRET}#g;s#{HUB_ADDRESS}#${HUB_ADDRESS}#g;s#{HUB_REGISTRY}#${HUB_REGISTRY}#g;s#{IMAGE_NAME}#${IMAGE_NAME}#g;s#{IMAGE_TAG}#${IMAGE_TAG}#g;s#{IMAGE_TAG_EXTEND}#${IMAGE_TAG_EXTEND}#g;s#{APPLICATION_PORT}#${APPLICATION_PORT}#g;' deployment.tql > deployment.yaml
cat deployment.yaml
kubectl --kubeconfig ${KUBE_CONFIG} apply -f deployment.yaml
"""
}
}
}
}
}
2、Java
项目地址:https://gitee.com/xumeng03/springboot-demo
Dcoekrfile
FROM maven:3.6.0-alpine as builder
COPY ./ /dest
WORKDIR /dest
COPY settings.xml /usr/share/maven/conf/settings.xml
RUN mvn package -Dmaven.test.skip=true
FROM openjdk:8-jdk-alpine
COPY --from=builder /dest/target/*.jar /app/app.jar
WORKDIR /app
EXPOSE 8080
ENTRYPOINT ["sh", "-c", "java -jar app.jar"]
PodTemplate.yaml
apiVersion: v1
kind: Pod
metadata:
name: jenkinsci
namespace: devops
spec:
containers:
- name: jnlp
image: jenkins/jnlp-slave:4.9-1-jdk11
imagePullPolicy: IfNotPresent
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
image: registry.cn-hangzhou.aliyuncs.com/ialso/idocker:latest
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
- name: docker
mountPath: /var/run/docker.sock
- name: kuberctl
image: registry.cn-hangzhou.aliyuncs.com/ialso/ikubectl:latest
imagePullPolicy: IfNotPresent
command:
- "cat"
tty: true
volumeMounts:
- name: timezone
mountPath: /etc/localtime
volumes:
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
- name: docker
hostPath:
path: /var/run/docker.sock
deployment.tql
# DEPLOY_NAMESPACE:部署命名空间
# DEPLOY_NAME:部署名称
# APPLICATION_REPLICAS:应用部署份数
# APPLICATION_NAME:应用名称
# IMAGE_SECRET:镜像下载密钥
# HUB_ADDRESS:镜像仓库地址
# HUB_REGISTRY:镜像仓库
# IMAGE_NAME:镜像名称
# IMAGE_TAG:镜像标签
# APPLICATION_PORT:应用端口
apiVersion: apps/v1
kind: Deployment
metadata:
name: {DEPLOY_NAME}
namespace: {DEPLOY_NAMESPACE}
labels:
app: {DEPLOY_NAME}
spec:
replicas: {APPLICATION_REPLICAS}
selector:
matchLabels:
app: {APPLICATION_NAME}
template:
metadata:
name: {APPLICATION_NAME}
labels:
app: {APPLICATION_NAME}
spec:
imagePullSecrets:
- name: {IMAGE_SECRET}
containers:
- name: {APPLICATION_NAME}
image: {HUB_ADDRESS}/{HUB_REGISTRY}/{IMAGE_NAME}:{IMAGE_TAG}-{IMAGE_TAG_EXTEND}
imagePullPolicy: IfNotPresent
ports:
- containerPort: {APPLICATION_PORT}
volumeMounts:
- name: timezone
mountPath: /etc/localtime
restartPolicy: Always
volumes:
# 时间处理
- name: timezone
hostPath:
path: /usr/share/zoneinfo/Asia/Shanghai
---
apiVersion: v1
kind: Service
metadata:
name: {APPLICATION_NAME}-service
namespace: {DEPLOY_NAMESPACE}
spec:
selector:
app: {APPLICATION_NAME}
ports:
- protocol: TCP
port: 80
targetPort: {APPLICATION_PORT}
type: ClusterIP
---
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: {APPLICATION_NAME}
namespace: {DEPLOY_NAMESPACE}
annotations:
kubernetes.io/ingress.class: "nginx"
# 路径重写,$2值为下面path中的(.*)
nginx.ingress.kubernetes.io/rewrite-target: /$2
spec:
rules:
- host: ialso.cn
http:
paths:
- pathType: Prefix
# 匹配/api下所有,$表示以当前字符串结尾
path: /api(/|$)(.*)
backend:
service:
name: {APPLICATION_NAME}-service
port:
number: 80
email.html
待补
Jenkinsfile
// Jenkinsfile
pipeline {
// 配置代理
agent {
// kubernetes代理信息
kubernetes {
// 这里要填写Jenkins configureClouds中配置的kubernetes信息
cloud 'kubernetes'
label "jenkinsci"
// 超时时间,单位秒
slaveConnectTimeout 600
// podTemplate
yamlFile 'PodTemplate.yaml'
}
}
environment {
// 项目地址
PROJECT_URL = "https://gitee.com/xumeng03/springboot-demo.git"
// 镜像名称
IMAGE_NAME = "springboot-demo"
// 获取构建时间作为镜像tag
IMAGE_TAG = sh(script: "date --date='0 days ago' +%Y%m%d%H%M%S", returnStdout: true).trim()
// 获取commit id(short形式)作为镜像tag extend
IMAGE_TAG_EXTEND = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
// 镜像仓库地址
HUB_ADDRESS = "registry.cn-hangzhou.aliyuncs.com"
// 镜像仓库
HUB_REGISTRY = "ialso"
// 账户信息
HUB_USER = credentials('aliyun-hub')
// 部署命名空间
DEPLOY_NAMESPACE = "ialso"
// 部署名称
DEPLOY_NAME = "springboot-demo"
// 应用部署份数
APPLICATION_REPLICAS = 1
// 应用名称
APPLICATION_NAME = "springboot-demo"
// 应用镜像下载密钥
IMAGE_SECRET = "aliyun-hub"
// 应用端口
APPLICATION_PORT = 8080
}
stages {
stage('Pull') {
steps {
checkout scmGit(branches: [[name: '*/master']], extensions: [], userRemoteConfigs: [[credentialsId: 'gitee', url: "${PROJECT_URL}"]])
}
}
stage('Build && Push') {
steps {
container(name: 'docker'){
sh """
docker build --no-cache -t ${HUB_ADDRESS}/${HUB_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}-${IMAGE_TAG_EXTEND} .
docker login -u ${HUB_USER_USR} -p ${HUB_USER_PSW} ${HUB_ADDRESS}
docker push ${HUB_ADDRESS}/${HUB_REGISTRY}/${IMAGE_NAME}:${IMAGE_TAG}-${IMAGE_TAG_EXTEND}
"""
}
}
}
stage('Deploy') {
environment {
KUBE_CONFIG = credentials('kubernetes-config')
}
steps {
container(name: 'kuberctl'){
sh """
sed -e 's#{DEPLOY_NAMESPACE}#${DEPLOY_NAMESPACE}#g;s#{DEPLOY_NAME}#${DEPLOY_NAME}#g;s#{APPLICATION_REPLICAS}#${APPLICATION_REPLICAS}#g;s#{APPLICATION_NAME}#${APPLICATION_NAME}#g;s#{IMAGE_SECRET}#${IMAGE_SECRET}#g;s#{HUB_ADDRESS}#${HUB_ADDRESS}#g;s#{HUB_REGISTRY}#${HUB_REGISTRY}#g;s#{IMAGE_NAME}#${IMAGE_NAME}#g;s#{IMAGE_TAG}#${IMAGE_TAG}#g;s#{IMAGE_TAG_EXTEND}#${IMAGE_TAG_EXTEND}#g;s#{APPLICATION_PORT}#${APPLICATION_PORT}#g;' deployment.tql > deployment.yaml
cat deployment.yaml
kubectl --kubeconfig ${KUBE_CONFIG} apply -f deployment.yaml
"""
}
}
}
}
}