1 环境概述
192.168.38.50 master
192.168.38.51 node1
192.168.38.52 node2
192.168.38.100 harbor/nfs
192.168.38.45 gitlab/sonarqube
2 Kubernetes安装Jenkins
2.1 PV/PVC
#创建命名空间
kubectl create namespace jenkins-k8s
(1) 创建PV
vim jenkins-pv.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: jenkins-pv
spec:
capacity:
storage: 10Gi
accessModes:
- ReadWriteMany
nfs:
path: /usr/local/kubernetes/volumes/jenkins
server: 192.168.38.100
#应用
kubectl apply -f jenkins-pv.yaml
(2) 创建PVC
vim jenkins-pvc.yaml
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: jenkins-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 10Gi
accessModes:
- ReadWriteMany
#应用
kubectl apply -f jenkins-pvc.yaml
2.2 ServiceAccount
#创建一个sa账号
kubectl create sa jenkins-k8s-sa -n jenkins-k8s
#授权,kubectl create clusterrolebinding 名称、名称空间、绑定clusterrole=cluster-admin
kubectl create clusterrolebinding jenkins-k8s-sa-cluster -n jenkins-k8s --clusterrole=cluster-admin --serviceaccount=jenkins-k8s:jenkins-k8s-sa
2.3 整合Jenkins
2.3.1 下载镜像
#node1、node2
docker pull jenkins/jenkins:latest
2.3.2 jenkins-deployment.yaml
vim jenkins-deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
replicas: 1
selector:
matchLabels:
app: jenkins
template:
metadata:
labels:
app: jenkins
spec:
serviceAccountName: jenkins-k8s-sa
containers:
- name: jenkins
image: jenkins/jenkins:latest
imagePullPolicy: IfNotPresent
ports:
- containerPort: 8080
name: web
protocol: TCP
- containerPort: 50000
name: agent
protocol: TCP
resources:
limits:
cpu: 1000m
memory: 1Gi
requests:
cpu: 500m
memory: 500Mi
livenessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
timeoutSeconds: 5
failureThreshold: 12
readinessProbe:
httpGet:
port: 8080
path: /login
initialDelaySeconds: 60
volumeMounts:
- mountPath: /var/jenkins_home
name: jenkins-volume
subPath: jenkins-home
volumes:
- name: jenkins-volume
persistentVolumeClaim:
claimName: jenkins-pvc
#应用
kubectl apply -f jenkins-deployment.yaml
#查看
kubectl get pod -n jenkins-k8s
#验证
kubectl get pod -o wide -n jenkins-k8s
curl 10.244.2.23:8080
2.3.3 jenkins-svc.yaml
#创建
vim jenkins-svc.yaml
apiVersion: v1
kind: Service
metadata:
name: jenkins-service
namespace: jenkins-k8s
labels:
app: jenkins
spec:
selector:
app: jenkins
type: ClusterIP
ports:
- port: 8080
name: web
targetPort: web
- name: agent
port: 50000
targetPort: agent
#应用
kubectl apply -f jenkins-svc.yaml
#查看
kubectl get svc -n jenkins-k8s
2.3.4 jenkins-ingress.yaml
vim jenkins-ingress.yaml
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: jenkins
namespace: jenkins-k8s
spec:
rules:
- host: rosh.jenkins.com
http:
paths:
- backend:
serviceName: jenkins-service
servicePort: 8080
#应用
kubectl apply -f jenkins-ingress.yaml
#查看
kubectl get ingress -n jenkins-k8s
2.4 Jenkins 初始化
(1) 查看初始化密码
#查看挂载日志,记录初始化密码,填入初始化页面
84f98993d6594792b7e420c74934eb89
(2)初始化插件
(3)初始化用户
2.5 中文插件
2.6 kubernetes插件
(1) 安装插件
(2) 配置Kubernetes
2.7 Jenkins从节点配置
2.7.1 制作镜像
(1) setting.xml
<?xml version="1.0" encoding="UTF-8"?>
<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 http://maven.apache.org/xsd/settings-1.0.0.xsd">
<localRepository>/usr/local/apache-maven/repo</localRepository>
<pluginGroups></pluginGroups>
<proxies></proxies>
<servers></servers>
<profiles></profiles>
<mirrors>
<mirror>
<id>central</id>
<mirrorOf>central</mirrorOf>
<name>aliyun maven</name>
<url>https://maven.aliyun.com/repository/public</url>
</mirror>
</mirrors>
</settings>
(2) Dockerfile
FROM cnych/jenkins:jnlp6
MAINTAINER rosh
# 切换到 root 账户进行操作
USER root
# 安装 maven
COPY apache-maven-3.6.2-bin.tar.gz .
RUN tar -zxf apache-maven-3.6.2-bin.tar.gz && \
mv apache-maven-3.6.2 /usr/local && \
rm -f apache-maven-3.6.2-bin.tar.gz && \
ln -s /usr/local/apache-maven-3.6.2/bin/mvn /usr/bin/mvn && \
ln -s /usr/local/apache-maven-3.6.2 /usr/local/apache-maven && \
mkdir -p /usr/local/apache-maven/repo
COPY settings.xml /usr/local/apache-maven/conf/settings.xml
(3) 制作镜像
docker build -t jenkins-slave-maven .
(4) 上传至harbor
docker tag jenkins-slave-maven:latest 192.168.38.100/library/jenkins-slave-maven:latest
docker push 192.168.38.100/library/jenkins-slave-maven:latest
(5) 查看
2.7.2 配置模板
master的/root/.kube目录拷贝到node1和node2节点:
scp -r /root/.kube node1:
scp -r /root/.kube node2:
2.7.3 测试
(1) 创建流水线
(2) 测试脚本
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
stage('k8s-测试'){
sh "kubectl get node"
}
stage('maven-测试'){
sh "mvn -v"
}
stage('docker 测试'){
sh "docker version"
}
}
(3) 结果
3 整合Gitlab
3.1 凭证
jenkins添加访问凭证
3.2 创建pipeline工程
3.3 整合Hook
3.4 Gitlab
3.5 Jenkinsfile
(1) 流水线配置
(2) 生成脚本
(3) Jenkinsfile
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代码'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代码成功"
}
}
(4) 推送代码到develop分支
(5) 查看jenkins 日志
3.6 Generic Webhook Trigger
查看Gitlab推送请求,防止频繁构建,现在限制develop分支推送代码时,触发自动构建,别的分支不触发构建。
3.6.1 修改Pipeline配置
3.6.2 修改Gitlab钩子
3.6.3 测试
master分支merge develop分支代码,推送master分支,没有触发jenkins自动化构建,查看gitlab响应信息
4 制作镜像、上传Harbor
4.1 Dockerfile
#######################网关#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 9000
ENTRYPOINT ["java","-jar","/app.jar"]
#######################student#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8001
ENTRYPOINT ["java","-jar","/app.jar"]
#######################teacher#######################
FROM java:8
RUN /bin/cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
RUN echo "Asia/Shanghai" > /etc/timezone
RUN bash -c 'touch /app.jar'
ARG JAR_FILE
COPY ${JAR_FILE} app.jar
EXPOSE 8002
ENTRYPOINT ["java","-jar","/app.jar"]
4.2 Maven插件
<!-- docker的maven插件-->
<plugin>
<groupId>com.spotify</groupId>
<artifactId>dockerfile-maven-plugin</artifactId>
<version>1.3.6</version>
<configuration>
<repository>${project.artifactId}</repository>
<buildArgs>
<JAR_FILE>target/${project.build.finalName}.jar</JAR_FILE>
</buildArgs>
</configuration>
</plugin>
4.3 集成Harbor
(1) 凭证
(2) 生成脚本
4.4 nfs创建maven目录
#nfs服务器
cd /usr/local/kubernetes/volumes
mkdir maven
chmod 777 ./maven/
chown nfsnobody ./maven/
vim /etc/exports
/usr/local/kubernetes/volumes/maven/ *(rw,sync,insecure,no_subtree_check,no_root_squash)
#重启
systemctl restart rpcbind && systemctl restart nfs
#其它节点测试
showmount -e 192.168.38.100
4.5 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor仓库名称
def harborRepository = "springcloud-k8s"
//构建版本tag
def tag = "latest"
//harbor 凭证
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服务列表
def microServerList = ['gateway-server','student-server','teacher-server']
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代码'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代码成功"
}
stage('制作镜像,上传harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作镜像: ${microServerList[i]}"
//服务名称
def serverName = "${microServerList[i]}"
//定义镜像名称
def imageName="${serverName}:${tag}"
//打包,构建镜像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上传镜像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登录
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上传
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//删除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
4.6 推送dev分支
4.7 jenkins查看结果
5 SonarQube
5.1 Sonarqube token
#记录token
c06945196968cf0a3e5e326e1e4b2646fc97b135
mvn sonar:sonar \
-Dsonar.host.url=http://192.168.38.45:9000/sonar \
-Dsonar.login=c06945196968cf0a3e5e326e1e4b2646fc97b135
5.2 Jenkins整合Sonarqube
(1) 安装插件
(2) 创建凭证
(3) 系统配置
(4) 全局配置
5.3 sonar-project.properties
#################################网关##########################################
#SonarQube 实例名称,唯一
sonar.projectKey=gateway-server
#项目名称
sonar.projectName=gateway-server
#版本号
sonar.projectVersion=1.0
#指定扫描目录
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#扫描编写类的项目
sonar.java.binaries=target/classes
#编码格式
sonar.sourceEncoding=UTF-8
#################################学生##########################################
#SonarQube 实例名称,唯一
sonar.projectKey=student-server
#项目名称
sonar.projectName=student-server
#版本号
sonar.projectVersion=1.0
#指定扫描目录
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#扫描编写类的项目
sonar.java.binaries=target/classes
#编码格式
sonar.sourceEncoding=UTF-8
#################################老师##########################################
#SonarQube 实例名称,唯一
sonar.projectKey=teacher-server
#项目名称
sonar.projectName=teacher-server
#版本号
sonar.projectVersion=1.0
#指定扫描目录
sonar.sources=.
sonar.exclusions=**/test/**
#jdk版本
sonar.java.source=1.8
sonar.java.target=1.8
#扫描编写类的项目
sonar.java.binaries=target/classes
#编码格式
sonar.sourceEncoding=UTF-8
5.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor仓库名称
def harborRepository = "springcloud-k8s"
//构建版本tag
def tag = "latest"
//harbor 凭证
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服务列表
def microServerList = ['gateway-server','student-server','teacher-server']
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代码'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代码成功"
}
stage('制作镜像,上传harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作镜像: ${microServerList[i]}"
//服务名称
def serverName = "${microServerList[i]}"
//定义镜像名称
def imageName="${serverName}:${tag}"
//打包,构建镜像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上传镜像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登录
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上传
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
//删除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
stage('代码审查'){
//定义当前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//引用当前JenkinsSonarQube环境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
}
5.5 推代码
5.6 查看结果
6 部署
6.1创建日志PV/PVC
(1) gateway
vim gateway-pv-pvc.yaml
kubectl apply -f gateway-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: gateway-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/gateway"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: gateway-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(2) teacher
vim teacher-pv-pvc.yaml
kubectl apply -f teacher-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: teacher-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/teacher"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: teacher-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
(3) student
vim student-pv-pvc.yaml
kubectl apply -f student-pv-pvc.yaml
apiVersion: v1
kind: PersistentVolume
metadata:
name: student-logs-pv
spec:
capacity:
storage: 2Gi
accessModes:
- ReadWriteMany
nfs:
path: "/usr/local/kubernetes/volumes/logs/student"
server: 192.168.38.100
---
kind: PersistentVolumeClaim
apiVersion: v1
metadata:
name: student-logs-pvc
namespace: jenkins-k8s
spec:
resources:
requests:
storage: 2Gi
accessModes:
- ReadWriteMany
6.2 创建权限
kubectl create secret docker-registry registry-secret --namespace=jenkins-k8s \
--docker-server=192.168.38.100 --docker-username=admin \
--docker-password=Harbor12345
kubectl get secret
6.3 Deployment
(1) gateway
##############Deployment######################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: gateway-server
name: gateway-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: gateway-server
template:
metadata:
labels:
app: gateway-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: gateway-server
image: 192.168.38.100/springcloud-k8s/gateway-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /gateway-server/logs
name: gateway-server
volumes:
- name: gateway-server
persistentVolumeClaim:
claimName: gateway-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: gateway-server
name: gateway-server
spec:
ports:
- port: 9000
protocol: TCP
targetPort: 9000
selector:
app: gateway-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: gateway-server
spec:
rules:
- host: rosh.gateway-server.com
http:
paths:
- backend:
serviceName: gateway-server
servicePort: 9000
(2) student
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: student-server
name: student-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: student-server
template:
metadata:
labels:
app: student-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: student-server
image: 192.168.38.100/springcloud-k8s/student-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /student-server/logs
name: student-server
volumes:
- name: student-server
persistentVolumeClaim:
claimName: student-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: student-server
name: student-server
spec:
ports:
- port: 8001
protocol: TCP
targetPort: 8001
selector:
app: student-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: student-server
spec:
rules:
- host: rosh.student-server.com
http:
paths:
- backend:
serviceName: student-server
servicePort: 8001
(3) teacher
##############Deployment####################
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: teacher-server
name: teacher-server
namespace: jenkins-k8s
spec:
replicas: 2
selector:
matchLabels:
app: teacher-server
template:
metadata:
labels:
app: teacher-server
spec:
imagePullSecrets:
- name: registry-secret
containers:
- name: teacher-server
image: 192.168.38.100/springcloud-k8s/teacher-server:latest
imagePullPolicy: Always
volumeMounts:
- mountPath: /teacher-server/logs
name: teacher-server
volumes:
- name: teacher-server
persistentVolumeClaim:
claimName: teacher-logs-pvc
---
##############Service####################
apiVersion: v1
kind: Service
metadata:
labels:
app: teacher-server
name: teacher-server
spec:
ports:
- port: 8002
protocol: TCP
targetPort: 8002
selector:
app: teacher-server
type: ClusterIP
---
##############Ingress####################
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
name: teacher-server
spec:
rules:
- host: rosh.teacher-server.com
http:
paths:
- backend:
serviceName: teacher-server
servicePort: 8002
6.4 Jenkinsfile
//harbor地址
def harborUrl = "192.168.38.100"
//harbor仓库名称
def harborRepository = "springcloud-k8s"
//构建版本tag
def tag = "latest"
//harbor 凭证
def harborAuth = "3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4"
//微服务列表
def microServerList = ['gateway-server','student-server','teacher-server']
//引用jenkins-slave的pod模块来构建Jenkins-Slave的pod
node("jenkins-slave"){
stage('拉取代码'){
git branch: 'develop', credentialsId: 'ad4d8da3-c362-4b92-b8da-e218f372d193', url: 'http://192.168.38.45/root/springboot-cloud-k8s.git'
echo "拉取代码成功"
}
stage('制作镜像,上传harbor'){
for(int i=0;i<microServerList.size();i++){
echo "制作镜像: ${microServerList[i]}"
//服务名称
def serverName = "${microServerList[i]}"
//定义镜像名称
def imageName="${serverName}:${tag}"
//打包,构建镜像
sh "mvn -f ${serverName} clean package dockerfile:build"
// 打tag
sh "docker tag ${imageName} ${harborUrl}/${harborRepository}/${imageName}"
// 上传镜像
withCredentials([usernamePassword(credentialsId: '3db8138a-792d-4d9b-8b4e-c7c62d9f6cf4', passwordVariable: 'password', usernameVariable: 'username')]) {
// some block
//登录
sh "docker login -u ${username} -p ${password} ${harborUrl}"
//上传
sh "docker push ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
stage('代码审查'){
//定义当前Jenkins的SonarQubeScanner工具
def scannerHome = tool 'sonarqube-scanner'
for(int i=0;i<microServerList.size();i++){
//引用当前JenkinsSonarQube环境
withSonarQubeEnv('sonarqube') {
sh """cd ${microServerList[i]}
${scannerHome}/bin/sonar-scanner"""
}
}
}
stage('k8s 部署'){
for(int i=0;i<microServerList.size();i++){
sh "kubectl apply -f ${microServerList[i]}/Deployment.yaml"
}
}
stage('删除本地镜像 '){
for(int i=0;i<microServerList.size();i++){
//服务名称
def serverName = "${microServerList[i]}"
//定义镜像名称
def imageName="${serverName}:${tag}"
//删除
sh "docker rmi -f ${imageName}"
sh "docker rmi -f ${harborUrl}/${harborRepository}/${imageName}"
}
}
}
6.5 结果