hume项目k8s的改造

hume项目k8s的改造

一、修改构建目录结构

1、在根目录下添加build-work文件夹

目录结构如下

[root@k8s-worker-01 build-work]# tree .
.
├── Dockerfile
├── hume
│   └── start.sh
└── Jenkinsfile

2、每个文件内容如下

Dockerfile

FROM ccr.ccs.tencentyun.com/xxxx/php_supervisor:kafka
USER root
ENV ZBE_PATH /biz-code
ADD hume /biz-code/hume
WORKDIR /biz-code/hume
CMD ./start.sh

Jenkinsfile

pipeline {
  agent {
    kubernetes {cloud 'kubernetes'
      cloud 'kubernetes-test'
      slaveConnectTimeout 1200
      workspaceVolume hostPathWorkspaceVolume(hostPath:"/opt/workspace", readOnly: false)
      yaml '''
apiVersion: v1
kind: Pod
spec:
  containers:
    - args: [\'$(JENKINS_SECRET)\', \'$(JENKINS_NAME)\']
      image: 'jenkins/jnlp-slave:latest-jdk11'
      name: jnlp
      imagePullPolicy: IfNotPresent
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "localtime"
          readOnly: false 
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      image: "ccr.ccs.tencentyun.com/xxxx/php_supervisor:kafka-k8s"
      imagePullPolicy: "IfNotPresent"
      name: "build"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "localtime"
        - mountPath: "/biz-code/hume/vendor"
          name: "phpdir"
          readOnly: false
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      image: "registry.cn-beijing.aliyuncs.com/citools/kubectl:self-1.17"
      imagePullPolicy: "IfNotPresent"
      name: "kubectl"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "localtime"
          readOnly: false
    - command:
        - "cat"
      env:
        - name: "LANGUAGE"
          value: "en_US:en"
        - name: "LC_ALL"
          value: "en_US.UTF-8"
        - name: "LANG"
          value: "en_US.UTF-8"
      image: "registry.cn-beijing.aliyuncs.com/citools/docker:19.03.9-git"
      imagePullPolicy: "IfNotPresent"
      name: "docker"
      tty: true
      volumeMounts:
        - mountPath: "/etc/localtime"
          name: "localtime"
          readOnly: false
        - mountPath: "/var/run/docker.sock"
          name: "dockersock"
          readOnly: false
  restartPolicy: "Never"
  imagePullSecrets:
  - name: qcloudregistrykey
  nodeSelector:
    build: "true"
  securityContext: {}
  volumes:
    - hostPath:
        path: "/var/run/docker.sock"
      name: "dockersock"
    - hostPath:
        path: "/usr/share/zoneinfo/Asia/Shanghai"
      name: "localtime"
    - name: "cachedir"
      hostPath:
        path: "/opt/gopkg"
    - name: "phpdir"
      hostPath:
        path: "/opt/phppkg"
'''
   }
}
stages {
  stage('Pulling Code') {
	parallel {
	  stage('Pulling Code by Jenkins') {
		when {
		  expression {
			env.giteeBranch == null
		  }
		}
		steps {
		  git(changelog: true, poll: true, url:'https://gitee.com/xxxx/hume.git', branch:"${BRANCH}", credentialsId: 'gitee-mima')
   		  script {
   		    COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
   		    TAG = BUILD_TAG + '-' + COMMIT_ID
   		    println "Current branch is ${BRANCH}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"

		  }
		}
  }
	  stage('Pulling Code by trigger') {
		when {
		  expression {
			env.giteeBranch != null
		  }
		}
		steps {
		  git(url: 'https://gitee.com/xxxx/hume.git', branch: env.giteeBranch, changelog: true, poll: true, credentialsId: 'gitee-mima')
		  script {
			COMMIT_ID = sh(returnStdout: true, script: "git log -n 1 --pretty=format:'%h'").trim()
			TAG = BUILD_TAG + '-' + COMMIT_ID
			println "Current branch is ${env.giteeBranch}, Commit ID is ${COMMIT_ID}, Image TAG is ${TAG}"
		  }
		}
	}
  }
}
	  stage('Building') {
		steps {
		  container(name: 'build') {
			sh """ 
                pwd
                whoami
			    cp Scripts/init.sh.dev init.sh
                chmod +x init.sh
                ./init.sh
                composer config -g --unset repos.packagist
                composer config repo.packagist composer https://mirrors.cloud.tencent.com/composer/
                composer update
                sudo chmod -R 777 ./*
                sudo rsync -avz --exclude build-work ./* build-work/hume/
			"""
		  }
		}
	  }
	  stage('Docker build for creating image') {
		environment {
		  HARBOR_USER = credentials('registry-secret')
		}
		steps {
		  container(name: 'docker') {
			sh """
              cd build-work
			  echo ${HARBOR_USER_USR} ${HARBOR_USER_PSW} ${TAG}
              docker login -u ${HARBOR_USER_USR} -p ${HARBOR_USER_PSW} ${HARBOR_ADDRESS}
			  docker build -t ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG} .
			  docker push ${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG}
			"""
		  }
		}
	  }
	  stage('Deploying to K8s') {
		environment {
		  MY_KUBECONFIG = credentials('k8s-config')
		}
		steps {
		  container(name: 'kubectl'){
			sh """
			  /usr/local/bin/kubectl --kubeconfig $MY_KUBECONFIG set image deploy -l app=${IMAGE_NAME} ${IMAGE_NAME}=${HARBOR_ADDRESS}/${REGISTRY_DIR}/${IMAGE_NAME}:${TAG} -n $NAMESPACE
			"""
		  }
		}
	  }
	}
  environment {
    COMMIT_ID = ""
    HARBOR_ADDRESS = "ccr.ccs.tencentyun.com"
    REGISTRY_DIR = "xxxx"
    IMAGE_NAME = "hume"
    NAMESPACE = "dev"
    TAG = ""
}
  parameters {
	gitParameter(branch: '', branchFilter: 'origin/(.*)', defaultValue: 'dev', description: 'Branch for build and deploy', name:'BRANCH', quickFilterEnabled: false, selectedValue: 'NONE', sortMode: 'NONE', tagFilter: '*', type: 'PT_BRANCH')
  }
}

hume/start.sh

mkdir /biz-code/hume/data/logs/supervisor
supervisord -c /biz-code/hume/Config/supervisor.conf
supervisorctl restart all

二、jenkins添加流水线任务，将镜像构建流程跑通

1、jenkins界面添加pipeline，注意提前安装好对应插件

Git
Git Parameter
Git Pipeline for Blue Ocean
GitLab
Credentials
Credentials Binding
Blue Ocean
Blue Ocean Pipeline Editor
Blue Ocean Core JS
Pipeline SCM API for Blue Ocean
Dashboard for Blue Ocean
Build With Parameters
Dynamic Extended Choice Parameter Plug-In
Dynamic Parameter Plug-in
Extended Choice Parameter
List Git Branches Parameter
Pipeline
Pipeline: Declarative
Kubernetes
Kubernetes CLI
Kubernetes Credentials
Image Tag Parameter
Active Choices

说明： 第一步和第二步的一个流程说明

首先，点击jenkins构建，会选择对应的分支（如果是自动触发会获取到提交的分支，具体看jenkinsfile流程），然后jenkins会通过k8s插件调用k8s，根据jenkinsfile中定义的模板启动一个任务pod，pod中会先拉取代码，然后执行一系列的初始化操作（init.sh、拉取依赖包、并拷贝所有文件到build-work/hume，以便于后面构建镜像），再然后会build镜像，推送到镜像仓库，最后就是使用kubectl命令执行更新服务器pod的镜像版本发布

三、业务运行的deploy和service

1、业务pod是以sidecar模式运行的，一个业务container，一个日志收集container，通过共享目录的形式来收集业务存在容器里面的日志

hume-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: hume
  labels:
    app: hume
spec:
  selector:
    matchLabels:
      app: hume
  replicas: 1
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 0
      maxSurge: 1
  # minReadySeconds: 30
  template:
    metadata:
      labels:
        app: hume
    spec:
      containers:
        - name: filebeat                        
          image: registry.cn-beijing.aliyuncs.com/dotbalo/filebeat:7.10.2 
          resources:
            requests:
              memory: "100Mi"
              cpu: "10m"
            limits:
              cpu: "200m"
              memory: "300Mi"
          imagePullPolicy: IfNotPresent
          env:
            - name: podIp
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: status.podIP
            - name: podName
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.name
            - name: podNamespace
              valueFrom:
                fieldRef:
                  apiVersion: v1
                  fieldPath: metadata.namespace
            - name: podDeployName
              value: hume
            - name: TZ
              value: "Asia/Shanghai"
          securityContext:
            runAsUser: 0
          volumeMounts:
            - name: logpath
              mountPath: /data/log/
            - name: filebeatconf
              mountPath: /usr/share/filebeat/filebeat.yml 
              subPath: usr/share/filebeat/filebeat.yml
        - name: hume
          image: ccr.ccs.tencentyun.com/xxxx/hume:jenkins-hume-dev-71-0f5bbb9a
          imagePullPolicy: IfNotPresent
          volumeMounts:
            - name: logpath
              mountPath: /biz-code/hume/data/logs/
          env:
            - name: TZ
              value: "Asia/Shanghai"
            - name: LANG
              value: C.UTF-8
            - name: LC_ALL
              value: C.UTF-8
          livenessProbe:
            failureThreshold: 2
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            tcpSocket:
              port: 7777
            timeoutSeconds: 2
          ports:
          - containerPort: 7777
            name: web
            protocol: TCP
          readinessProbe:
            failureThreshold: 2
            initialDelaySeconds: 30
            periodSeconds: 10
            successThreshold: 1
            tcpSocket:
              port: 7777
            timeoutSeconds: 2
          resources:
            limits:
              cpu: 994m
              memory: 1170Mi
            requests:
              cpu: 300m
              memory: 300Mi
      dnsPolicy: ClusterFirst
      imagePullSecrets:
      - name: qcloudregistrykey
      restartPolicy: Always
      securityContext: {}
      serviceAccountName: default
      volumes:
        - name: logpath
          emptyDir: {}
        - name: filebeatconf
          configMap:
            name: filebeatconf
            items:
              - key: filebeat.yml
                path: usr/share/filebeat/filebeat.yml

hume-service.yaml

---
apiVersion: v1
kind: Service
metadata:
  creationTimestamp: null
  labels:
    app: hume
  name: hume-service
  namespace: dev
spec:
  ports:
  - name: hume
    port: 7777
    protocol: TCP
    targetPort: 7777
  selector:
    app: hume
  sessionAffinity: None
  type: NodePort
status:
  loadBalancer: {}

四、日志收集流程

日志收集流程：采用filebeat+kafka+logstash+es+kibana的形式来做的

filebeat容器以sidecar模式与业务容器绑定，收集日志推送到kafka，在kafka中创建topic，logstash会读取kafka中topic消费业务日志，并推送至es，然后由kibana进行展示

下面需要文件可在如下连接查找

https://github.com/cs81/k8s-infrastructure/tree/master/efk-7.10.2/filebeat

1、安装helm

wget https://get.helm.sh/helm-v3.1.2-linux-amd64.tar.gz
tar xf helm-v3.1.2-linux-amd64.tar.gz
mv linux-amd64/ helm
cd helm/
cp -r helm /usr/local/bin/
helm version

添加两个仓库
helm repo add bitnami https://charts.bitnami.com/bitnami
helm repo add ali-stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts

2、安装kafka和zookeeper

helm pull bitnami/zookeeper
tar xf zookeeper-11.1.2.tgz
cd zookeeper/
helm install zookeeper -n logging --set auth.enabled=false --set allowAnonymousLogin=true --set persistence.enabled=false .
cd ../
helm pull bitnami/kafka
tar xf kafka-20.0.6.tgz
cd kafka/
helm install kafka -n logging --set zookeeper.enabled=false --set replicaCount=1 --set externalZookeeper.servers=zookeeper --set persistence.enabled=false .

kubectl get pod -n logging

3、安装logstash

filebeat-cm.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: filebeatconf
data:
  filebeat.yml: |-
    filebeat.inputs:
    - input_type: log
      paths:
        - /data/log/*/*.log
      tail_files: true
      fields:
        pod_name: '${podName}'
        pod_ip: '${podIp}'
        pod_deploy_name: '${podDeployName}'
        pod_namespace: '${podNamespace}'
    output.kafka:
      hosts: ["kafka.logging:9092"]
      topic: "filebeat-sidecar"
      codec.json:
        pretty: false
      keep_alive: 30s

logstash-cm.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: logstash-configmap
data:
  logstash.yml: |
    http.host: "0.0.0.0"
    path.config: /usr/share/logstash/pipeline
  logstash.conf: |
    # all input will come from filebeat, no local logs
    input {
      kafka {
              enable_auto_commit => true
              auto_commit_interval_ms => "1000"
              bootstrap_servers => "kafka:9092"
              topics => ["filebeat-sidecar"]
              type => ["filebeat-sidecar"]
              codec=>multiline{
                        pattern=>"^\d{4}-"
                        negate=>true
                        what=>"previous"
                }
          }
    }

    output {
       stdout{ codec=>rubydebug}
       if [type] == "filebeat-sidecar"{
           elasticsearch {
                   hosts => ["172.16.64.12:6123"]
             index => "filebeat-%{+YYYY.MM.dd}"
          }
       } else{
          elasticsearch {
             hosts => ["172.16.64.12:6123"]
             index => "other-input-%{+YYYY.MM.dd}"
          }
       }
    }

logstash-deploy.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: logstash-deployment
spec:
  selector:
    matchLabels:
      app: logstash
  replicas: 1
  template:
    metadata:
      labels:
        app: logstash
    spec:
      containers:
      - name: logstash
        image: registry.cn-beijing.aliyuncs.com/dotbalo/logstash:7.10.1 
        ports:
        - containerPort: 5044
        volumeMounts:
          - name: config-volume
            mountPath: /usr/share/logstash/config
          - name: logstash-pipeline-volume
            mountPath: /usr/share/logstash/pipeline
      volumes:
      - name: config-volume
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.yml
              path: logstash.yml
      - name: logstash-pipeline-volume
        configMap:
          name: logstash-configmap
          items:
            - key: logstash.conf
              path: logstash.conf

logstash-service.yaml

kind: Service
apiVersion: v1
metadata:
  name: logstash-service
spec:
  selector:
    app: logstash
  ports:
  - protocol: TCP
    port: 5044
    targetPort: 5044
  type: ClusterIP

然后es和kibana因为之前装过，并且使用docker安装的，这里就不写了，上面logstash配置里面注意填写对应的es地址

---

整个流程基本就是这样，后续再添加监控及其他