k8s 1.23.1部署gitlab、 gitlab-runn配置k8s集群编写.gitlab-ci.yml实现根据tag,分支发布CICD流程

1. k8s环境部署：https://blog.csdn.net/weixin_43606975/article/details/119947061?spm=1001.2014.3001.5501
2. k8s部署gitlab

#cat gitlab-deploy.yaml
apiVersion: v1
kind: Service
metadata:
  name: gitlab
spec:
  type: NodePort
  ports:
  # Port上的映射端口
  - port: 443
    targetPort: 443
    nodePort: 443
    name: gitlab443
  - port: 80
    nodePort: 80
    targetPort: 80
    name: gitlab80
  - port: 22
    targetPort: 22
    name: gitlab22
  selector:
    app: gitlab

---

apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab
spec:
  selector:
    matchLabels:
      app: gitlab
  revisionHistoryLimit: 2
  template:
    metadata:
      labels:
        app: gitlab
    spec:
      containers:
      # 应用的镜像
      - image: gitlab/gitlab-ce
        name: gitlab
        imagePullPolicy: IfNotPresent
        # 应用的内部端口
        ports:
        - containerPort: 443
          name: gitlab443
        - containerPort: 80
          name: gitlab80
        - containerPort: 22
          name: gitlab22
        volumeMounts:
        # gitlab持久化
        - name: gitlab-persistent-config
          mountPath: /etc/gitlab
       # - name: gitlab-persistent-logs
       #   mountPath: /var/log/gitlab
        - name: gitlab-persistent-data
          mountPath: /var/opt/gitlab
      imagePullSecrets:
      - name: devops-repo
      volumes:
      # 使用本地存储，用于测试，当然你可以不用k8s部署，k8s稍微快一点。
      - name: gitlab-persistent-config
        hostPath:
          path: /home/gitlab/config
      - name:  gitlab-persistent-data
        hostPath:
          path: /home/gitlab/data

3. 启动访问

kubectl apply -f gitlab-deploy.yaml
#查看密码：在挂载的node的目录上或者进去容器里面
cat initial_root_password
#容器
cat /etc/gitlab/initial_root_password

4. gitlab配置k8s集群
   
   
   
   Kubernetes cluster name: 随意
   API URL：

kubectl cluster-info

CA Certificate：

kubectl get secret $(kubectl get secret | grep default-token | awk '{print $1}') -o jsonpath="{['data']['ca\.crt']}" | base64 --decode

Enter new Service Token：
需要创建gitlab认证：

#cat gitlab-admin-service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: gitlab-admin
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: gitlab-admin
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: gitlab-admin
  namespace: kube-system

kubectl apply -f gitlab-admin-service-account.yaml
kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep gitlab-admin | awk '{print $1}')

5. 保存可能会报错：

Requests to the local network are not allowed

gitlab需要设置一下：

6. 部署gitlab-Runner
   官网：https://docs.gitlab.com/runner/install/kubernetes-agent.html

#cat runner-manifest.yaml
---
# Source: gitlab-runner/templates/service-account.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  annotations:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
---
# Source: gitlab-runner/templates/secrets.yaml
apiVersion: v1
kind: Secret
metadata:
  name: "gitlab-runner"
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
type: Opaque
data:
  runner-registration-token: "UUNHRGY1WVM0QXNpdFA0eDZRQkw="   #echo VhSA7XiDmU65LoHxN-Gs | base64 -w0
  runner-token: ""
---
# Source: gitlab-runner/templates/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: gitlab-runner
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
data:
  entrypoint: |
    #!/bin/bash
    set -e

    mkdir -p /home/gitlab-runner/.gitlab-runner/

    cp /configmaps/config.toml /home/gitlab-runner/.gitlab-runner/

    # Set up environment variables for cache
    if [[ -f /secrets/accesskey && -f /secrets/secretkey ]]; then
      export CACHE_S3_ACCESS_KEY=$(cat /secrets/accesskey)
      export CACHE_S3_SECRET_KEY=$(cat /secrets/secretkey)
    fi

    if [[ -f /secrets/gcs-applicaton-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-applicaton-credentials-file"
    elif [[ -f /secrets/gcs-application-credentials-file ]]; then
      export GOOGLE_APPLICATION_CREDENTIALS="/secrets/gcs-application-credentials-file"
    else
      if [[ -f /secrets/gcs-access-id && -f /secrets/gcs-private-key ]]; then
        export CACHE_GCS_ACCESS_ID=$(cat /secrets/gcs-access-id)
        # echo -e used to make private key multiline (in google json auth key private key is oneline with \n)
        export CACHE_GCS_PRIVATE_KEY=$(echo -e $(cat /secrets/gcs-private-key))
      fi
    fi

    if [[ -f /secrets/azure-account-name && -f /secrets/azure-account-key ]]; then
      export CACHE_AZURE_ACCOUNT_NAME=$(cat /secrets/azure-account-name)
      export CACHE_AZURE_ACCOUNT_KEY=$(cat /secrets/azure-account-key)
    fi

    if [[ -f /secrets/runner-registration-token ]]; then
      export REGISTRATION_TOKEN=$(cat /secrets/runner-registration-token)
    fi

    if [[ -f /secrets/runner-token ]]; then
      export CI_SERVER_TOKEN=$(cat /secrets/runner-token)
    fi

    # Validate this also at runtime in case the user has set a custom secret
    if [[ ! -z "$CI_SERVER_TOKEN" && "1" -ne "1" ]]; then
      echo "Using a runner token with more than 1 replica is not supported."
      exit 1
    fi

    # Register the runner
    if ! sh /configmaps/register-the-runner; then
      exit 1
    fi

    # Run pre-entrypoint-script
    if ! bash /configmaps/pre-entrypoint-script; then
      exit 1
    fi

    # Start the runner
    exec /entrypoint run --user=gitlab-runner \
      --working-directory=/home/gitlab-runner

  config.toml: |
    concurrent = 10
    check_interval = 30
    log_level = "info"
    listen_address = ':9252'


  config.template.toml:   |
    [[runners]]
      [runners.kubernetes]
        namespace = "gitlab"
        image = "ubuntu:16.04"


  register-the-runner: |
    #!/bin/bash
    MAX_REGISTER_ATTEMPTS=30

    for i in $(seq 1 "${MAX_REGISTER_ATTEMPTS}"); do
      echo "Registration attempt ${i} of ${MAX_REGISTER_ATTEMPTS}"
      /entrypoint register \
        --run-untagged=true \
        --template-config /configmaps/config.template.toml \
        --non-interactive

      retval=$?

      if [ ${retval} = 0 ]; then
        break
      elif [ ${i} = ${MAX_REGISTER_ATTEMPTS} ]; then
        exit 1
      fi

      sleep 5
    done

    exit 0

  check-live: |
    #!/bin/bash
    if /usr/bin/pgrep -f .*register-the-runner; then
      exit 0
    elif /usr/bin/pgrep gitlab.*runner; then
      exit 0
    else
      exit 1
    fi

  pre-entrypoint-script: |
---
# Source: gitlab-runner/templates/role.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: "Role"
metadata:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
  namespace: "gitlab"
rules:
- apiGroups: [""]
  resources: ["pods","pods/exec","secrets"]
  verbs: ["get","list","watch","create","patch","delete"]
---
# Source: gitlab-runner/templates/role-binding.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: "RoleBinding"
metadata:
  name: gitlab-runner
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
  namespace: "gitlab"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: "Role"
  name: gitlab-runner
subjects:
- kind: ServiceAccount
  name: gitlab-runner
  namespace: "gitlab"
---
# Source: gitlab-runner/templates/deployment.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
  name: gitlab-runner
  namespace: "gitlab"
  labels:
    app: gitlab-runner
    chart: gitlab-runner-0.42.0
    release: "gitlab-runner"
    heritage: "Helm"
spec:
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: gitlab-runner
  template:
    metadata:
      labels:
        app: gitlab-runner
        chart: gitlab-runner-0.42.0
        release: "gitlab-runner"
        heritage: "Helm"
      annotations:
        checksum/configmap: 0eb770c0b5b3282a9833fb5ccec0ecf843db4e79ac5a3c6fb696077df5abde21
        checksum/secrets: 0b3f7c87e11d8e0da7056f3ade5bd18756073ff52a72e519c98c0dff5059e9c2
        prometheus.io/scrape: 'true'
        prometheus.io/port: "9252"
    spec:
      securityContext:
        runAsUser: 100
      terminationGracePeriodSeconds: 3600
      serviceAccountName: gitlab-runner
      containers:
      - name: gitlab-runner
        image: registry.gitlab.com/gitlab-org/gitlab-runner:alpine-v15.1.0
        imagePullPolicy: "IfNotPresent"
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          privileged: false
          readOnlyRootFilesystem: false
          runAsNonRoot: true
          runAsUser: 100
        lifecycle:
          preStop:
            exec:
              command: ["/entrypoint", "unregister", "--config=/home/gitlab-runner/.gitlab-runner/config.toml"]
        lifecycle:
          preStop:
            exec:
              command: ["/entrypoint", "unregister", "--all-runners"]
        command: ["/usr/bin/dumb-init", "--", "/bin/bash", "/configmaps/entrypoint"]
        env:

        - name: CI_SERVER_URL
          value: "http://192.168.10.100/"    #gitlab地址
        - name: CLONE_URL
          value: ""
        - name: RUNNER_EXECUTOR
          value: "kubernetes"
        - name: REGISTER_LOCKED
          value: "false"
        - name: RUNNER_TAG_LIST
          value: "k8s-runner,k8s"
        - name: KUBERNETES_PRIVILEGED
          value: "true"
        livenessProbe:
          exec:
            command: ["/bin/bash", "/configmaps/check-live"]
          initialDelaySeconds: 60
          timeoutSeconds: 1
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
        readinessProbe:
          exec:
            command: ["/usr/bin/pgrep","gitlab.*runner"]
          initialDelaySeconds: 10
          timeoutSeconds: 1
          periodSeconds: 10
          successThreshold: 1
          failureThreshold: 3
        ports:
        - name: "metrics"
          containerPort: 9252
        volumeMounts:
        - name: projected-secrets
          mountPath: /secrets
        - name: etc-gitlab-runner
          mountPath: /home/gitlab-runner/.gitlab-runner
        - name: configmaps
          mountPath: /configmaps
        resources:
          {}
      volumes:
      - name: runner-secrets
        emptyDir:
          medium: "Memory"
      - name: etc-gitlab-runner
        emptyDir:
          medium: "Memory"
      - name: projected-secrets
        projected:
          sources:
            - secret:
                name: "gitlab-runner"
                items:
                  - key: runner-registration-token
                    path: runner-registration-token
                  - key: runner-token
                    path: runner-token
      - name: configmaps
        configMap:
          name: gitlab-runner

手动安装：
helm安装：https://blog.csdn.net/weixin_43606975/article/details/119944602?spm=1001.2014.3001.5501

#添加源
helm repo add gitlab https://charts.gitlab.io

egrep -v "#" gitlab-runn.yaml | grep -v "^$"
imagePullPolicy: IfNotPresent
#gitlab服务器地址
gitlabUrl: http://192.168.10.100/
#runner注册token
runnerRegistrationToken: "mNaxzsqyxpwQQCxJkSTa"  #从gitlab-runner获取的如下图
#当停止管道时等待其他作业终止时间
terminationGracePeriodSeconds: 3600
#最大并发作业数量
concurrent: 10
#新作业检查时隔
checkInterval: 30
sessionServer:
  enabled: false
rbac:
  create: true
  resources: ["pods", "pods/exec", "secrets"]
  verbs: ["get", "list", "watch", "create", "patch", "delete"]
  rules: []
  clusterWideAccess: false
  podSecurityPolicy:
    enabled: false
    resourceNames:
    - gitlab-runner
metrics:
  enabled: true
  portName: metrics
  port: 9252
  serviceMonitor:
    enabled: false
service:
  enabled: false
  type: ClusterIP
runners:
  config: |
    [[runners]]
      [runners.kubernetes]
        namespace = "{{.Release.Namespace}}"
        image = "ubuntu:16.04"
  #执行器类型
  executor: kubernetes
  #是否锁定false
  locked: false
  #你的tags
  tags: "k8s-runner,k8s"
  #是否运行没有标签的项目
  runUntagged: true
  #开启docker in docker
  privileged: true
  cache: {}
  builds: {}
  services: {}
  helpers: {}
securityContext:
  runAsUser: 100
  fsGroup: 65533
resources: {}
affinity: {}
nodeSelector: {}
tolerations: []
hostAliases: []
podAnnotations: {}
podLabels: {}
secrets: []
configMaps: {}

helm template --namespace gitlab  gitlab-runner -f gitlab-runn.yaml gitlab/gitlab-runner > runner-manifest.yaml
#执行
kubectl  -f apply runner-manifest.yaml

k8s 1.23.1 会报错fsGroup的问题，删了就好。

7. 查看gitlab是否注册上
   
8. 配置连接k8s的api
   REGISTRY_USERNAME：镜像仓库用户名 root
   REGISTRY_PASSWORD：镜像仓库密码 123456789
   kube_config：KubeConfig的编码字符串

echo $(cat ~/.kube/config | base64) | tr -d " "

9. 配置.gitlab-ci.yml
   

#.gitlab-ci.yml
stages:
  - package
  - buidl
  - deploy

package:
    stage: package
    image: golang:1.17.8-alpine3.15
    tags: 
      - k8s
    script:
     - "go build -o gotest test.go"
     - "ls ./;pwd"
    only:
     - "tags"
    artifacts:             //下阶段可以使用此文件
      paths:
        - ./gotest

buidl:
    stage: buidl
    tags: 
      - k8s
    script:
     - "ls ./;pwd"
    only:
     - "tags"
     
deploy:
    stage: deploy
    image: registry.cn-hangzhou.aliyuncs.com/haoshuwei24/kubectl:1.16.6
    tags: 
      - k8s
    script:
     - mkdir -p $HOME/.kube
     - echo "$KUBERNETES_SECRET" >> "$HOME/.kube/config"
     - echo "$KUBERNETES_SECRET"
     - kubectl version
     - kubectl  get pods -A
    only:
     - "tags"  //用的tags，也可以用master,tages需要设置，master不需要。
     

#cat test.go
package main

import (
    "fmt"
)

func main() {
   // %d 表示整型数字，%s 表示字符串
    var stockcode=123
    var enddate="2020-12-31"
    var url="Code=%d&endDate=%s"
    var target_url=fmt.Sprintf(url,stockcode,enddate)
    fmt.Println(target_url)
}

9. 打tags自动发布
   
   
   
   

runner跑完了job任务就会自动删除。
kaniko-executor打包镜像后期添加，主要是没有镜像仓库。
k8s部署的gitlab问题：clone地址问题，不能解析到。临时解决方案：
进入容器修改地址，重启gitlab。

#kubectl  exec -it gitlab-58bd6f9c-w5hpb  bash
#cat /var/opt/gitlab/gitlab-rails/etc/gitlab.yml
...
  gitlab:
    ## Web server settings (note: host is the FQDN, do not include http://)
    host: 192.168.10.100
    port: 80
    https: false

...
#gitlab-ctl restart 

收工