八、数据库备份工具

一、构建数据库备份工具

镜像地址: docker.io/duanshuaixing02/databases-backup:v001
源码地址: https://github.com/duanshuaixing/tools/tree/master/databases-backup

二、通过k8s cronjob检测数据库是否卡主

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: check-mongodb-status
data:
  mongodb-check-status.sh: |
    <<!
     **********************************************************
     * Author        : duanshuaixing
     * Email         : duanshuaixing@gmail.com
     * Last modified : 2021-10-30 20:48
     * Filename      : check-status.sh
     * Description   : check-mongodb-status 
     * *******************************************************
    !
    #!/bin/bash
    
    DINGTALK_TOKEN=https://oapi.dingtalk.com/robot/send?access_token=钉钉群机器人(关键词:集群)
    Check_time=`date +%Y/%m/%d-%T`
    HTTP_PROXY_ENABLED=false（设置为true时需要和HTTP_PROXY一起用）
    HTTP_PROXY=http_proxy:port(根据实际代理地址填写,用于发送钉钉告警)
    
    send_message(){
    
         if [ "$HTTP_PROXY_ENABLED" = true ];then
             curl --proxy $HTTP_PROXY "$DINGTALK_TOKEN" -H 'Content-Type: application/json' -d "{\"actionCard\": {\"text\": \"$context\",},\"msgtype\": \"actionCard\"}"
         else
             curl "$DINGTALK_TOKEN" -H 'Content-Type: application/json' -d "{\"actionCard\": {\"text\": \"$context\",},\"msgtype\": \"actionCard\"}"
        fi
    }
    
    cluster01(){
    
        Cluster_name=k8s-mongodb
        Cluster_replSetName=rs0
        Cluster_members=192.168.86.xxx:27017,192.168.86.xxx:27017,192.168.86.xxx:27017
        Cluster_admin_user=<admin-user>
        Cluster_admin_password=<admin-password>
    
        check_hang(){
       
            timeout -k 5s 15s  mongo admin --host $Cluster_replSetName/$Cluster_members -u $Cluster_admin_user -p $Cluster_admin_password --authenticationDatabase "admin" --eval 'db.system.version.find()'
        
            if [ $? -eq 0 ];then
               context="集群访问正常:$Cluster_name\n\n检测时间: $Check_time"
               echo $context
               send_message
            else
               context="[关键通知]\n\n集群卡主: $Cluster_name\n\n集群地址: \n\n$Cluster_members\n\n检测时间: $Check_time"
               echo $context
               send_message
            fi
        }
     
        check_hang
    
    }
    
    check_status(){
    
        cluster01
    }
    
    check_status
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: mongodb-check-status
spec:
  schedule: "*/1 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          volumes:
            - name: scripts
              configMap:
                name: check-mongodb-status
                defaultMode: 0777
          containers:
            - name: check
              image: duanshuaixing02/databases-backup:v001
              env:
                - name: TZ
                  value: Asia/Shanghai
              command: ["/bin/bash", "-c", "/tmp/mongodb-check-status.sh"]
              resources:
                limits:
                cpu: "100m"
                memory: "200Mi"
              requests:
                cpu: "50m"
                memory: "100Mi"
              securityContext:
                runAsUser: 0
              volumeMounts:
                - name: scripts
                  mountPath: /tmp/mongodb-check-status.sh
                  subPath: mongodb-check-status.sh

三、通过k8s cronjob 备份mongodb数据库上传到s3存储

1、部署s3存储服务(本次使用minio作为s3存储服务)

docker run -itd --name minio --restart=always -p 9000:9000 -p 9001:9001 -e "MINIO_ACCESS_KEY=admin" -e "MINIO_SECRET_KEY=admin123456" -v /data/minio/data:/data -v /data/minio/config:/root/.minio minio/minio server /data --console-address ":9001"

2、访问ip+9091登录管理页面创建bucket

登录用户名:admin
登录密码:admin123456

3、创建cronjob

---
kind: ConfigMap
apiVersion: v1
metadata:
  name: mongodb-backup-script
data:
  backup.sh: |
    #!/bin/bash
    export PATH=~/.local/bin:$PATH

    DINGTALK_TOKEN=https://oapi.dingtalk.com/robot/send?access_token=钉钉机器人地址
    ADMIN_USERNAME=root（mongo管理员账号）
    ADMIN_PASSWORD=rootPassw0rd（mongo管理员密码）
    REPLICA_NAME=rs0
    REPLICA_MEMBERS=192.168.86.xxx:27017,192.168.86.xxx:27017,192.168.86.xxx:27017（集群地址）

    MONGODB_CLUSTER_NAME=mongodb-backup-test（集群名称根据需要自定义）
    MONGODB_DATA_BACKUP=/tmp/mongodb-backup
    MONGODB_BACKUP_NAME=`date +%Y%m%d`
    MONGODB_DATA_BACKUP_CLUSTER_DIR=$MONGODB_DATA_BACKUP/$MONGODB_CLUSTER_NAME/$MONGODB_BACKUP_NAME

    OSS_BUCKET=mongodb-backup（s3对象存储bucket名称）
    OSS_AK=admin（对象存储AK）
    OSS_SK=admin123456(对象存储AK）
    OSS_ENDPOINT=http://192.168.86.40:9000(对象存储endpoint)

    REMOVE_DIR=$MONGODB_DATA_BACKUP/$MONGODB_CLUSTER_NAME
    REMOVE_DATE=`date +%Y%m%d -d "14 day ago"`
    REMOVE_OSS_BACKUP_DIR=s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME

    #HTTP_PROXY_ENABLED=false
    #HTTP_PROXY=<http-proxy>:<port>

    send_message(){

         if [ "$HTTP_PROXY_ENABLED" = true ];then
             curl --proxy $HTTP_PROXY "$DINGTALK_TOKEN" -H 'Content-Type: application/json' -d "{\"actionCard\": {\"text\": \"$context\",},\"msgtype\": \"actionCard\"}"
         else
             curl "$DINGTALK_TOKEN" -H 'Content-Type: application/json' -d "{\"actionCard\": {\"text\": \"$context\",},\"msgtype\": \"actionCard\"}"
        fi
    }

    mongo_backup(){

        if [ ! $MONGODB_DATA_BACKUP ] && [ ! $MONGODB_CLUSTER_NAME ];then
            context="获取变量失败，$MONGODB_CLUSTER_NAME 集群备份失败,请检查集群备份地址和集群名称"
            send_message
            echo $context
            exit 1
        fi

        mkdir -p $MONGODB_DATA_BACKUP/$MONGODB_CLUSTER_NAME
        cd $MONGODB_DATA_BACKUP/$MONGODB_CLUSTER_NAME && rm -rf `date +%Y%m%d`
        mkdir -p $MONGODB_DATA_BACKUP_CLUSTER_DIR
        mongodump -h $REPLICA_NAME/$REPLICA_MEMBERS -u$ADMIN_USERNAME -p$ADMIN_PASSWORD --readPreference=secondary --oplog --forceTableScan --authenticationDatabase admin --gzip -o $MONGODB_DATA_BACKUP_CLUSTER_DIR

        if [ $? -eq 0 ];then

            upload_oss(){

                mkdir -p ~/.aws
                echo "[default]" >~/.aws/config
                echo "[default]" >>~/.aws/credentials
                echo "aws_access_key_id = oss_ak" >>~/.aws/credentials
                echo "aws_secret_access_key = oss_sk" >>~/.aws/credentials
                sed -i "s/oss_ak/$OSS_AK/g" ~/.aws/credentials
                sed -i "s/oss_sk/$OSS_SK/g" ~/.aws/credentials

                aws --endpoint-url=$OSS_ENDPOINT s3 sync $MONGODB_DATA_BACKUP_CLUSTER_DIR s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME/$MONGODB_BACKUP_NAME
                if [ $? -eq 0 ];then
                    MONGODB_BACKUP_SIZE=`aws --endpoint-url=$OSS_ENDPOINT s3 ls s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME/$MONGODB_BACKUP_NAME --recursive --human-readable --summarize|grep "Total Size"|awk -F ':' '{print $2}'`
                    MONGODB_BACKUP_TOTAL_NUMBER=`aws --endpoint-url=$OSS_ENDPOINT s3 ls s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME/ |wc -l`
                    MONGODB_BACKUP_TOTAL_SIZE=`aws --endpoint-url=$OSS_ENDPOINT s3 ls s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME/ --recursive --human-readable --summarize|grep "Total Size"|awk -F ':' '{print $2}'`
                    context="MongoDB集群：$MONGODB_CLUSTER_NAME 备份成功\n\n备份路径：s3://$OSS_BUCKET/$MONGODB_CLUSTER_NAME/$MONGODB_BACKUP_NAME\n\n备份大小：$MONGODB_BACKUP_SIZE\n\n集群总备份数量：$MONGODB_BACKUP_TOTAL_NUMBER \n\n集群总备份大小：$MONGODB_BACKUP_TOTAL_SIZE\n\n获取备份请联系管理员"
                    send_message
                    sleep 3000
                else
                    context="$MONGODB_CLUSTER_NAME 集群备份上传oss失败,请检查cronjob日志"
                    send_message
                    exit 1
                fi
            }

            upload_oss

        else
            context="$MONGODB_CLUSTER_NAME 集群mongodump备份失败,请检查cronjob日志"
            send_message
            exit 1
        fi
    }

    backup_clean(){

        for subdir in `ls $REMOVE_DIR`;
        do
            if [ "${subdir}" \< "${REMOVE_DATE}" ];then

                oss_clean(){
                    aws --endpoint-url=$OSS_ENDPOINT s3 rm --recursive $REMOVE_OSS_BACKUP_DIR/$subdir
                    if [ $? -eq 0 ];then
                        context="$MONGODB_CLUSTER_NAME集群过期备份清理成功，清理文件地址:$REMOVE_OSS_BACKUP_DIR/$subdir"
                        send_message
                    else
                        context="$MONGODB_CLUSTER_NAME集群过期备份清理失败，请检查cronjob日志。清理文件地址:$REMOVE_OSS_BACKUP_DIR/$subdir"
                        send_message
                    fi

                }

                local_clean(){
                    rm -rf $REMOVE_DIR/$subdir >/dev/null
                    if [ $? -eq 0 ];then
                        context="$MONGODB_CLUSTER_NAME集群备份$REMOVE_DIR/$subdir 清理成功"
                        echo $context
                    else
                        context="$MONGODB_CLUSTER_NAME集群备份$REMOVE_DIR/$subdir 清理失败，请检查cronjob日志"
                        echo $context
                    fi
                }

                oss_clean
                local_clean
            fi
        done

    }

    main(){
        mongo_backup
        backup_clean
    }
    main
---
apiVersion: batch/v1beta1
kind: CronJob
metadata:
  name: mongodb-backup
spec:
  schedule: "*/1 * * * *"
  jobTemplate:
    spec:
      template:
        spec:
          restartPolicy: OnFailure
          volumes:
            - name: backup-script
              configMap:
                name: mongodb-backup-script
                defaultMode: 0777
            - name: mongodb-backup-cronjobs
              hostPath:
                path: /data/mongodb-backup
          containers:
            - name: mongodump
              image: duanshuaixing02/databases-backup:v001
              command: ["/bin/bash", "-c", "/scripts/backup.sh"]
              securityContext:
                runAsUser: 0
              volumeMounts:
                - name: backup-script
                  mountPath: /scripts/backup.sh
                  subPath: backup.sh
                - name: mongodb-backup-cronjobs
                  mountPath: /tmp/mongodb-backup