1.自建ES集群下载repository-s3插件
首先先要在es插件目录安装repository-s3的插件,必须在每个节点上都安装。
./bin/elasticsearch-plugin install repository-s32
修改elasticsearch的jvm.options集群所有服务器节点都要操作,在配置文件最后添加如下内容
-Des.allow_insecure_settings=true
3 验证插件重启各节点上的elasticsearch服务,然后在浏览器中输入
http://esip地址/_cat/plugins,马上能看到所有节点安装的插件,则表示安装成功
4 创建s3桶下面的文件夹
mkdir -p backup_estouch backup_es/1.txts3cmd put --recursive backup_es s3://cxcloud-resource-test/backup_es/
5.创建es快照存储仓库
curl -XPUT http://192.168.1.64:9200/_snapshot/my_backup_03 -H 'Content-Type: application/json' -d '{"type": "s3","settings": {"access_key": "oss的ak","secret_key": "oss的sk","base_path": "backup_es","bucket": "cxcloud-resource-test","endpoint": "obs.cn-south-1.myhuaweicloud.com"}}'
注意:查看刚创建的存储仓库:
curl -XGET localhost:9200/_snapshot/my_backup_03?pretty
查看所有的存储桶:
curl -XGET localhost:9200/_snapshot/_all?pretty
6.开始做快照
curl -s -XPUT 'http://localhost:9200/_snapshot/my_backup_03/snapshot_20210923?wait_for_completion=true'
7.确认备份仓库是否创建成功:
curl -XPOST http://localhost:9200/_snapshot/backup/_verify?pretty
8.查看s3中是否存在快照文件
s3cmd ls s3://cxcloud-resource-test
9.恢复
curl -X POST "localhost:9200/_snapshot/my_backup_03/snapshot_20210923/_restore"
10.服务器添加定时任务
crontab -e # auto backup es data 0 2 * * * sh /usr/local/src/es_backup.sh >/dev/null 2>&1
参考文档:
https://blog.51cto.com/u_15060551/2651774
https://blog.csdn.net/ale2012/article/details/82702128
es_backup.sh 备份脚本
#!/bin/bash
webhook_url="https://oapi.dingtalk.com/robot/send?access_token=6da7591c4f4bc627279ce76fbcf4047983c62e11a6e014ee02a14753f4368382"
log_file="/tmp/esBackup.log"
log_result="/tmp/esBackupResult.log"
notity_phone="17626517060"
pre_day=`date -d "1 day ago" +"%Y%m%d"`
delete_day=`date -d "30 day ago" +"%Y%m%d"`
delete_snapshotname="cxcloud_backup_${delete_day}"
snapshotname="cxcloud_backup_${pre_day}"
snapshotStore="my_backup_03"
snapshot_status=`curl -s -w %{http_code} -o /dev/null localhost:9200/_snapshot/$snapshotStore/$snapshotname`
if [[ $snapshot_status == 200 ]];
then
echo "snapshot exsit" #>> /tmp/snapshot_exsit.log
exit 1
else
#curl -X DELETE "localhost:9200/_snapshot/'$snapshotStore'/'$delete_snapshotname'" >/dev/null 2>&1
curl -s -XPUT 'http://localhost:9200/_snapshot/'$snapshotStore'/'$snapshotname'?wait_for_completion=true' > $log_result
snapshot=`cat $log_result |jq .snapshot.snapshot |tr -d '"'`
state=`cat $log_result |jq .snapshot.state |tr -d '"'`
stime=`cat $log_result |jq .snapshot.start_time |tr -d '"'`
start_time=`date -d ${stime} +"%Y-%m-%d %H:%M:%S"`
etime=`cat $log_result |jq .snapshot.end_time | tr -d '"'`
end_time=`date -d "${etime}" +"%Y-%m-%d %H:%M:%S"`
duration_in_millis=`cat $log_result |jq .snapshot.duration_in_millis | tr -d '"'`
if [[ $state == "SUCCESS" ]];
then
new_state="成功"
else
new_state=$state
fi
echo -e "Elasticsearch数据备份通知:" > $log_file
echo "产品线: 广汽合创" | tee -a $log_file
echo "日志类型: Business" | tee -a $log_file
echo "快照名称: ${snapshot}" | tee -a $log_file
echo "备份状态:${new_state}" | tee -a $log_file
echo "开始时间: ${start_time}" | tee -a $log_file
echo "结束时间: ${end_time}" | tee -a $log_file
echo "持续时间: $[duration_in_millis/1000]秒" | tee -a $log_file
msg=`cat $log_file`
if [[ $snapshotname == $snapshot ]];
then
notity_phone=""
else
echo "get snapshot var error"
fi
/usr/local/src/dingding.sh "$webhook_url" "$msg" "$notity_phone"
fi
dingding.sh 钉钉告警
#!/bin/bash
if [ $# -eq 3 ]; then
webhook_url=$1
content=$2
atMobiles=$3
curl -d "{
\"msgtype\": \"text\",
\"text\":
{\"content\": \"${content}\"},
\"at\":
{
\"atMobiles\": [\"${atMobiles}\"],
isAtAll: false
}
}" -H 'Content-Type: application/json' "$webhook_url"
else
content=$1
atMobiles=$2
curl -d "{
\"msgtype\": \"text\",
\"text\":
{\"content\": \"${content}\"},
\"at\":
{
\"atMobiles\": [\"${atMobiles}\"],
isAtAll: false
}
}" -H 'Content-Type: application/json' "$webhook_url"
fi