ACK搭建ES并恢复阿里云ES实例备份

最新推荐文章于 2024-10-09 18:27:18 发布

次顿刚刚刚

最新推荐文章于 2024-10-09 18:27:18 发布

阅读量287

点赞数 9

文章标签： elasticsearch 阿里云 jenkins

本文链接：https://blog.csdn.net/wwwwww_1/article/details/140718259

版权

阿里ES恢复到自建ES
概述
创建OSS
用于阿里云ES实例进行snapshot备份
供自建ES进行恢复备份使用
自建ES实例
需要安装阿里云ES-oss插件
备份阿里云ES实例数据到1创建的oss中
自建ES关联1中的oss仓库
恢复备份的数据到自建ES中

创建OSS
	OSS创建参考 阿里云操作
	为ES实例创建可用于外部使用的仓库

PUT _snapshot/thingx_restore/ # 仓库名称
{
“type”: “oss”,
“settings”: {
“endpoint”: “https://oss-cn-beijing-internal.aliyuncs.com”, # 上步中创建的 oss 内网访问地址
“access_key_id”: “<access_key_id>”, # 账号ak
“secret_access_key”: “<secret_access_key>”, # 账号sk
“bucket”: “uino-sre-test”, # bucket 名称
“compress”: true, # 是否压缩
“chunk_size”: “500mb”, # 传输速率
“base_path”: “es-recover/” # 仓库路径
}
}

•	自动备份到ES数据到仓库【以下命令作为计划任务执行】

curl -u USERNAME:PASSWORD -X PUT http://es00jtt1v.elasticsearch.aliyuncs.com:9200/snapshot/thingx_restore/thingjsx-snapshot$(date ‘+%Y%m%d’)

2. 自建ES
•	使用elasticsearch官网镜像，版本和生产尽量保持一致
•	对镜像安装阿里云es-oss插件
# dockerfile, 插件选择和ES版本对应  插件地址：https://github.com/aliyun/elasticsearch-repository-oss/releases
FROM elasticsearch:6.8.6
COPY elasticsearch-repository-oss /usr/share/elasticsearch/plugins/elasticsearch-repository-oss  
•	上传自有镜像库
•	ACK创建ES并关联snapshot仓库到阿里云的OSS中，执行阿里云ES实例创建snapshot时的命令
PUT _snapshot/thingx_restore/   # 仓库名称
{
   "type": "oss",
   "settings": {
       "endpoint": "https://oss-cn-beijing-internal.aliyuncs.com",	# oss 内网访问地址
       "access_key_id": "<access_key_id>",						# 账号ak
       "secret_access_key": "<secret_access_key>",			# 账号sk
       "bucket": "uino-sre-test",								# bucket 名称
       "compress": true,											# 是否压缩
       "chunk_size": "500mb",									# 传输速率
       "base_path": "es-recover/"								# 仓库路径
   }
}
•	恢复数据到ES中
# 恢复数据到自建ES,恢复所有索引（除.开头的系统索引） 
curl -XPOST -H 'Content-Type:application/json' ES_POD_IP:9200/_snapshot/thingx_restore/thingjsx-snapshot_$(date '+%Y%m%d')/_restore?pretty -d '
    {"indices":"*,-.monitoring*,-.security_audit*","ignore_unavailable":"true"}'
 # 恢复单独索引
    curl -H 'Content-Type:application/json' -XPOST ES_POD_IP:9200/_snapshot/thingx_restore/thingjsx-snapshot_20230303/_restore -d '
    {
        "indices": "mon_eap_event_all_202303",
        "ignore_index_settings": [
            "index.apack.cube.following_index"
        ]
    }'
•	ack配置ES相关yaml文件
# ES 配置文件 6.8.6
apiVersion: v1
kind: ConfigMap
metadata:
  name: es-config
  namespace: restore-thingjsx-v4
data:
  elasticsearch.yml: |
    cluster.name: "uino-test-elasticsearch"
    node.name: "${POD_NAME}"
    network.host: 0.0.0.0
    discovery.zen.ping.unicast.hosts: "es-svc-headless"
    discovery.zen.minimum_master_nodes: 2
    path.repo: [""]
---   # elasticseaarch pv配置，使用SIT环境的NAS
apiVersion: v1
kind: PersistentVolume
metadata:
  name: elasticsearch-pv1
spec:
  capacity:
    storage: 200Gi
  accessModes:
    - ReadWriteOnce
  storageClassName: elasticsearch-sc
  mountOptions:
    - vers=3,nolock,proto=tcp,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
  nfs:
    server: 123e61-hgn93.cn-beijing.nas.aliyuncs.com
    path: /data/elasticsearch/data01

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: elasticsearch-pv2
spec:
  capacity:
    storage: 200Gi
  accessModes:
    - ReadWriteOnce
  storageClassName: elasticsearch-sc
  mountOptions:
    - vers=3,nolock,proto=tcp,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
  nfs:
    server: 123b61-hgn93.cn-beijing.nas.aliyuncs.com
    path: /data/elasticsearch/data02

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: elasticsearch-pv3
spec:
  capacity:
    storage: 200Gi
  accessModes:
    - ReadWriteOnce
  storageClassName: elasticsearch-sc
  mountOptions:
    - vers=3,nolock,proto=tcp,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,noresvport
  nfs:
    server: 123b61-hgn93.cn-beijing.nas.aliyuncs.com
    path: /data/elasticsearch/data03

--- # service 用户数据传输
apiVersion: v1
kind: Service
metadata:
  name: es-svc-headless
  namespace: restore-thingjsx-v4
  labels:
    k8s-app: elasticsearch
spec:
  selector:
    k8s-app: elasticsearch
  clusterIP: None
  ports:
  - name: in
    port: 9300
    protocol: TCP

--- # service 用户http访问
apiVersion: v1
kind: Service
metadata:
  name: es-svc
  namespace: restore-thingjsx-v4
  labels:
    k8s-app: elasticsearch
spec:
  selector:
    k8s-app: elasticsearch
  ports:
  - name: out
    port: 9200
    protocol: TCP

--- # ES 容器配置
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: elasticsearch
  namespace: restore-thingjsx-v4
  labels:
    k8s-app: elasticsearch
spec:
  replicas: 3
  serviceName: es-svc-headless
  selector:
    matchLabels:
      k8s-app: elasticsearch
  template:
    metadata:
      labels:
        k8s-app: elasticsearch
    spec:
      initContainers:
      - command:
        - /sbin/sysctl
        - -w
        - vm.max_map_count=262144
        image: alpine:3.6
        imagePullPolicy: IfNotPresent
        name: elasticsearch-init
        resources: {}
        securityContext:
          privileged: true
      - name: fix-permissions
        image: alpine:3.6
        command: ["sh", "-c", "chown -R 1000:1000 /usr/share/elasticsearch/data"]
        securityContext:
          privileged: true
        volumeMounts:
        - name: es-data-volume
          mountPath: /usr/share/elasticsearch/data
      containers:
      - name: elasticsearch
        image: dk.uino.cn/elastic/elasticsearch-ali-oss:6.8.6
        imagePullPolicy: IfNotPresent
        env:
          - name: POD_NAME
            valueFrom:
              fieldRef:
                fieldPath: metadata.name
        resources:
          limits:
            cpu: '1'
            memory: 2Gi
          requests:
            cpu: '1'
            memory: 2Gi
        ports:
        - containerPort: 9200
          name: db
          protocol: TCP
        - containerPort: 9300
          name: transport
          protocol: TCP
        volumeMounts:
          - name: es-config-volume
            mountPath: /usr/share/elasticsearch/config/elasticsearch.yml
            subPath: elasticsearch.yml
          - name: es-data-volume
            mountPath: /usr/share/elasticsearch/data
      volumes:
        - name: es-config-volume
          configMap:
            name: es-config
            items:
            - key: elasticsearch.yml
              path: elasticsearch.yml
  volumeClaimTemplates:
  - metadata:
      name: es-data-volume
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: "elasticsearch-sc"
      resources:
        requests:
          storage: 200Gi

--- # kibana app
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kibana
  namespace: restore-thingjsx-v4
  labels:
    app: kibana
spec:
  selector:
    matchLabels:
      app: "kibana"
  template:
    metadata:
      labels:
        app: kibana
    spec:
      containers:
      - name: kibana
        image: kibana:6.8.6
        resources:
          limits:
            cpu: 1000m
          requests:
            cpu: 100m
        env:
          - name: ELASTICSEARCH_HOSTS
            value: http://es-svc:9200
          - name: SERVER_NAME
            value: kibana-logging
          - name: SERVER_REWRITEBASEPATH
            value: "false"
        ports:
        - containerPort: 5601
        volumeMounts:
          - name: config
            mountPath: /usr/share/kibana/config/
      volumes:
        - name: config
          configMap:
            name: kibana-config

--- # kibana config file
apiVersion: v1
kind: ConfigMap
metadata:
  name: kibana-config
  namespace: restore-thingjsx-v4
data:
  kibana.yml: |-
    elasticsearch.requestTimeout: 90000
    server.host: "0"
    xpack.monitoring.ui.container.elasticsearch.enabled: true

--- # kibana service
apiVersion: v1
kind: Service
metadata:
  name: kibana
  namespace: restore-thingjsx-v4
  labels:
    app: kibana
spec:
  ports:
  - port: 5601
    protocol: TCP
    targetPort: 5601
  type: ClusterIP
  selector:
    app: kibana

--- # kibana ingress kong
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: kibana
  namespace: restore-thingjsx-v4
spec:
  ingressClassName: kong
  rules:
  - host: kibana-recover-thingjsx.3dlink.cn
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service: 
            name: kibana
            port:
              number: 5601