Elasticsearch索引克隆还原修改创建数据

背景

        ES有一条索引security_log_2024.04.24，想以此索引为目标生成security_log_2023.12.24索引和数据。

导出mapping

elasticdump \
        --input=http://192.168.233.2:29200/security_log_2024.04.24 \
        --output=/tmp/es/security_log_2024.04.24-mapping.json \
        --type=mapping \
        --limit=10000 --concurrency=100 --concurrencyInterval=30 --intervalCap=50

导入mapping

elasticdump \
   --input=/tmp/es/mapping/mapping.json \
   --output=http://192.168.233.2:29200/security_log_2023.12.24 \
   --type=mapping \
   --limit=10000 --concurrency=100 --concurrencyInterval=30 --intervalCap=50

导出数据

elasticdump \
        --input=http://192.168.233.2:29200/security_log_2024.04.24 \
        --output=/tmp/es/security_log_2024.04.24-data.json \
		--searchBody='{"query": { "match_all": {} }, "stored_fields": ["*"], "_source": true }' \
        --limit=10000 --concurrency=100 --concurrencyInterval=30 --intervalCap=50

导入数据

elasticdump \
   --input=/tmp/es/security_log_2024.04.24-data.json \
   --output=http://192.168.233.2:29200/security_log_2023.12.24 \
   --type=data \
   --limit=10000 --concurrency=100 --concurrencyInterval=30 --intervalCap=50

修改创建时间字段

curl -X POST "http://192.168.233.2:29200/security_log_2023.12.18/_update_by_query"  -d  \
'{
  "script": {
    "inline": "ctx._source.create_time -= params.value",
    "params": {
      "value": 10540800701
    }
  },
  "query": {
    "match_all": {}
  }
}'

查询数据

curl -XGET http://192.168.233.2:29200/security_log_2023.12.24/_search -H 'Content-Type: application/json' -d '{"query":{"bool":{"must":[{"match_all":{}}],"must_not":[],"should":[]}},"from":0,"size":10,"sort":[],"aggs":{}}'

根据条件查询数据总条数

curl -X GET "http://172.16.6.81:29200/security_log_2024.02.22/_count" -H 'Content-Type: application/json' -d '{"query":{"bool":{"must":[{"term":{"eqpt_ip":"172.16.61.16"}}]}}}'

查询event_type字段值为以下枚举值其中一个，随机获取1000行

curl -X GET "http://172.16.6.11:29200/security_log_2024.02.22/_count" -H 'Content-Type: application/json' -d '
{
  "size": 1000,
  "query": {
    "function_score": {
      "query": {
        "terms": {
          "event_type": [
            "信息泄露",
            "SQL注入",
            "命令执行",
            "缓存溢出",
            "DOS攻击"
          ]
        }
      },
      "random_score": {}
    }
  }
}'

查询存在event_type字段，只输出create_time字段

{
  "query": {
    "exists": {
      "field": "event_type"
    }
  },
  "_source": ["create_time"],
  "size": 10,
  "from": 0,
  "sort": []
}

查询_id="d49ce03e"的数据

{
  "query": {
    "query_string": {
      "query": "_id:\"d49ce03e\""
    }
  },
  "size": 10,
  "from": 0,
  "sort": []
}

查询id字段以"0x"开头，action字段值为"pass"，event_type取枚举值的数据

{
  "query": {
    "bool": {
      "must": [
        {
          "prefix": {
            "id": "0x"
          }
        },
        {
          "match": {
            "action": "pass"
          }
        },
        {
          "terms": {
           "event_type": [
        "请求访问",
        "信息泄露",
        "SQL注入",
        "命令执行",
        "缓存溢出",
        "DOS攻击"
      ]
          }
        }
      ]
    }
  },
  "from": 0,
  "size": 10
}

删除2024-07-25 00:00:00-2024-07-26 00:00:00之间的数据

#2024-07-25 00:00:00   2024-07-26 00:00:00
#1721836800000         1721923200000
curl -X POST "http://172.31.125.12:29200/security_log_2024.07.26/_delete_by_query" -H 'Content-Type: application/json' -d'
{
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "create_time": {
              "gte": "1721836800000",
              "lte": "1721923200000"
            }
          }
        }
      ]
    }
  }
}'

更新#2024-07-28 00:00:00-2024-07-29 00:00:00之间的数据的create_time和idss_collect_time值，都减去2天

1天的时间戳
86400000

2天的时间戳
172800000

3天的时间戳
259200000

#2024-07-28 00:00:00   2024-07-29 00:00:00
#1722096000000         1722182400000
curl -X POST "http://172.31.125.11:29200/security_log_2024.07.26/_update_by_query"  -d  \
'{
  "script": {
    "inline": "ctx._source.create_time -= params.value;ctx._source.idss_collect_time -= params.value",
    "params": {
      "value": 172800000
    }
  },
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "create_time": {
              "gte": "1722096000000",
              "lte": "1722182400000"
            }
          }
        }
      ]
    }
  }
}'

更新索引security_log_2024.07.26字段raw_log值，把"<142>Jul 28"替换为"<142>Jul 26"

#更新索引security_log_2024.07.26字段raw_log值，把"<142>Jul 28"替换为"<142>Jul 26"
curl -X POST "http://172.31.125.12:29200/security_log_2024.07.26/_update_by_query" -H 'Content-Type: application/json' -d '
{
  "script": {
    "inline": "ctx._source.raw_log=ctx._source.raw_log.replace(params.oldString,params.newString);",
        "params": {"oldString":"Jul 28","newString":"Jul 26"}
  },
"query": {
    "bool": {
      "must": [
        {
          "range": {
            "create_time": {
              "gte": "1722167605000",
              "lte": "1722182400000"
            }
          }
        }
      ]
    }
  }
}'

导出#2024-07-28 20:12:30-2024-07-29 00:00:00之间的数据

#2024-07-28 20:12:30   2024-07-29 00:00:00
#1722168750000         1722182400000
elasticdump \
        --input=http://172.31.125.11:29200/security_log_2024.07.28 \
        --output=/es-log-bak/es/security_log_2024.07.28-25.json \
        --type=data \
                --searchBody='{"query":{"bool":{"must":[{"range":{"create_time":{"gte":"1722168750000","lte":"1722182400000"}}}],"must_not":[],"should":[]}},"from":0,"size":10,"sort":[],"aggs":{},"stored_fields":["*"],"_source":true}' \
        --limit=10000 --concurrency=100 --concurrencyInterval=30 --intervalCap=50

修改2024-07-28 20:12:30-2024-07-29 00:00:00之间的时间字段和raw_log字符串的日期

#2024-07-28 20:12:30   2024-07-29 00:00:00
#1722168750000         1722182400000
修改时间字段和raw_log的日期
curl -X POST "http://172.31.125.11:29200/security_log_2024.07.25/_update_by_query"  -d  \
'{
  "script": {
    "inline": "ctx._source.create_time -= params.value;ctx._source.idss_collect_time -= params.value;ctx._source.raw_log=ctx._source.raw_log.replace(params.oldString1,params.newString1);ctx._source.raw_log=ctx._source.raw_log.replace(params.oldString2,params.newString2);",
    "params": {
      "value": 259200000,"oldString1":"Jul 28","newString1":"Jul 25","oldString2":"Jul 27","newString2":"Jul 25"
    }
  },
  "query": {
    "bool": {
      "must": [
        {
          "range": {
            "create_time": {
              "gte": "1722168750000",
              "lte": "1722182400000"
            }
          }
        }
      ]
    }
  }
}'

故障解决方案

es-数据导出 elastice_dump 报错 parsing_exception The field [fields] is no longer supported-CSDN博客