[elasticsearch笔记]Managing the index lifecycle

note

• 索引的生命周期
  • hot
  • warm
  • cold
  • delete
• 如果 index.lifecycyle.indexing_complete=true, 索引不会通过 ilm policy rollover
• 应用 ilm policy 到已有index中有两个主要问题：
  • 不能有 rollover 相关设置，因为rollover是针对新建的index的
  • 避免一设置就进行 hot、warm、cold、delete相关policy，因为每个阶段的 min_age 都是比对索引的原始创建时间。 通过修改 min_age 规避这个问题。min_age的比对时间是上个阶段的起始时间，还是索引的创建时间？？？
• rollover 新建的索引不会继承旧索引的policy，不建议在建立索引的时候设置包含rollover的ilm policy。因为rollover后的索引不会应用ilm policy。采用定义模板的方式使用 ilm policy
• 索引生命周期中优先级越高，系统启动时，会越早恢复

demo

#
# hot: 数据量达到50G或者7天后 rollover。
# warm: 进入 host 30天后，进入 warm，同时设置 replica为2，强制 merge，shrink。
# cold: 进入 warm 60天后，进入cold阶段，分配到cold节点。
# delete: 进入 cold 90天后，删除索引。
#
PUT _ilm/policy/full_policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_age": "7d",
            "max_size": "50G",
            "max_docs": 100000000
          }
        }
      },
      "warm": {
        "min_age": "30d",
        "actions": {
          "forcemerge": {
            "max_num_segments": 1
          },
          "shrink": {
            "number_of_shards": 1
          },
          "allocate": {
            "number_of_replicas": 2
          }
        }
      },
      "cold": {
        "min_age": "60d",
        "actions": {
          "allocate": {
            "require": {
              "type": "cold"
            }
          }
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

#
# rollover 需要指定 index.lifecycle.rollover_alias
#
PUT _template/datastream_template
{
  "index_patterns": ["datastream-*"],                 
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "full_policy",      
    "index.lifecycle.rollover_alias": "datastream"    
  }
}

PUT datastream-000001
{
  "aliases": {
    "datastream": {
      "is_write_index": true
    }
  }
}

#
# rollover 新建的索引不会继承旧索引的policy
# 不建议在建立索引的时候设置包含rollover的ilm policy。因为rollover后的索引不会应用ilm policy。
# 采用定义模板的方式使用 ilm policy
#
PUT test-index
{
  "settings": {
    "number_of_shards": 1,
    "number_of_replicas": 1,
    "index.lifecycle.name": "full_policy"
  }
}

GET datastream-*/_ilm/explain

PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "warm": {
        "actions": {
          "allocate" : {
            "include" : {
              "box_type": "hot,warm"
            }
          }
        }
      }
    }
  }
}

PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "warm": {
        "actions": {
          "allocate" : {
            "number_of_replicas": 1,
            "require" : {
              "box_type": "cold"
            }
        }
        }
      }
    }
  }
}

PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "cold": {
        "actions": {
          "freeze" : { }
        }
      }
    }
  }
}

PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "warm": {
        "actions": {
          "readonly" : { }
        }
      }
    }
  }
}

PUT my_index
{
  "settings": {
    "index.lifecycle.name": "my_policy",
    "index.lifecycle.rollover_alias": "my_data"
  },
  "aliases": {
    "my_data": {
      "is_write_index": true
    }
  }
}

# 更新 index 的 ilm policy
PUT my_index/_settings
{
  "lifecycle.name": "my_other_policy"
}

#
# 优先级越高，系统启动时，会越早恢复
#
PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "warm": {
        "actions": {
          "set_priority" : {
            "priority": 50
          }
        }
      }
    }
  }
}

PUT _ilm/policy/my_policy
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "unfollow" : {}
        }
      }
    }
  }
}

#
# update policy
#
PUT _ilm/policy/policy_change_demo
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "25GB"
          }
        }
      },
      "delete": {
        "min_age": "30d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

PUT _ilm/policy/policy_change_demo
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "25GB"
          }
        }
      },
      "delete": {
        "min_age": "10d", 
        "actions": {
          "delete": {}
        }
      }
    }
  }
}

GET _ilm/status
POST _ilm/stop
POST _ilm/start