[elasticsearch笔记]Rolling up historical data

note

• this functionality is experimental
• 时间相关性的数据随着时间的推移，其实时可访问的诉求会逐步降低，直到基本没有访问诉求
• 所以超过一定时间的数据 rolling up 到低成本的存储设备/存储方案，是很有必要的
• /_rollup_search 和 /_search 可接收的 Query DSL 一样，只是支持功能上有些限制
• 支持 live data 和 rolled data 搜索结果 merge。当 live 和 rolled 都有数据时，选择 live 数据，提高准确性。/_rollup_search 搜索的结果就会包含live data，不需要特别处理
• 在设计 _rollup 任务时，要考虑未来如何使用数据，如果做数据聚合。根据未来诉求，设计 _rollup。否则很多类型或者字段的搜索、聚合、聚合的力度会不支持

demo

DELETE sensor*

PUT sensor-info
{
  "mappings": {
    "properties": {
      "node": {
        "type": "keyword"
      },
      "temperature": {
        "type": "long"
      },
      "timestamp": {
        "type": "date"
      },
      "voltage": {
        "type": "float"
      }
    }
  }
}

PUT sensor-info/_doc/1
{
  "timestamp": 1516729294000,
  "temperature": 200,
  "voltage": 5.2,
  "node": "a"
}

PUT sensor-product
{
  "mappings": {
    "properties": {
      "title": {
        "type": "keyword"
      },
      "price": {
        "type": "long"
      },
      "timestamp": {
        "type": "date"
      }
    }
  }
}

PUT sensor-product/_doc/1
{
  "timestamp": 1516729294000,
  "price": 123,
  "title": "Foo"
}

POST _rollup/job/sensor/_stop
DELETE _rollup/job/sensor
PUT _rollup/job/sensor
{
  "index_pattern": "sensor-*",
  "rollup_index": "sensor_rollup1",
  "cron": "*/30 * * * * ?",
  "page_size": 1000,
  "groups": {
    "date_histogram": {
      "field": "timestamp",
      "fixed_interval": "60m"
    },
    "terms": {
      "fields": [
        "node"
      ]
    }
  },
  "metrics": [
    {
      "field": "temperature",
      "metrics": [
        "min",
        "max",
        "sum"
      ]
    },
    {
      "field": "voltage",
      "metrics": [
        "avg"
      ]
    }
  ]
}

DELETE _rollup/job/combined
PUT _rollup/job/combined
{
    "index_pattern": "sensor-*",
    "rollup_index": "sensor_rollup2",
    "cron": "*/30 * * * * ?",
    "page_size" :1000,
    "groups" : {
      "date_histogram": {
        "field": "timestamp",
        "fixed_interval": "1h",
        "delay": "7d"
      },
      "terms": {
        "fields": ["node", "title"]
      }
    },
    "metrics": [
        {
            "field": "temperature",
            "metrics": ["min", "max", "sum"]
        },
        {
            "field": "price",
            "metrics": ["avg"]
        }
    ]
}

POST _rollup/job/sensor/_start
POST _rollup/job/combined/_start

GET /sensor_rollup1/_rollup_search
{
  "size": 0,
  "aggregations": {
    "max_temperature": {
      "max": {
        "field": "temperature"
      }
    }
  }
}

GET /sensor_rollup2/_rollup_search
{
  "size": 0,
  "aggregations": {
    "min_temperature": {
      "min": {
        "field": "temperature"
      }
    }
  }
}