06.德国博士练习_08_query_dsl

1. exercise01: query size,from,should,highlight,pagination,sort

# ** EXAM OBJECTIVE: QUERIES **
# GOAL: Create search queries for analyzed text, highlight,
# pagination, and sort# REQUIRED SETUP:
# (i) a running Elasticsearch cluster with at least one node and
# a Kibana instance,
# (ii) add the "Sample web logs" and "Sample eCommerce orders" to
# Kibana

# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# string "Firefox"


GET kibana_sample_data_logs/_search
{
  "query": {
    "match": {
      "message": "Firefox"
    }
  }
}

# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# string "Firefox" and return (up to) 50 results.
# As above, but return up to 50 results with an offset of 50 from
# the first




GET kibana_sample_data_logs/_search
{
  "query": {
    "match": {
      "message": "Firefox"
    }
  },
  "from": 50, 
  "size": 50
}

# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# strings "Firefox" or "Kibana"


GET kibana_sample_data_logs/_search
{
  "query": {
  "match": {
    "message": "Firefox Kibana"
  }
  }
}

# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing both the
# strings "Firefox" and "Kibana"
GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "message": "Firefox"
          }
        },
        {
          "match": {
            "message": "Kibana"
          }
        }
      ]
    }
  }
}



# Search for documents with the `message` field containing at least
# two of the following strings: "Firefox", "Kibana",
# "159.64.35.129"
GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "match": {
            "message": "Firefox"
          }
        },
        {
          "match": {
            "message": "Kibana"
          }
        },
        {
          "match": {
            "message": "159.64.35.129"
          }
        }
      ],
      "minimum_should_match": 2
    }
  }
}

# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# strings "Firefox" or "Kibana"
# As above, but also return the highlights for the `message` field
# As above, but also wrap the highlights in "{{" and "}}"

GET kibana_sample_data_logs/_search
{
  "query": {
    "match": {
      "message": "Firefox Kibana"
    }
  },
  "highlight": {
    "fields": {
      "message": {"pre_tags": "{{","post_tags": "}}"}
    }
  }
}


# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# phrase "HTTP/1.1 200 51"

GET kibana_sample_data_logs/_search
{
  "query": {
    "match_phrase": {
      "message": "HTTP/1.1 200 51"
    }
  }
}


# Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `message` field containing the
# phrase "HTTP/1.1 200 51", and sort the results by the
# `machine.os` field in descending order
# As above, but also sort the results by the `timestamp` field in
# ascending order

GET kibana_sample_data_logs/_search
{
  "query": {
    "match_phrase": {
      "message": "HTTP/1.1 200 51"
    }
  },
  "sort": [
    {
      "machine.os.keyword": {
        "order": "desc"
      }
    },
    {
      "timestamp": {
        "order": "asc"
      }
    }
  ]
}

### Run the next queries on the `kibana_sample_data_ecommerce` index
# Search for documents with the `day_of_week` field containing the
# string "Monday"
# As above, but sort the results by the `products.base_price` field 
#in descending order, picking the lowest value of the array

GET kibana_sample_data_ecommerce/_mapping
GET kibana_sample_data_ecommerce/_search
{
  "query": {
    "match": {
      "day_of_week": "Monday"
    }
  },
  "sort": [
    {
      "products.base_price": {
        "order": "desc",
        "mode": "min"
      }
    }
  ]
}

2. exercise02: term query,compound query, fuzzy query, date query

# ** EXAM OBJECTIVE: QUERIES **
# GOAL: Create search queries for terms, numbers, dates, fuzzy, and
# compound queries
# REQUIRED SETUP:
# (i) a running Elasticsearch cluster with at least one node and a
# Kibana instance,
# (ii) add the "Sample web logs" and "Sample flight data" to Kibana###



#Run the next queries on the `kibana_sample_data_logs` index
# Filter documents with the `response` field greater or equal to 400
# and less than 500
# As above, but add a second filter for documents with the `referer`
# field matching "http://twitter.com/success/guion-bluford"

filter是可以使用数组的奥, referer是keyword类型的
GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "filter": [
        {
          "range": {
            "response": {
              "gte": 400,
              "lt": 500
            }
          }
        },
        {
          "term": {
            "referer": "http://twitter.com/success/guion-bluford"
          }
        }
      ]
    }
  }
}




#Run the next queries on the `kibana_sample_data_logs` index
# Filter documents with the `referer` field that starts by
# "http://twitter.com/success"
# Filter documents with the `request` field that starts by "/people"


这里忘了term级别的前缀匹配使用prefix即可，在term level查询中有
GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "filter": [
        {
          "prefix": {
            "referer": "http://twitter.com/success"
          }
        },
        {
          "match_phrase_prefix": {
            "request": "/people"
          }
        }
      ]
    }
  }
}


#Run the next queries on the `kibana_sample_data_logs` index
# Filter documents with the `memory` field containing any indexed
# value
# (opposite of above) Filter documents with the `memory` field not
# containing any indexed value

GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "filter": {
        "exists": {
          "field": "memory"
        }
      }
    }
  }
}


GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "exists": {
            "field": "memory"
          }
        }
      ]
    }
  }
}

#Run the next queries on the `kibana_sample_data_logs` index
# Search for documents with the `agent` field containing the string
# "Windows" and the `url` field containing the string "name:john"
# As above, but also filter documents with the `phpmemory` field
# containing any indexed value

GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "match": {
            "agent": "Windows"
          }
        },
        {
          "match_phrase": {
            "url": "name:john"
          }
        }
      ],
      "filter": {
        "exists": {
          "field": "phpmemory"
        }
      }
    }
  }
}



# Search for documents that have either the `response` field greater
# or equal to 400 or the `tags` field having the string "error"
GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "range": {
            "response": {
              "gte": 40
            }
          }
        },
        {
          "match": {
            "tags": "error"
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}


# Search for documents with the `tags` field that does not contain
# any of the following strings: "warning", "error", "info"
GET kibana_sample_data_logs/_mapping

GET kibana_sample_data_logs/_search
{
  "query": {
    "bool": {
      "must_not": [
        {
          "match": {
            "tags": "warning, error, info"
          }
        }
      ]
    }
  }
}



#Run the next queries on the `kibana_sample_data_logs` index
# Filter documents with the `timestamp` field containing a date
# between today and one week ago

这个date的处理还需要学习
GET kibana_sample_data_logs/_search
{
  "query": {
    "range": {
      "timestamp": {
        "gte": "now-1w/d",
        "lte": "now/d"
      }
    }
  },
  "sort": [
    {
      "timestamp": {
        "order": "asc"
      }
    }
  ]
}






### Run the next queries on the `kibana_sample_data_flights` index
# Filter documents with either the `OriginCityName` or the
# `DestCityName` fields matching the string "Sydney"
# As above, but allow inexact fuzzy matching, with a maximum allowed
# “Levenshtein Edit Distance” set to 2. Test that the query
# strings "Sydney", "Sidney" and "Sidnei" always return the same
# number of results

GET kibana_sample_data_flights/_mapping
GET kibana_sample_data_flights/_search
{
  "query": {
    "bool": {
      "should": [
        {
          "match": {
            "OriginCityName": {
              "query": "Sidnei",
              "fuzziness": 2
            }
          }
        },
        {
          "match": {
            "DestCityName": {
              "query": "Sidnei",
              "fuzziness": 2
            }
          }
        }
      ],
      "minimum_should_match": 1
    }
  }
}

第二种解法
GET kibana_sample_data_flights/_search
{
  "query": {
    "multi_match": {
      "query": "Sidnei",
      "fields": ["OriginCityName","DestCityName"],
      "fuzziness": 2,
      "type": "best_fields"
    }
  }
}

3. exercise03: search template, script query, scroll query.

这里的template的使用还是要熟悉一下才行，update ， query_template在使用stored的script的方式

# ** EXAM OBJECTIVE: QUERIES **
# GOAL: Use scroll API, search templates, script queries
# REQUIRED SETUP:
# (i) a running Elasticsearch cluster with at least one node and a
# Kibana instance,
# (ii) add the "Sample web logs" and "Sample flight data" to Kibana


# Search for all documents in all indices
# As above, but use the scroll API to return the first 100 results
# while keeping the search context alive for 2 minutes
# Use the scroll id included in the response to the previous query
# and retrieve the next batch of results

POST /_search?scroll=1m
{
  "size": 100,
  "query": {"match_all": {}}
}

POST /_search/scroll 
{
    "scroll" : "1m", 
    "scroll_id" : "DnF1ZXJ5VGhlbkZldGNoBwAAAAAAAZFPFnZsLWtJUW0yU2d5Nl9POVM1dXVsTFEAAAAAAAGRUBZ2bC1rSVFtMlNneTZfTzlTNXV1bExRAAAAAAABkVEWdmwta0lRbTJTZ3k2X085UzV1dWxMUQAAAAAAAZFTFnZsLWtJUW0yU2d5Nl9POVM1dXVsTFEAAAAAAAGRUhZ2bC1rSVFtMlNneTZfTzlTNXV1bExRAAAAAAABkVQWdmwta0lRbTJTZ3k2X085UzV1dWxMUQAAAAAAAZFVFnZsLWtJUW0yU2d5Nl9POVM1dXVsTFE="
}


### Run the next queries on the `kibana_sample_data_logs` index
# Filter documents with the `response` field greater or equal to 400
这里可能有问题，因为这个字段被设置为string类型了，是不是要用filter,script来处理了。
GET kibana_sample_data_logs/_search
{
  "query": {
    "range": {
      "response.keyword": {
        "gte": 400
      }
    }
  }
}

# Create a search template for the above query, so that the template
# (i) is named "with_response_and_tag", 
# (ii) has a parameter "with_min_response" to represent the lower bound of the `response` field, 
# (iii) has a parameter "with_max_response" to represent the upper bound of the `response` field, 
# (iv) has a parameter "with_tag" to represent a possible value of the `tags`  field

# Test the "with_response_and_tag" search template by setting the
# parameters as follows: 
# (i) "with_min_response": 400, 
# (ii) "with_max_response": 500 
# (iii) "with_tag": "security"

先写query
{
  "template_output" : {
    "query" : {
      "bool" : {
        "must" : [
          {
            "range" : {
              "response" : {
                "gte" : "400",
                "lte" : "500"
              }
            }
          },
          {
            "match" : {
              "tags" : "security"
            }
          }
        ]
      }
    }
  }
}

根据query写script

PUT _scripts/with_response_and_tag
{
  "script": {
    "lang": "mustache",
    "source": {
      "query": {
        "bool": {
          "must": [
            {
              "range": {
                "response": {
                  "gte": "{{with_min_response}}", #这两个值会变成字符串的形式，理论上是有问题的，不能这样做的。正确的做法是将整个source中的json做成一个转义后的字符串才行。
                  "lte": "{{with_max_response}}"
                }
              }
            },
            {
              "match": {
                "tags": "{{with_tag}}"
              }
            }
          ]
        }
      }
    }
  }
}


GET kibana_sample_data_logs/_search/template
{
  "id": "with_response_and_tag",
  "params": {
    "with_min_response": 400,
    "with_max_response": 500,
    "with_tag": "security"
  }
}

GET _render/template/with_response_and_tag
{
  "params": {
    "with_min_response": 400,
    "with_max_response": 500,
    "with_tag": "security"
  }
}

返回
{
  "template_output" : {
    "query" : {
      "bool" : {
        "must" : [
          {
            "range" : {
              "response" : {
                "gte" : "400",
                "lte" : "500"
              }
            }
          },
          {
            "match" : {
              "tags" : "security"
            }
          }
        ]
      }
    }
  }
}


更好的template应该写成(使用kibana的特殊展示功能，挺好的)
PUT _scripts/with_response_and_tag05
{
  "script": {
    "lang": "mustache",
    "source":"""{
      "query": {
        "bool": {
          "must": [
            {
              "range": {
                "response": {
                  "gte": {{with_min_response}}, 
                  "lte": {{with_max_response}}
                }
              }
            },
            {
              "match": {
                "tags": "{{with_tag}}"
              }
            }
          ]
        }
      }
    }
    """
  }
}

# Update the "with_response_and_tag" search template, so that (i) if
# the "with_max_response" parameter is not set, then don't set an #
upper bound to the `response` value, and (ii) if the "with_tag"
# parameter is not set, then do not apply that filter at all
# Test the "with_response_and_tag" search template by setting only
# the "with_min_response" parameter to 500
# Test the "with_response_and_tag" search template by setting the
# parameters as follows: (i) "with_min_response": 500, (ii)
# "with_tag": "security"

先写query
{
  "template_output" : {
    "query" : {
      "bool" : {
        "must" : [
          {
            "range" : {
              "response" : {
                "gte" : "400",
                "lte" : "500"
              }
            }
          },
          {
            "match" : {
              "tags" : "security"
            }
          }
        ]
      }
    }
  }
}


然后使用mustache的条件处理逻辑

POST _scripts/with_response_and_tag03
{
  "script": {
    "lang": "mustache",
    "source": """
    {
      "query": {
        "bool": {
          "must": [
            {
              "range": {
                "response": {
                  "gte": {{with_min_response}}{{#with_max_response}},
                  "lte": {{with_max_response}}{{/with_max_response}}
                }
              }
            }{{#with_tag}},
            {
              "match": {
                "tags": "{{with_tag}}"
              }
            }{{/with_tag}}
          ]
        }
      }
    }
"""
  }
}



可以把后面两个参数去掉进行测试
GET _render/template/with_response_and_tag03
{
  "params": {
    "with_min_response": 400,
    "with_max_response": 500,
    "with_tag": "security"
  }
}