logstash同步父子文档

父子文档，可以理解为关系型数据库中的一对多的关系。使用logstash同步MySQL数据，有时候需要同步父子文档。

使用ES父子文档功能需要满足两个条件

• 指定一个type是另一个type的父
• 在存储子文档的时候，通过parent参数指定父文档的Id

第一步：定义父子关系

curl -XPUT "http://localhost:9200/product?pretty" -d '

{
  "mappings": { 
    "service_judge" : { }, 
    "service_judge_detail":{ 
      "_parent": { 
        "type": "service_judge"}}
}
复制代码

第二步：配置文件

input{
     jdbc {
         jdbc_driver_library => "/some/config-dir/mysql-connector-java-5.1.46-bin.jar"
         jdbc_driver_class => "com.mysql.jdbc.Driver"
         jdbc_connection_string => "jdbc:mysql://localhost:3306/product"
         jdbc_user => "user"
         jdbc_password => "123456"
         jdbc_paging_enabled => "true"
         jdbc_page_size => "1000"
         jdbc_default_timezone =>"Asia/Shanghai"
         schedule => "* * * * *"
         statement => "SELECT * FROM  service_judge WHERE `status` = 1"
         type => "service_judge"
       }
       jdbc {
         jdbc_driver_library => "/some/config-dir/mysql-connector-java-5.1.46-bin.jar"
         jdbc_driver_class => "com.mysql.jdbc.Driver"
         jdbc_connection_string => "jdbc:mysql://localhost:3306/product"
         jdbc_user => "user"
         jdbc_password => "123456"
         jdbc_paging_enabled => "true"
         jdbc_page_size => "1000"
         jdbc_default_timezone =>"Asia/Shanghai"
         schedule => "* * * * *"
         statement => "SELECT * FROM  service_judge_detail WHERE `status` = 1"
         type => "service_judge_detail"
       } 
} 
filter {
    json {
        source => "message"
        remove_field => ["message"]
    }
}
output{
      stdout {
        codec => json_lines
      }
      if[type] == "service_judge"{
      elasticsearch {
         hosts => "localhost:9200"
         user => "elastic"
         password => "123456"
         index => "service_judge"
         document_type => "service_judge"
         document_id => "%{id}"
      }
      }
      if[type] == "service_judge_detail"{
      elasticsearch {
         hosts => "localhost:9200"
         user => "elastic"
         password => "123456"
         index => "service_judge"
         document_type => "service_judge_detail"
         document_id => "%{id}"
         # 这里通过parent指定父文档：参数为子表中存的父表的id
         parent => "%{service_judge_id}"
      }
      }

 } 
复制代码

查询验证

查询：评价标签带有非常差的评价

GET product/service_judge/_search
{
  "query": {
    "has_child": {
      "type": "service_judge_detail",
      "query": {
        "match": {
          "content": "非常差"
        }
      }
    }
  }
}

复制代码

结果：

{
  "took": 9,
  "timed_out": false,
  "_shards": {
    "total": 5,
    "successful": 5,
    "failed": 0
  },
  "hits": {
    "total": 1,
    "max_score": 1,
    "hits": [
      {
        "_index": "service_judge",
        "_type": "service_judge",
        "_id": "2c94bf8362ffe67d016300f3b2ca0001",
        "_score": 1,
        "_source": {
          "@timestamp": "2018-05-07T10:40:00.076Z",
          "service_id": "4028b8815ca0a5ca015ca0a67ad90000",
          "name": "衣柜送货管家评价",
          "@version": "1",
          "memo": "评价33444",
          "addname": "1",
          "id": "2c94bf8362ffe67d016300f3b2ca0001",
          "target_role": "003",
          "type": "service_judge",
          "value": null,
          "status": "1"
        }
      }
    ]
  }
}
复制代码