标题elasticsearch使用logstash同步多条sql语句mysql 多表的数据同步

标题elasticsearch使用logstash同步多条sql语句mysql 多表的数据同步

input {
        stdin {
        }
        jdbc {
         #索引类型,下面做相应多判断
         type => "mi_product"
         #jdbc连接数据库
         jdbc_connection_string => "jdbc:mysql://112.126.101.170:3306/mi?characterEncoding=UTF-8&useSSL=false&autoReconnect=true"
         #数据库用户
         jdbc_user => "root"
         #数据库密码
         jdbc_password => "123456"
		 #连接数据库jar包的绝对路径
         jdbc_driver_library => "/usr/local/logstash-6.3.0/lib/jars/mysql-connector-java-5.1.43.jar"
		 #驱动
         jdbc_driver_class => "com.mysql.jdbc.Driver"
         jdbc_paging_enabled => "true"
         jdbc_page_size => "50000"
         codec => plain { charset => "UTF-8"}
 
         tracking_column => update_time
         record_last_run => true
		 #lastrun文件夹下的文件
         last_run_metadata_path => "/usr/local/logstash-6.3.0/lastrun/.logstash_jdbc_last_run"
		 #时区为上海 
         jdbc_default_timezone => "Asia/Shanghai"
		 #bin文件夹下的sql文件
		 statement_filepath => "/usr/local/logstash-6.3.0/bin/jdbc.sql"
         clean_run => false
		 schedule => "* * * * *"
		 }
                jdbc {
                type => "mi_product_classification"
         jdbc_connection_string => "jdbc:mysql://112.126.101.170:3306/mi?characterEncoding=UTF-8&useSSL=false&autoReconnect=true"
         jdbc_user => "root"
         jdbc_password => "123456"

         jdbc_driver_library => "/usr/local/logstash-6.3.0/lib/jars/mysql-connector-java-5.1.43.jar"

                 jdbc_driver_class => "com.mysql.jdbc.Driver"

                 jdbc_paging_enabled => "true"
                 jdbc_page_size => "50000"
                 codec => plain { charset => "UTF-8"}


         tracking_column => update_time
                 record_last_run => true

                 last_run_metadata_path => "/usr/local/logstash-6.3.0/lastrun/.logstash_jdbc_last_run"

                 jdbc_default_timezone => "Asia/Shanghai"


                 statement_filepath => "/usr/local/logstash-6.3.0/bin/jdbc1.sql"


         clean_run => false


                        schedule => "* * * * *"

                  }
                jdbc {
                type => "mi_product_search_criteria"
         jdbc_connection_string => "jdbc:mysql://112.126.101.170:3306/mi?characterEncoding=UTF-8&useSSL=false&autoReconnect=true"
         jdbc_user => "root"
         jdbc_password => "123456"

         jdbc_driver_library => "/usr/local/logstash-6.3.0/lib/jars/mysql-connector-java-5.1.43.jar"

                 jdbc_driver_class => "com.mysql.jdbc.Driver"

                 jdbc_paging_enabled => "true"
                 jdbc_page_size => "50000"
                 codec => plain { charset => "UTF-8"}


         tracking_column => update_time
                 record_last_run => true

                 last_run_metadata_path => "/usr/local/logstash-6.3.0/lastrun/.logstash_jdbc_last_run"

                 jdbc_default_timezone => "Asia/Shanghai"


                 statement_filepath => "/usr/local/logstash-6.3.0/bin/jdbc2.sql"


         clean_run => false


                        schedule => "* * * * *"
                  }
          }

          filter {

                 json {

                         source => "message"

                         remove_field => ["message"]

                 }

         }


        output {
        #对应上面指定的type
        if[type]=="mi_product"{
     elasticsearch {
		 #es的路径
		 #配置集群 hosts => ["192.168.1.1:9200", "192.168.1.2:9200", "192.168.1.3:9200"]
         hosts => "192.168.100.5:9200"
         #索引名
         index => "mi"
		 #使用唯一的id
         document_id => "%{id}"
                }
        }
        if[type]=="mi_product_classification"{
        elasticsearch {



         hosts => "192.168.100.5:9200"



         index => "miclassification"



         document_id => "%{id}"
                }
        }
        if[type]=="mi_product_search_criteria"{
        elasticsearch {



         hosts => "192.168.100.5:9200"



         index => "misearchcriteria"



         document_id => "%{id}"
                }
        }
     stdout {



         codec => json_lines

     }
 }
发布了2 篇原创文章 · 获赞 0 · 访问量 98
展开阅读全文

没有更多推荐了,返回首页

©️2019 CSDN 皮肤主题: 数字20 设计师: CSDN官方博客

分享到微信朋友圈

×

扫一扫,手机浏览