Logstash同步sql数据到Es
- 安装好
ElasticSearch
和Logstash
- 把
mysql
驱动放到Logstash
的bin
目录下新建的文件夹mysq-test
下 - 在
mysql-test
文件夹下面建立mysql.config
文件
input {
stdin {
}
jdbc {
# mysql jdbc connection string to our backup databse
jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/hr"
# the user we wish to excute our statement as
jdbc_user => "root"
jdbc_password => "0000"
# the path to our downloaded jdbc driver
jdbc_driver_library => "F:\Logstah\logstash-7.4.0\logstash-7.4.0\bin\mysql-test\mysql-connector-java-5.1.47.jar"
# the name of the driver class for mysql
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement_filepath => "F:\Logstah\logstash-7.4.0\logstash-7.4.0\bin\mysql-test\mysql.sql"
schedule => "* * * * *" #这是进行更新的定时任务计划,有增量更新和全量更新
type => "jdbc"
}
}
filter {
json {
source => "message"
remove_field => ["message"]
}
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"] #ES的IP和端口
index => "mysql1" #导入后ES中的节点名称,类似数据库名称
document_id => "%{id}"
}
stdout {
codec => json_lines
}
}
select * from employee
logstash -f mysql-test\mysql.conf
- 到
kibana
界面查看索引