1.需求:采集同一个数据库中不同的表到同一个es服务的不同索引中
2.配置文件
input {
stdin {
}
#操作mysql取出数据
jdbc {
jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/数据库名称?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
#用户名
jdbc_user => "*****"
#密码
jdbc_password => "****"
jdbc_driver_library => "mysql数据库连接jar包位置"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
schedule => "* * * * *"
record_last_run => true
last_run_metadata_path => "采集记录的时间戳文件路径"
type => "表名1"
tags => "表名1"
}
jdbc {
jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/数据库名称?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
#用户名
jdbc_user => "*****"
#密码
jdbc_password => "****"
jdbc_driver_library => "mysql数据库连接jar包位置"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement => "select * from teachplan_media_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
schedule => "* * * * *"
record_last_run => true
last_run_metadata_path => "采集记录的时间戳文件路径"
type => "表名2"
tags => "表名2"
}
}
output {
if [type] == "表名1"{
elasticsearch {
hosts => "localhost:9200"
index => "索引1"
document_id => "%{id}"
document_type => "doc"
template =>"模板位置"
template_name =>"模板名称"
template_overwrite =>"true"
}
}
if [type] == "表名2"{
elasticsearch {
hosts => "localhost:9200"
index => "索引2"
document_id => "%{teachplan_id}"
document_type => "doc"
template =>"模板位置"
template_name =>"模板名称"
template_overwrite =>"true"
}
}
stdout {
codec => json_lines
}
}
2.运行,进入logstash的bin目录,
.\logstash -f ..\config\配置文件的名称.conf
参考:https://www.cnblogs.com/sanduzxcvbnm/p/12859292.html