aliyun RDS同步这里主要使用阿里云logstash工具来同步,如下图所示
logstash创建后需要创建数据管道方式来实现DB与ES的数据增量同步
管道配置如下:
input {
jdbc {
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_driver_library => ""
jdbc_connection_string => "jdbc:mysql://数据库地址:端口/数据库名?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowLoadLocalInfile=false&autoDeserialize=false"
jdbc_user => "数据库账号"
jdbc_password => "数据库密码"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement => "SELECT * from 表名 where updated_at>=:sql_last_value"
schedule => "* * * * * *"
record_last_run => true
last_run_metadata_path => ""
clean_run => false
tracking_column_type => "timestamp"
use_column_value => true
tracking_column => "updated_at"
}
}
filter {
json {
source => "message"
}
}
output {
stdout {
codec => json_lines
}
elasticsearch {
hosts => "ES地址:9200"
user => "ES账号"
password => "ES密码"
index => "ES的索引"
document_id => "%{DB or ES唯一主键}"
}
file_extend {
path => "debug日志文件path"
}
}