接上文安装Elasticsearch
1. 准备
cd /home/elk/
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.13.3-linux-x86_64.tar.gz
tar -xf logstash-7.13.3-linux-x86_64.tar.gz --strip-components 1 -C logstash
./bin/logstash-plugin list
./bin/logstash-plugin install logstash-input-jdbc
./bin/logstash-plugin install logstash-output-elasticsearch
wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-8.0.26.tar.gz
tar -xf mysql-connector-java-8.0.26.tar.gz
2.配置
2.1配置 config/jvm.options
-Xms512m
-Xms512m
2.2配置mysql的同步规则
vim config/mysql_dbname_tbname1.conf
input{
jdbc{
jdbc_driver_library=>"/home/elk/logstash/mysql-connector-java-8.0.26/mysql-connector-java-8.0.26.jar"
jdbc_driver_class=>"com.mysql.jdbc.Driver"
jdbc_connection_string=>"jdbc:mysql://127.0.0.1:3306/dbname"
jdbc_user=>"mysql_user_name"
jdbc_password=>"mysql_user_passwd"
schedule=>"* * * * *"
statement=>"SELECT * FROM table_name WHERE mtime >= :sql_last_value"
use_column_value=>true
tracking_column=>"mtime"
tracking_column_type=>"timestamp"
last_run_metadata_path=>"syncpoint_table"
}
}
output{
elasticsearch{
hosts=>"127.0.0.1:9200"
template_overwrite=>true
template=>"/home/elk/logstash/template/dbname_tbname.json"
}
stdout{
codec=>json_lines
}
}
3. 启动
3.1 单个启动
./bin/logstash -f config/mysql_dbname_tbname1.conf
./bin/logstash -f config/
3.2 若有多个表需要同步,采用批量启动
vim config/pipelines.yml
- pipeline.id: tablename1
path.config: "/home/elk/logstash/config/mysql_dbname_tbname1.conf"
- pipeline.id: tablename2
path.config: "/home/elk/logstash/config/mysql_dbname_tbname2.conf"
./bin/logstash