一、下载解压
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.7.0.tar.gz
tar -zxvf logstash-7.7.0.tar.gz
二、修改配置文件
(1)修改内存使用大小
cd config
vim jvm.options
修改
-Xms256m
-Xmx256m
三、自定义配置文件
cd /usr/local/elasticsearch/logstash-7.7.0/bin
mkdir mysql
cd mysql
在mysql目录下分别创建三个文件:
(1)数据传输调度配置
input {
stdin { }
jdbc {
type => "lgb_test"
codec => plain { charset => "UTF-8"}
lowercase_column_names => false
jdbc_connection_string => "jdbc:mysql://192.168.1.105:3306/es"
jdbc_user => "root"
jdbc_password => "root"
jdbc_driver_library => "/usr/local/elasticsearch/logstash-7.7.0/lib/mysql-connector-java-8.0.13.jar"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_paging_enabled => "true"
#添加增量类配置
record_last_run => "true"
use_column_value => "true"
tracking_column => "event_id"
tracking_column_type => numeric
clean_run => "false"
last_run_metadata_path => "/usr/local/elasticsearch/logstash-7.7.0/bin/mysql/sql_last_value.txt"
jdbc_page_size => "50000"
#获取到记录的SQL查询语句
statement_filepath => "/usr/local/elasticsearch/logstash-7.7.0/bin/mysql/jdbc.sql"
#定时字段 各字段含义(由左至右)分、时、天、月、年,全部为*默认含义为每>分钟都更新
schedule => "10 * * * *"
}
}
output {
stdout {
codec => json_lines
}
if[type]=="lgb_test"{
elasticsearch {
#ESIP地址与端口
hosts => "kafka1:9200"
#ES索引名称(自己定义的)
index => "lgb_index_test"
#文档类型
document_type => "lgb_type_test"
#文档类型id
document_id => "%{event_id}"
}
}
}
(2)查询sql语句
select event_id,timestmp,formatted_message,level_string,caller_class from logging_event where event_id > :sql_last_value ORDER BY timestmp
# :sql_last_value 是内置变量,不要改,此处可以实现增量同步mysql数据到es
(3)创建sql_last_value.txt,用于保存追踪列的最后一条记录
vim sql_last_value.txt
touch sql_last_value.txt
(4)在lib目录,增加mysql驱动包
mysql-connector-java-8.0.13
(5)启动
su es
cd /usr/local/elasticsearch/logstash-7.7.0/bin
# 命令行启动
./logstash -f mysql/dataTran.conf
# 后台启动
nohup ./logstash -f mysql/dataTran.conf &