Docker中部署logstash同步mysql数据
一、安装
下载镜像
sudo docker pull logstash:7.5.1
启动logstash
sudo docker run -d --name=logstash logstash:7.5.1
等待30秒,查看日志
sudo docker logs -f logstash
启动成功
拷贝数据,授予权限
sudo docker cp logstash:/usr/share/logstash /home/zx
mkdir /home/zx/logstash/config/conf.d
sudo chmod 777 -R /home/zx/logstash
修改配置文件
vi logstash/config/logstash.yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://127.0.0.1:9200" ]
#path.config: /usr/share/logstash/config/conf.d/*.conf
path.logs: /user/share/logstash/logs
删除之前的容器,重新创建新的容器
#删除
sudo docker rm -f logstash
#重启
sudo docker run \
--name logstash \
--restart=always \
-p 5044:5044 \
-p 9600:9600 \
-e ES_JAVA_OPTS="-Duser.timezone=Asia/Shanghai" \
-v /home/zx/logstash/config:/usr/share/logstash/config \
-v /home/zx/logstash/data:/usr/share/logstash/data \
-v /home/zx/logstash/pipeline:/usr/share/logstash/pipeline \
-d logstash:7.5.1
#参数说明
sudo docker run \
--name logstash\ 将容器命名为 logstash
--restart=always \ 容器自动重启
-p 5044:5044 \ 将容器的5044端口映射到宿主机5044端口 logstash的启动端口
-p 9600:9600 \ 将容器的9600端口映射到宿主机9600 端口,api端口
-e ES_JAVA_OPTS="-Duser.timezone=Asia/Shanghai" \ 设置时区
-v /home/zx/logstash/config:/usr/share/logstash/config \
-v /home/zx/logstash/data:/usr/share/logstash/data \
-v /home/zx/logstash/pipeline:/usr/share/logstash/pipeline \ 挂载
-d logstash:7.5.1 后台运行容器,并返回容器ID
查看是否启动成功
sudo docker ps -a
sudo docker logs -f --tail 200 logstash
二、mysql同步配置
进入容器
sudo docker exec -it logstash bash
查询是否有 logstash-input-jdbc
./bin/logstash-plugin list --verbose
如果没有就安装插件(一般默认安装)
./bin/logstash-plugin install logstash-input-jdbc
退出容器
exit
在文件目录/home/zx/logstash/config/conf.d下创建jdbc.conf文件
input{
jdbc{
# 连接数据库
jdbc_connection_string => "jdbc:mysql://47.108.13.175:3306/edu?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false"
jdbc_user => "root"
jdbc_password => "HBcloud.1024"
# 连接数据库的驱动包
jdbc_driver_library => "/usr/share/logstash/config/jars/mysql-connector-java-8.0.20.jar"
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
codec => plain { charset => "UTF-8" }
# 数据追踪
# 追踪的字段
tracking_column => "update_time"
# 上次追踪的元数据存放位置
last_run_metadata_path => "/usr/share/logstash/config/lastrun/logstash_jdbc_last_run"
# 设置时区
jdbc_default_timezone => "Asia/Shanghai"
# sql 文件地址
# statement_filepath => ""
# sql
statement => "SELECT g.merchant_id AS id,g.nickname AS nickname,g.avatar_name AS avatarName,g.`desc` AS `desc`,g.contacts AS contacts,g.province AS province,g.city AS city,g.district AS district,g.address AS address, ST_AsText(g.coordinate) AS coordinate FROM wbc_merchant g WHERE g.update_time > :sql_last_value"
# 是否清除 last_run_metadata_path 的记录,如果为真那么每次都相当于从头开始查询所有的数据库记录
clean_run =>false
# 这是控制定时的,重复执行导入任务的时间间隔,第一位是分钟 不设置就是1分钟执行一次
schedule => "* * * * *"
}
}
output{
elasticsearch{
# 要导入到的Elasticsearch所在的主机
hosts => "127.0.0.1:9200"
# 要导入到的Elasticsearch的索引的名称
index => "merchant_index"
# 类型名称(类似数据库表名)
document_type => "merchanteso"
# 主键名称(类似数据库表名)
document_id => "%{id}"
}
stdout{
# JSON 格式输出
codec => json_lines
}
}
关闭logstash.yml 中path.config: /usr/share/logstash/config/conf.d/*.conf的注释
重启容器
vi logstash/config/logstash.yml
http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "http://127.0.0.1:9200" ]
path.config: /usr/share/logstash/config/conf.d/*.conf
path.logs: /user/share/logstash/logs