安装包下载:
- 官网下载路径:https://artifacts.elastic.co/downloads/logstash/logstash-7.1.0.tar.gz
- 网盘下载链接:https://pan.baidu.com/s/1-bLS6YUOk93BvtPhrPiUDg 提取码:0poq
构建logstash镜像
- 解压压缩包:tar -xf logstash-7.1.0.tar.gz
- 编写镜像配置文件:vi Dockerfile
# 本地构建
FROM java:8
MAINTAINER xxxxxx@qq.com
ADD logstash-7.1.0 /usr/share/logstash
RUN cd /usr/share/logstash
ADD run.sh /run.sh
RUN chmod 755 /*.sh
EXPOSE 5047
CMD ["/run.sh"]
- 编写logstash启动脚本:vi run.sh
#!/bin/bash
/usr/share/logstash/bin/logstash
- 编写构建镜像脚本:vi build.sh
docker stop logstash
docker rm logstash
docker rmi custom/logstash-7.1.0
docker build -t custom/logstash-7.1.0 .
- 启动脚本,构建镜像:sh build.sh
使用docker-compose启动logstash
- 创建文件目录:mkdir -p config lib pipeline
- 把mysql-connector-java-8.0.21.jar放到lib目录
- 编写logstash启动配置文件:vi config/logstash.yml
config:
reload:
automatic: true
interval: 3s
xpack:
management.enabled: false
monitoring.enabled: false
- 编写管道配置文件:vi config/pipelines.yml
#- pipeline.id: logstash_dev
# path.config: "/usr/share/logstash/pipeline/logstash_dev.conf"
- pipeline.id: logstash_mysql
path.config: "/usr/share/logstash/pipeline/logstash_mysql.conf"
- 编写mysql管道文件:vi pipeline/logstash_mysql.conf
input {
jdbc {
# 数据库连接配置
jdbc_connection_string => "jdbc:mysql://127.0.0.1:3306/member?useUnicode=true&characterEncoding=utf8&serverTimezone=UTC"
jdbc_user => "root"
jdbc_password => "123456"
# 数据库驱动配置
jdbc_driver_library => "/usr/share/logstash/lib/extra/mysql-connector-java-8.0.21.jar"
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_validate_connection => "true"
jdbc_paging_enabled => "true"
jdbc_page_size => "1000"
jdbc_default_timezone => "Asia/Shanghai"
# 执行的查询语句(全量更新)
statement => "select * from user_member"
schedule => "* * * * *"
use_column_value => true
tracking_column_type => "numeric"
tracking_column => "id"
clean_run => false
type => "jdbc"
}
}
output {
elasticsearch {
hosts => ["http://127.0.0.1:9200"]
index => "user_member"
document_type => "_doc"
document_id => "%{id}"
}
stdout {
codec => json_lines
}
}
- 编写默认管道配置(tcp方式):vi pipeline/logstash_dev.conf
input {
tcp {
mode => "server"
host => "0.0.0.0"
port => 5047
codec => json_lines
}
}
filter{
}
output {
elasticsearch {
hosts => ["127.0.0.1:9200"]
index => "logstash-dev-%{+YYYY.MM.dd}"
}
stdout { codec => rubydebug }
}
- 编写docker-compose配置文件:vi docker-compose.yml
version: "3"
logstash:
container_name: logstash
hostname: logstash
image: custom/logstash-7.1.0:latest
privileged: true
ports:
- 5047:5047
- 9600:9600
volumes:
- ./logstash/lib/:/usr/share/logstash/lib/extra
- ./logstash/config/:/usr/share/logstash/config/
- ./logstash/pipeline/:/usr/share/logstash/pipeline/
- 启动logstash日志采集服务
- 启动之前需要将确保elasticsearch服务启动,可参考:Docker部署elasticsearch
- 使用docker-compose启动:docker-compose up -d
- 查看启动日志:docker exec -it logstash /bin/bash
- 在kibana查看数据是否添加到es中