Docker搭建ELK,并将MySql数据同步到ES中_docker maxwell 同步到es-CSDN博客

本文链接：https://blog.csdn.net/u012075383/article/details/106724395

创建Docker自定义网络

# 查看一下 docker network 用法
[root@daigd dgd]# docker network --help
Usage:  docker network COMMAND
Manage networks
Commands:
  connect     Connect a container to a network
  create      Create a network
  disconnect  Disconnect a container from a network
  inspect     Display detailed information on one or more networks
  ls          List networks
  prune       Remove all unused networks
  rm          Remove one or more networks
# 查看一下 docker network create 用法
[root@daigd dgd]# docker network create --help
Usage:  docker network create [OPTIONS] NETWORK
Create a network
Options:
      --attachable           Enable manual container attachment
      --aux-address map      Auxiliary IPv4 or IPv6 addresses used by Network driver (default map[])
      --config-from string   The network from which copying the configuration
      --config-only          Create a configuration only network
  -d, --driver string        Driver to manage the Network (default "bridge")
      --gateway strings      IPv4 or IPv6 Gateway for the master subnet
      --ingress              Create swarm routing-mesh network
      --internal             Restrict external access to the network
      --ip-range strings     Allocate container ip from a sub-range
      --ipam-driver string   IP Address Management Driver (default "default")
      --ipam-opt map         Set IPAM driver specific options (default map[])
      --ipv6                 Enable IPv6 networking
      --label list           Set metadata on a network
  -o, --opt map              Set driver specific options (default map[])
      --scope string         Control the network's scope
      --subnet strings       Subnet in CIDR format that represents a network segment

结合帮助手册，创建一个自定义网络：

# -d bridge 指定网络驱动方式为桥接模式（默认也是这个）
# --subnet 指定子网掩码
# --gateway 指定网关
[root@daigd dgd]# docker network create -d bridge --subnet 192.162.0.0/16 --gateway 192.162.0.1 mynet

用Docker搭建ElasticSearch

# --name 指定容器名字
# --net 指定网络
# -e "discovery.type=single-node" 使用开发模式，即只有一个节点
# -e ES_JAVA_OPTS="-Xms128m -Xmx512m" 限制JVM内存最多使用512M，因为ES非常占内存，默认启动会占去1G多
docker run -d --name es --net mynet -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ES_JAVA_OPTS="-Xms128m -Xmx512m" elasticsearch:7.7.1

# 测试是否启动成功
# 有如下信息出来表示ES已启动成功
[root@daigd dgd]# curl localhost:9200
{
  "name" : "fffa6a086b64",
  "cluster_name" : "docker-cluster",
  "cluster_uuid" : "ypkwhS_TSQm7IG1LqHNWTA",
  "version" : {
    "number" : "7.7.1",
    "build_flavor" : "default",
    "build_type" : "docker",
    "build_hash" : "ad56dce891c901a492bb1ee393f12dfff473a423",
    "build_date" : "2020-05-28T16:30:01.040088Z",
    "build_snapshot" : false,
    "lucene_version" : "8.5.1",
    "minimum_wire_compatibility_version" : "6.8.0",
    "minimum_index_compatibility_version" : "6.0.0-beta1"
  },
  "tagline" : "You Know, for Search"
}

用Dockers搭建Kibana

docker run -d --name kibana --net mynet  -p 5601:5601 kibana:7.7.1
# 测试一下Kibana容器是否启动成功
[root@daigd dgd]# curl localhost:5601
Kibana server is not ready yet

# 提示Kibana server is not ready yet，启动失败了，我们看下容器启动日志
# --tail 10 查看日志的最后10条记录

[root@daigd dgd]# docker logs --tail 10 kibana 
....
{"type":"log","@timestamp":"2020-06-06T10:40:11Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"Unable to revive connection: http://elasticsearch:9200/"}
{"type":"log","@timestamp":"2020-06-06T10:40:11Z","tags":["warning","elasticsearch","admin"],"pid":6,"message":"No living connections"}

通过察看Kibana容器的启动日志，我们看到一行关键日志：message":"Unable to revive connection: http://elasticsearch:9200/。

通过http://elasticsearch:9200/无法访问ES，进容器修改下Kibana配置。

# 以交互模式进入到kibana容器
[root@daigd dgd]# docker exec -it kibana /bin/bash
bash-4.2$ ls
LICENSE.txt  NOTICE.txt  README.txt  bin  built_assets  config  data  node  node_modules  optimize  package.json  plugins  src  webpackShims  x-pack
bash-4.2$ cd config/
bash-4.2$ ls
kibana.yml
# 修改配置文件
bash-4.2$ vi kibana.yml 
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
# 将http://elasticsearch:9200 修改成 http://es:9200 es是我们ES容器取的名字，因为都指定了自定义网络，故容器之间可通过容器名访问
elasticsearch.hosts: [ "http://elasticsearch:9200" ]
monitoring.ui.container.elasticsearch.enabled: true

# 修改后保存退出,查看一下是否修改正确
bash-4.2$ cat kibana.yml 
#
# ** THIS IS AN AUTO-GENERATED FILE **
#

# Default Kibana configuration for docker target
server.name: kibana
server.host: "0"
elasticsearch.hosts: [ "http://es:9200" ]
monitoring.ui.container.elasticsearch.enabled: true

# 重启kibana 容器
[root@daigd dgd]# docker restart kibana
kibana

# 再测试一下Kibana容器是否启动成功，无任何内容表示，表示没问题，这时我们再在浏览器上输入ip(docker所在主机IP地址):5601，来访问kibana，可以成功访问了
[root@daigd dgd]# curl localhost:5601

用Docker搭建Mysql

# --name 指定容器名
# --p 暴露端口
# --net 指定自定义网络
# -e MYSQL_ROOT_PASSWORD=123456 指定数据库密码
# -v "$PWD/mysql/data":/var/lib/mysql 将mysql的/var/lib/myql 目录挂载到当前路径下的/mysql/data目录
# -v "$PWD/mysql/conf":/etc/mysql/conf.d 将mysql的配置挂载到当前路径下的/mysql/conf目录
docker run -d --name mysql8 -p 3306:3306 --net mynet \
 -e MYSQL_ROOT_PASSWORD=123456 \
 -v "$PWD/mysql/data":/var/lib/mysql \
 -v "$PWD/mysql/conf":/etc/mysql/conf.d \
 mysql:8

docker 启动mysql后远程无法访问：

# 以交互模式进入容器
[root@daigd dgd]# docker exec -it mysql8 /bin/bash
# 连接数据库
root@c038801dbb6c:/# cd /var/lib/mysql/
root@c038801dbb6c:/var/lib/mysql# mysql -uroot -p123456root@c038801dbb6c:/# cd /var/lib/mysql/

mysql> show databases;
+--------------------+
| Database           |
+--------------------+
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+
4 rows in set (0.00 sec)

# 切换到mysql库
mysql> use mysql;

mysql> select host,user from user where user='root';
+-----------+------+
| host      | user |
+-----------+------+
| %         | root |
| localhost | root |
+-----------+------+
2 rows in set (0.00 sec)

# 修改user表
mysql> ALTER USER 'root'@'%' IDENTIFIED WITH mysql_native_password BY '123456';
Query OK, 0 rows affected (0.00 sec)

创建数据库及相关表

# 创建一个表
DROP TABLE IF EXISTS `article`;
CREATE TABLE `article`(
id BIGINT NOT NULL PRIMARY KEY auto_increment COMMENT '文章ID',
title VARCHAR(16) NOT NULL COMMENT '标题',
content text NOT NULL COMMENT '内容',
author VARCHAR(16) COMMENT '作者',
created_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
created_by VARCHAR(16) NOT NULL COMMENT '创建人',
updated_by VARCHAR(16) NOT NULL COMMENT '更新人',
updated_at datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间'
);

INSERT INTO `article`(`title`,`content`,`author`,`created_by`,`updated_by`) VALUES ('Go编程学习','第一章节:Go历史；第二章节：Hello Go!','daigd','sys','sys');

用Docker搭建Logstash

# 使用具名挂载logstash的配置目录和jar包目录
docker run -d --name logstash --net mynet \
-v logstash-config:/usr/share/logstash/config \
-v logstash-lib:/usr/share/logstash/lib \
docker.elastic.co/logstash/logstash:7.7.1

# 容器成功启动后，进入容器
docker exec -it logstash /bin/bash
# 查看当前目录
bash-4.2$ pwd
/usr/share/logstash
bash-4.2$ 
# 进入config 目录下，创建一个 jdbc.conf 文件
bash-4.2$ cd config/
bash-4.2$ pwd
/usr/share/logstash/config

jdbc.conf 文件内容：

mysql-connector-java-8.0.20.jar为Myslq驱动包，在挂载目录 logstash-lib上传该文件，就会自动同步到容器内。

input {
  jdbc {
    jdbc_driver_library => "../lib/mysql-connector-java-8.0.20.jar"
    jdbc_driver_class => "Java::com.mysql.jdbc.Driver"
    jdbc_connection_string => "jdbc:mysql://mysql8:3306/forum"
    jdbc_user => "root"
    jdbc_password => "123456"
    statement => "select `id`,`title`,`content`,`author`,`created_at`,`created_by` FROM forum.article"
    jdbc_paging_enabled => "true"
    jdbc_page_size => "50000"
  }
}

filter {
}

output {
  stdout {
    codec => rubydebug
  }

  elasticsearch {
    hosts => ["es:9200"]
    index => "forum-mysql"
  }
}

# 执行脚本
../bin/logstash -f jdbc.conf 
# 发现如下输出如下日志
......
[2020-06-06T14:54:57,659][FATAL][logstash.runner          ] Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.
[2020-06-06T14:54:57,661][ERROR][org.logstash.Logstash    ] java.lang.IllegalStateException: Logstash stopped processing because of an error: (SystemExit) exit

在网上找到该错误的解决方法：

If you want to run multiple logstash instances, you need to define the path.data either by command,make sure the directory is writable

bin/logstash -f <config_file.conf> --path.data PATH

创建一个目录mydata,重新执行以下命令：

 ../bin/logstash -f jdbc.conf --path.data ../mydata/

稍等片刻之后，看到输出以下日志：

......
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
    "created_by" => "sys",
            "id" => 1,
       "content" => "第一章节:Go历史；第二章节：Hello Go!",
         "title" => "Go编程学习",
    "@timestamp" => 2020-06-06T15:07:16.364Z,
        "author" => "daigd",
    "created_at" => 2020-06-06T09:46:50.000Z,
      "@version" => "1"
}
[2020-06-06T15:07:18,077][INFO ][logstash.javapipeline    ] Pipeline terminated {"pipeline.id"=>".monitoring-logstash"}
[2020-06-06T15:07:18,854][INFO ][logstash.runner          ] Logstash shut down.

程序正常退出，去Kibana上查看索引情况，在Dev Tools上执行以下命令：

GET forum-mysql/_search

有如下结果输出，表明Logstash已正常将Mysql数据到ES中。

{
  "took" : 1,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 1,
      "relation" : "eq"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "forum-mysql",
        "_type" : "_doc",
        "_id" : "mOorinIBhQ3tXszZ-CUZ",
        "_score" : 1.0,
        "_source" : {
          "created_by" : "sys",
          "id" : 1,
          "content" : "第一章节:Go历史；第二章节：Hello Go!",
          "title" : "Go编程学习",
          "@timestamp" : "2020-06-06T15:07:16.364Z",
          "author" : "daigd",
          "created_at" : "2020-06-06T09:46:50.000Z",
          "@version" : "1"
        }
      }
    ]
  }
}
        "_source" : {
          "created_by" : "sys",
          "id" : 1,
          "content" : "第一章节:Go历史；第二章节：Hello Go!",
          "title" : "Go编程学习",
          "@timestamp" : "2020-06-06T15:07:16.364Z",
          "author" : "daigd",
          "created_at" : "2020-06-06T09:46:50.000Z",
          "@version" : "1"
        }
      }
    ]
  }
}