Docker构建elasticsearch

1，配置节点配置文件

# 节点1 es1.yml
cluster.name: elasticsearch-cluster                  # 集群名称
node.name: es-node1                                  # 节点名称
network.bind_host: 0.0.0.0                           
network.publish_host: 192.168.18.204                 
http.port: 9201                                      
transport.tcp.port: 9301
http.cors.enabled: true                                # 跨域配置
http.cors.allow-origin: "*"                            # 跨域配置2
node.master: true                            
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.18.204:9301","192.168.18.204:9302","192.168.18.204:9303"] # 集群所有节点
discovery.zen.minimum_master_nodes: 2

# 节点2 es2.yml
cluster.name: elasticsearch-cluster
node.name: es-node2
network.bind_host: 0.0.0.0
network.publish_host: 192.168.18.204
http.port: 9202
transport.tcp.port: 9302
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.18.204:9301","192.168.18.204:9302","192.168.18.204:9303"]
discovery.zen.minimum_master_nodes: 2

# 节点3 es3.yml
cluster.name: elasticsearch-cluster
node.name: es-node3
network.bind_host: 0.0.0.0
network.publish_host: 192.168.18.204
http.port: 9203
transport.tcp.port: 9303
http.cors.enabled: true
http.cors.allow-origin: "*"
node.master: true
node.data: true
discovery.zen.ping.unicast.hosts: ["192.168.18.204:9301","192.168.18.204:9302","192.168.18.204:9303"]
discovery.zen.minimum_master_nodes: 2

2,创建容器

docker create --name e1 --net host -v /home/es/es1/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /home/es/es1/es-cluster/data:/usr/share/elasticsearch/data elasticsearch:6.8.5

docker create --name e2 --net host -v /home/es/es2/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /home/es/es2/es-cluster/data:/usr/share/elasticsearch/data elasticsearch:6.8.5

docker create --name e3 --net host  -v /home/es/es3/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml -v /home/es/es3/es-cluster/data:/usr/share/elasticsearch/data elasticsearch:6.8.5


docker create --name e1 \
--net host \ # 使用docker本地网络，跟宿主机统一ip
-v /home/es/es1/es.yml:/usr/share/elasticsearch/config/elasticsearch.yml \ # 将步骤1的配置文件映射到容器 /usr/share/elasticsearch/config/elasticsearch.yml 地址
 -v /home/es/es1/es-cluster/data:/usr/share/elasticsearch/data  \ # 将数据存放映射到容器
elasticsearch:6.8.5

3，启动容器

将本地映射路径进行授权后执行容器启动

docker start es1 es2 es3

访问 地址:9201 查看服务启动是否正常

4，logstash 启动

docker run -d --name=logstash logstash:6.8.12

5，将logstash配置文件拷贝到本地

新建 /home/logstash 文件夹

 docker cp logstash:/usr/share/logstash/config /home/logstash
 docker cp logstash:/usr/share/logstash/data /home/logstash
 docker cp logstash:/usr/share/logstash/pipeline/opt/docker/logstash/ /home/logstash

6，配置logstash  /home/logstash/config/

http.host: "0.0.0.0"
xpack.monitoring.elasticsearch.hosts: [ "192.168.18.204:9201","192.168.18.204:9202","192.168.18.204:9203" ] # 集群地址
path.config: /usr/share/logstash/config/conf.d/*.conf               # 加载mysql配置文件

6，配置mysql 在 /home/logstash/config/ 创建 conf.d 目录，然后创建 jdbc.conf 配置mysql同步数据

 input{
     jdbc{
          # 连接数据库
          jdbc_connection_string => "jdbc:mysql://localhost:3303/demo?serverTimezone=Asia/Shanghai&characterEncoding=utf8&useSSL=false"
          jdbc_user => ""
          jdbc_password => ""
          # 连接数据库的驱动包
          jdbc_driver_library => "/usr/share/logstash/config/jars/mysql-connector-java-8.0.20.jar"
          jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
          jdbc_paging_enabled => "true"
          jdbc_page_size => "50000"
          codec => plain { charset => "UTF-8" }

          # 数据追踪
          # 追踪的字段
        #   tracking_column => "update_time"
          # 上次追踪的元数据存放位置
        #   last_run_metadata_path => "/usr/share/logstash/config/lastrun/logstash_jdbc_last_run"
          # 设置时区
        #   jdbc_default_timezone => "Asia/Shanghai"  
          # sql 文件地址
          # statement_filepath => ""
          # sql
          statement => "SELECT * FROM device"
          # 是否清除 last_run_metadata_path 的记录,如果为真那么每次都相当于从头开始查询所有的数据库记录
        #   clean_run =>false
          # 这是控制定时的，重复执行导入任务的时间间隔，第一位是分钟 不设置就是1分钟执行一次
        #   schedule => "* * * * *"
     }
}
output{
      elasticsearch{
                   # 要导入到的Elasticsearch所在的主机
                   hosts => "192.168.18.204:9201"
                   # 要导入到的Elasticsearch的索引的名称
                   index => "iot"
                   # 类型名称（类似数据库表名）
                   document_type => "device"
                   # 主键名称（类似数据库表名）
                   document_id => "%{id}"
      }

      stdout{
            # JSON 格式输出
            codec => json_lines
      } 
}

7，删除镜像后启动镜像，开始同步

docker run --name logstash
 -p 5044:5044 -p 9600:9600 
-e ES_JAVA_OPTS="-Duser.timezone=Asia/Shanghai" 
-v /home/logstash/config:/usr/share/logstash/config # 映射配置文件
-v /home/logstash/data:/usr/share/logstash/data 
-v /home/logstash/pipeline:/usr/share/logstash/pipeline 
-d logstash:6.8.12

8，启动 elasticsearch 查询工具

docker run -d -p 9100:9100 docker.io/mobz/elasticsearch-head:5

elasticsearch-head 会有406问题，使用 exec -it 进入容器内部，安装vim 后修改 

/usr/src/app/_site/vendor.js

6886行 /contentType: "application/x-www-form-urlencoded 
    改成 contentType: "application/json;charset=UTF-8"

7574行 var inspectData = s.contentType ==`= "application/x-www-form-urlencoded" &&` 
    改成 var inspectData = s.contentType === "application/json;charset=UTF-8"