这是生产上通过docker compose部署的单节点grayLog
思路是只需要扩展opensearch和graylog就行,至于mongo,在当前规模的集群下,共用即可。
欢迎交流,私信。
version: "3.8"
services:
mongodb:
image: "mongo:5.0"
volumes:
- "/home/server/graylog/mongodb:/data/db"
restart: "on-failure"
opensearch:
image: "opensearchproject/opensearch:2.4.0"
environment:
- "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
- "bootstrap.memory_lock=true"
- "discovery.type=single-node"
- "action.auto_create_index=false"
- "plugins.security.ssl.http.enabled=false"
- "plugins.security.disabled=true"
ulimits:
memlock:
hard: -1
soft: -1
nofile:
soft: 65536
hard: 65536
volumes:
- "/home/server/graylog/opensearch:/usr/share/opensearch/data"
restart: "on-failure"
graylog:
hostname: "server"
image: "${GRAYLOG_IMAGE:-graylog/graylog:5.0}"
depends_on:
opensearch:
condition: "service_started"
mongodb:
condition: "service_started"
entrypoint: "/usr/bin/tini -- wait-for-it opensearch:9200 -- /docker-entrypoint.sh"
environment:
GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/config/node-id"
GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
GRAYLOG_HTTP_EXTERNAL_URI: "http://xxxxx:9000/" #实际地址
GRAYLOG_ELASTICSEARCH_HOSTS: "http://opensearch:9200"
GRAYLOG_MONGODB_URI: "mongodb://mongodb:27017/graylog"
GRAYLOG_ROOT_TIMEZONE: "Asia/Shanghai"
GRAYLOG_ALLOW_HIGHLIGHTING: "true"
ports:
- "5044:5044/tcp" # Beats
- "5140:5140/udp" # Syslog
- "5140:5140/tcp" # Syslog
- "5555:5555/tcp" # RAW TCP
- "5555:5555/udp" # RAW TCP
- "9000:9000/tcp" # Server API
- "12202:12202/tcp" # GELF TCP
- "22201:12202/udp" # GELF UDP
#- "10000:10000/tcp" # Custom TCP port
#- "10000:10000/udp" # Custom UDP port
- "13301:13301/tcp" # Forwarder data
- "13302:13302/tcp" # Forwarder config
volumes:
- "/home/server/graylog/graylog/graylog_data:/usr/share/graylog/data/data"
- "/home/server/graylog/graylog/graylog_journal:/usr/share/graylog/data/journal"
restart: "on-failure"
volumes:
mongodb_data:
os_data:
graylog_data:
graylog_journal:
修改本配置使之支持opensearch和graylog的集群模式
version: "3.8"
services:
mongodb:
image: "mongo:5.0"
volumes:
- "/home/server/graylog/mongodb:/data/db"
restart: "on-failure"
ports:
- "27017:27017" # 对外暴露mongo的端口27017
opensearch:
image: "opensearchproject/opensearch:2.4.0"
environment:
- "cluster.name=bblgGrayLogcluster" #设置集群名称
- "node.name=master" #设置当前节点名称
- "discovery.type=zen" #设置集群发现模式
- "network.publish_host=192.168.0.164" #当前节点发布地址
- "discovery.seed_hosts=192.168.0.164,192.168.0.165" #已知的节点地址
- "cluster.initial_cluster_manager_nodes=192.168.0.164,192.168.0.165" #可成为主节点的节点
- "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
- "bootstrap.memory_lock=true"
# - "discovery.type=single-node" 删除
- "action.auto_create_index=false"
- "plugins.security.ssl.http.enabled=false"
- "plugins.security.disabled=true"
ulimits:
memlock:
hard: -1
soft: -1
nofile:
soft: 65536
hard: 65536
volumes:
- "/home/server/graylog/opensearch:/usr/share/opensearch/data"
restart: "on-failure"
ports:
- "9200:9200" # opensearch对外暴露的地址
- "9300:9300" # opensearch对外暴露的地址
graylog:
hostname: "server"
image: "${GRAYLOG_IMAGE:-graylog/graylog:5.0}"
depends_on:
opensearch:
condition: "service_started"
mongodb:
condition: "service_started"
entrypoint: "/usr/bin/tini -- wait-for-it opensearch:9200 -- /docker-entrypoint.sh"
environment:
GRAYLOG_IS_LEADER: true #设置当前节点是主机点
GRAYLOG_CLUSTER_MODE_ENABLED: "true" #开启集群模式
GRAYLOG_CLUSTER_NODES: "192.168.0.164:12900,192.168.0.165:12900" #集群节点
GRAYLOG_CLUSTER_NODE_ID: "graylog_master" #当前节点id
GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/config/node-id"
GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
GRAYLOG_HTTP_EXTERNAL_URI: "http://192.168.0.164:9000/"
GRAYLOG_HTTP_PUBLISH_URI: "http://192.168.0.164:9000/" #当前节点对外发布地址
GRAYLOG_ELASTICSEARCH_HOSTS: "http://192.168.0.164:9200,http://192.168.0.165:9200" #直接采用跨容器的ip地址
GRAYLOG_MONGODB_URI: "mongodb://192.168.0.164:27017/graylog" #直接采用跨容器的ip
GRAYLOG_ROOT_TIMEZONE: "Asia/Shanghai"
GRAYLOG_ALLOW_HIGHLIGHTING: "true"
ports:
- "5044:5044/tcp" # Beats
- "5140:5140/udp" # Syslog
- "5140:5140/tcp" # Syslog
- "5555:5555/tcp" # RAW TCP
- "5555:5555/udp" # RAW TCP
- "9000:9000/tcp" # Server API
- "12202:12202/tcp" # GELF TCP
- "22201:12202/udp" # GELF UDP
#- "10000:10000/tcp" # Custom TCP port
#- "10000:10000/udp" # Custom UDP port
- "13301:13301/tcp" # Forwarder data
- "13302:13302/tcp" # Forwarder config
volumes:
- "/home/server/graylog/graylog/graylog_data:/usr/share/graylog/data/data"
- "/home/server/graylog/graylog/graylog_journal:/usr/share/graylog/data/journal"
restart: "on-failure"
volumes:
mongodb_data:
os_data:
graylog_data:
graylog_journal:
在从节点新增docker compose配置
#安装centos8的yum源
wget -O /etc/yum.repos.d/CentOS-Ali.repo http://mirrors.aliyun.com/repo/Centos-8.repo
yum cleanall
yum makecache
#关闭selinux
setenforce 0
vim /etc/selinux/config #在这个文件里面修改SELINUX为disabled
#关闭swap
swapoff -a
sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
#关闭防火墙
systemctl stop firewalld
systemctl disable firewalld.service
#如果没有yum-config-manager的话,安装
sudo yum install -y yum-utils
#添加docker的阿里巴巴yum源
sudo yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sudo vim /etc/docker/daemon.json
# 追加:
{
"registry-mirrors": ["https://ogp4c6i5.mirror.aliyuncs.com"]
}
# 重启docker
sudo systemctl daemon-reload
sudo systemctl restart docker
version: "3.8"
services:
opensearch:
image: "opensearchproject/opensearch:2.4.0"
environment:
- "cluster.name=bblgGrayLogcluster"
- "node.name=data1"
- "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
- "bootstrap.memory_lock=true"
- "discovery.type=zen"
- "network.publish_host=192.168.0.165"
- "discovery.seed_hosts=192.168.0.164,192.168.0.165"
- "cluster.initial_cluster_manager_nodes=192.168.0.164,192.168.0.165"
- "action.auto_create_index=false"
- "plugins.security.ssl.http.enabled=false"
- "plugins.security.disabled=true"
ulimits:
memlock:
hard: -1
soft: -1
nofile:
soft: 65536
hard: 65536
volumes:
- "/home/server/graylog/opensearch:/usr/share/opensearch/data"
restart: "on-failure"
ports:
- "9200:9200"
- "9300:9300"
graylog:
hostname: "server2"
image: "${GRAYLOG_IMAGE:-graylog/graylog:5.0}"
depends_on:
opensearch:
condition: "service_started"
entrypoint: "/usr/bin/tini -- wait-for-it 192.168.0.165:9200 -- /docker-entrypoint.sh"
environment:
GRAYLOG_IS_LEADER: false
GRAYLOG_CLUSTER_MODE_ENABLED: "true"
GRAYLOG_CLUSTER_NODES: "192.168.0.164:12900,192.168.0.165:12900"
GRAYLOG_CLUSTER_NODE_ID: "graylog_data1"
GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/config/node-id"
GRAYLOG_PASSWORD_SECRET: "${GRAYLOG_PASSWORD_SECRET:?Please configure GRAYLOG_PASSWORD_SECRET in the .env file}"
GRAYLOG_ROOT_PASSWORD_SHA2: "${GRAYLOG_ROOT_PASSWORD_SHA2:?Please configure GRAYLOG_ROOT_PASSWORD_SHA2 in the .env file}"
GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
GRAYLOG_HTTP_EXTERNAL_URI: "http://192.168.0.165:9000/"
GRAYLOG_HTTP_PUBLISH_URI: "http://192.168.0.165:9000/"
GRAYLOG_ELASTICSEARCH_HOSTS: "http://192.168.0.164:9200,http://192.168.0.165:9200"
GRAYLOG_MONGODB_URI: "mongodb://192.168.0.164:27017/graylog"
GRAYLOG_ROOT_TIMEZONE: "Asia/Shanghai"
GRAYLOG_ALLOW_HIGHLIGHTING: "true"
ports:
- "5044:5044/tcp" # Beats
- "5140:5140/udp" # Syslog
- "5140:5140/tcp" # Syslog
- "5555:5555/tcp" # RAW TCP
- "5555:5555/udp" # RAW TCP
- "9000:9000/tcp" # Server API
- "12202:12202/tcp" # GELF TCP
- "22201:12202/udp" # GELF UDP
#- "10000:10000/tcp" # Custom TCP port
#- "10000:10000/udp" # Custom UDP port
- "13301:13301/tcp" # Forwarder data
- "13302:13302/tcp" # Forwarder config
volumes:
- "/home/server/graylog/graylog/graylog_data:/usr/share/graylog/data/data"
- "/home/server/graylog/graylog/graylog_journal:/usr/share/graylog/data/journal"
restart: "on-failure"
volumes:
mongodb_data:
os_data:
graylog_data:
graylog_journal:
安装nginx对greylog写入日志的udp流量进行负载均衡
可能需要配置本节点的文件句柄,因为udp很费句柄的
# For more information on configuration, see:
# * Official English Documentation: http://nginx.org/en/docs/
# * Official Russian Documentation: http://nginx.org/ru/docs/
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
worker_rlimit_nofile 655360;
# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
include /usr/share/nginx/modules/*.conf;
events {
worker_connections 32000;
multi_accept on;
use epoll;
}
stream{
upstream backend_22201 {
server 192.168.0.164:22201;
server 192.168.0.165:22201;
}
server {
listen 22201 udp reuseport;
proxy_pass backend_22201;
proxy_responses 0;
#proxy_timeout 1s;
}
}