基于docker 的 minio 联邦扩容（ETCD）2023最新

qq_38343011

已于 2023-06-07 17:42:45 修改

阅读量1.1k

点赞数

文章标签： java docker 开发语言

于 2023-06-07 17:34:28 首次发布

本文链接：https://blog.csdn.net/qq_38343011/article/details/131091189

版权

Vmware 配置启动一个etcd 用于管理两个种群，ip地址为192.168.80.134
1、部署两个minio集群
集群1

ip	hostname
192.168.80.120	minio-1
192.168.80.131	minio-2
192.168.80.132	minio-3
192.168.80.133	minio-4

集群2

ip	hostname
192.168.80.112	minio-1
192.168.80.111	minio-2

安装etcd

2、安装etcd

yum install -y etcd

直接复制下面的/etc/etcd/etcd.conf,设置的是etcd单节点启动

#[Member]
#ETCD_CORS=""
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
#ETCD_WAL_DIR=""
ETCD_LISTEN_PEER_URLS="http://0.0.0.0:2380"
ETCD_LISTEN_CLIENT_URLS="http://0.0.0.0:2379"
#ETCD_MAX_SNAPSHOTS="5"
#ETCD_MAX_WALS="5"
ETCD_NAME="etcd_minio"
#ETCD_SNAPSHOT_COUNT="100000"
#ETCD_HEARTBEAT_INTERVAL="100"
#ETCD_ELECTION_TIMEOUT="1000"
#ETCD_QUOTA_BACKEND_BYTES="0"
#ETCD_MAX_REQUEST_BYTES="1572864"
#ETCD_GRPC_KEEPALIVE_MIN_TIME="5s"
#ETCD_GRPC_KEEPALIVE_INTERVAL="2h0m0s"
#ETCD_GRPC_KEEPALIVE_TIMEOUT="20s"
#
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://192.168.80.134:2380"
ETCD_ADVERTISE_CLIENT_URLS="http://192.168.80.134:2379"
#ETCD_DISCOVERY=""
#ETCD_DISCOVERY_FALLBACK="proxy"
#ETCD_DISCOVERY_PROXY=""
#ETCD_DISCOVERY_SRV=""
#ETCD_INITIAL_CLUSTER="etcd_minio=http://192.168.80.134:2380"
#ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
#ETCD_INITIAL_CLUSTER_STATE="new"
#ETCD_STRICT_RECONFIG_CHECK="true"
#ETCD_ENABLE_V2="true"
#
#[Proxy]
#ETCD_PROXY="off"
#ETCD_PROXY_FAILURE_WAIT="5000"
#ETCD_PROXY_REFRESH_INTERVAL="30000"
#ETCD_PROXY_DIAL_TIMEOUT="1000"
#ETCD_PROXY_WRITE_TIMEOUT="5000"
#ETCD_PROXY_READ_TIMEOUT="0"
#
#[Security]
#ETCD_CERT_FILE=""
#ETCD_KEY_FILE=""
#ETCD_CLIENT_CERT_AUTH="false"
#ETCD_TRUSTED_CA_FILE=""
#ETCD_AUTO_TLS="false"
#ETCD_PEER_CERT_FILE=""
#ETCD_PEER_KEY_FILE=""
#ETCD_PEER_CLIENT_CERT_AUTH="false"
#ETCD_PEER_TRUSTED_CA_FILE=""
#ETCD_PEER_AUTO_TLS="false"
#
#[Logging]
#ETCD_DEBUG="false"
#ETCD_LOG_PACKAGE_LEVELS=""
#ETCD_LOG_OUTPUT="default"
#
#[Unsafe]
#ETCD_FORCE_NEW_CLUSTER="false"
#
#[Version]
#ETCD_VERSION="false"
#ETCD_AUTO_COMPACTION_RETENTION="0"
#
#[Profiling]
#ETCD_ENABLE_PPROF="false"
#ETCD_METRICS="basic"
#
#[Auth]
#ETCD_AUTH_TOKEN="simple"

ETCD_DATA_DIR=数据存放位置
ETCD_LISTEN_PEER_URLS=监听的etcd节点URL，最好是指填写本集群内的所有节点，0.0.0.0表示监听全部地址
ETCD_LISTEN_CLIENT_URLS=监听的客户端URLs，最好填写MinIO集群的地址，只允许MinIO访问，但是考虑到未来扩容，这里监听全部
ETCD_NAME=etcd节点的名称，需要唯一

ETCD_INITIAL_ADVERTISE_PEER_URLS=广播节点URL
ETCD_ADVERTISE_CLIENT_URLS=广播客户端URL
ETCD_INITIAL_CLUSTER=集群内所有节点
ETCD_INITIAL_CLUSTER_TOKEN=令牌，每个节点需要相同
ETCD_INITIAL_CLUSTER_STATE=集群状态

所有的节点都进行上述配置。参数ETCD_INITIAL_CLUSTER填写的节点信息，必须与各节点上配置的ETCD_NAME参数以及ETCD_INITIAL_ADVERTISE_PEER_URLS参数进行对应

通过下面这个命令用来查看部署是否成功

[root@nginx-minio admin]# etcdctl endpoint health --endpoints=http://192.168.80.134:2380
http://192.168.80.134:2380 is healthy: successfully committed proposal: took = 14.34982m

3、使用联邦模式配置MinIO集群1，网上都是基于二进制模式的，这里是docker指令

docker run --name minio-01 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.120,192.168.80.131,192.168.80.132,192.168.80.133" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.120:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...4}/data{1...2}


docker run --name minio-02 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.120,192.168.80.131,192.168.80.132,192.168.80.133" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.131:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...4}/data{1...2}

docker run --name minio-03 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.120,192.168.80.131,192.168.80.132,192.168.80.133" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.132:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...4}/data{1...2}

docker run --name minio-04 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.120,192.168.80.131,192.168.80.132,192.168.80.133" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.133:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...4}/data{1...2}

配置集群2

docker run --name minio-01 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.120,192.168.80.131,192.168.80.132,192.168.80.133" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.112:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...2}/data{1...2}

docker run --name minio-02 --restart=always --net=host \
-e "MINIO_ROOT_USER=admin" \
-e "MINIO_ROOT_PASSWORD=admin123" \
-e "MINIO_ETCD_ENDPOINTS=http://192.168.80.134:2380" \
-e "MINIO_PUBLIC_IPS=192.168.80.112,192.168.80.111" \
-e "MINIO_DOMAIN=mydomain.com" \
-v /data/minio/update:/data1 \
-v /data/minio/bakup:/data2 \
minio/minio:latest server \
--address 192.168.80.111:9000 \
--console-address '0.0.0.0:9999' http://minio-{1...2}/data{1...2}

注意：MINIO_ETCD_ENDPOINTS参数需与搭建的ETCD集群所有节点IP相对应；MINIO_PUBLIC_IPS参数则为该集群的所有节点IP；MINIO_DOMAIN参数必须进行配置，即使你并不通过域名访问存储桶，否则联邦无法生效，只有MINIO_DOMAIN参数值相同的集群，才会组成联盟。

验证联邦集群搭建成功方式：
1、在192.168.80.120上创建bucket，在192.168.80.111上可以看到，过一段时间后查看，
在这里插入图片描述
fengzeming文件夹是在192.168.80.120上创建的，但是在192.168.80.111可以看到。

2、在etcd主机上 192.168.80.134上输入下面指令
ETCDCTL_API=3 etcdctl get --from-key ''
命令查看etcd中是否已经写入刚刚创建的Bucket的相关记录，记录如下

/skydns/com/mydomain/11111/192.168.80.111
{"host":"192.168.80.111","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.13922945Z"}
/skydns/com/mydomain/11111/192.168.80.112
{"host":"192.168.80.112","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.13922945Z"}
/skydns/com/mydomain/fengzeming/192.168.80.120
{"host":"192.168.80.120","port":9000,"ttl":30,"creationDate":"2023-06-07T08:05:01.444707314Z"}
/skydns/com/mydomain/fengzeming/192.168.80.131
{"host":"192.168.80.131","port":9000,"ttl":30,"creationDate":"2023-06-07T08:05:01.444707314Z"}
/skydns/com/mydomain/fengzeming/192.168.80.132
{"host":"192.168.80.132","port":9000,"ttl":30,"creationDate":"2023-06-07T08:05:01.444707314Z"}
/skydns/com/mydomain/fengzeming/192.168.80.133
{"host":"192.168.80.133","port":9000,"ttl":30,"creationDate":"2023-06-07T08:05:01.444707314Z"}
/skydns/com/mydomain/hello/192.168.80.111
{"host":"192.168.80.111","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.161574888Z"}
/skydns/com/mydomain/hello/192.168.80.112
{"host":"192.168.80.112","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.161574888Z"}
/skydns/com/mydomain/yanyan/192.168.80.111
{"host":"192.168.80.111","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.158747136Z"}
/skydns/com/mydomain/yanyan/192.168.80.112
{"host":"192.168.80.112","port":9000,"ttl":30,"creationDate":"2023-06-07T07:45:34.158747136Z"}
config/iam/format.json
{"version":1}
config/iam/sts/GLVB8MJOD32OZAU958D9/identity.json
{"version":1,"credentials":{"accessKey":"GLVB8MJOD32OZAU958D9","secretKey":"gHR7Aqz8+h6yTRtlINkzJe0nplJVkBpZgd2yBa+h","sessionToken":"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJHTFZCOE1KT0QzMk9aQVU5NThEOSIsImV4cCI6MTY4NjE1MDk2NCwicGFyZW50IjoiYWRtaW4ifQ.RUcny9x2Mgl-Hai100uFJUEOLg4-6Jcg6NYsHBU3LX8wmFs_Pogrucu--UD683PW_ldS1yb8H3JyKfiGb2owXQ","expiration":"2023-06-07T15:16:04Z","status":"on","parentUser":"admin"},"updatedAt":"2023-06-07T03:16:04.960400325Z"}
config/iam/sts/JSE8B9E1R59WT85KGP37/identity.json
{"version":1,"credentials":{"accessKey":"JSE8B9E1R59WT85KGP37","secretKey":"V9uDbcCJXMxIqsC0KDtJs4AtKOTofTqjCXcfJ8jg","sessionToken":"eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJKU0U4QjlFMVI1OVdUODVLR1AzNyIsImV4cCI6MTY4NjE2NzIwMywicGFyZW50IjoiYWRtaW4ifQ.Rbh-p7IBykj_DrLA5Z6I1nAR-DJv3YduCbF0ufGWRrsnt_7mETty8x9KBnDeX4fjLbqacXpQI1iLwIkTf5OthA","expiration":"2023-06-07T19:46:43Z","status":"on","parentUser":"admin"},"updatedAt":"2023-06-07T07:46:43.375296224Z"}