安装etcd集群：自签证书

0, etcd快速开始

下载二进制包：https://github.com/etcd-io/etcd/releases/download/v3.5.0/etcd-v3.5.0-linux-amd64.tar.gz
集群配置参考：https://etcd.io/docs/v3.5/op-guide/clustering/

#安装包-加速站 ghproxy.com
#解压，启动服务
[root@c73 ~]# cd /export/server/
[root@c73 server]#  tar -xf etcd-v3.5.0-linux-amd64.tar.gz
[root@c73 server]#  cd etcd-v3.5.0-linux-amd64/
[root@c73 etcd-v3.5.0-linux-amd64]# ll
total 56300
drwxr-xr-x. 3 114762 114762       40 Jun 16  2021 Documentation
-rwxr-xr-x. 1 114762 114762 23560192 Jun 16  2021 etcd
-rwxr-xr-x. 1 114762 114762 17969152 Jun 16  2021 etcdctl
-rwxr-xr-x. 1 114762 114762 16048128 Jun 16  2021 etcdutl
-rw-r--r--. 1 114762 114762    42066 Jun 16  2021 README-etcdctl.md
-rw-r--r--. 1 114762 114762     7359 Jun 16  2021 README-etcdutl.md
-rw-r--r--. 1 114762 114762     9394 Jun 16  2021 README.md
-rw-r--r--. 1 114762 114762     7896 Jun 16  2021 READMEv2-etcdctl.md
[root@c73 etcd-v3.5.0-linux-amd64]# ./etcd
{"level":"info","ts":"2022-07-07T10:15:36.779+0800","caller":"etcdmain/etcd.go:72","msg":"Running: ","args":["./etcd"]}
{"level":"warn","ts":"2022-07-07T10:15:36.779+0800","caller":"etcdmain/etcd.go:104","msg":"'data-dir' was empty; using default","data-dir":"default.etcd"}
{"level":"info","ts":"2022-07-07T10:15:36.779+0800","caller":"embed/etcd.go:131","msg":"configuring peer listeners","listen-peer-urls":["http://localhost:2380"]}
{"level":"info","ts":"2022-07-07T10:15:36.781+0800","caller":"embed/etcd.go:139","msg":"configuring client listeners","listen-client-urls":["http://localhost:2379"]}

连接使用
参考：https://etcd.io/docs/v3.5/quickstart/

##另开一个窗口，设置环境变量，连接服务
[root@c73 ~]# echo 'export PATH=$PATH:/export/server/etcd-v3.5.0-linux-amd64/' > /etc/profile.d/etcd.sh
[root@c73 ~]# source /etc/profile

######读写数据，语法类似于redis
[root@c73 ~]# etcdctl put a 123
OK
[root@c73 ~]# etcdctl get a
a
123

######设置角色，用户名密码
#用户授权
[root@c73 ~]# export ETCDCTL_API=3
[root@c73 ~]# ENDPOINTS=localhost:2379
[root@c73 ~]#
[root@c73 ~]# etcdctl --endpoints=${ENDPOINTS} role add root
Role root created
[root@c73 ~]# etcdctl --endpoints=${ENDPOINTS} role get root
Role root
KV Read:
KV Write:
[root@c73 ~]# etcdctl  user add root
Password of root:
Type password of root again for confirmation:
User root created
[root@c73 ~]# etcdctl  user grant-role root root
Role root is granted to user root
[root@c73 ~]# etcdctl  user get root
User: root
Roles: root


[root@c73 ~]# etcdctl  role add role0
Role role0 created
[root@c73 ~]# etcdctl  role grant-permission role0 readwrite foo
Role role0 updated
[root@c73 ~]# etcdctl  user add user0
Password of user0:
Type password of user0 again for confirmation:
User user0 created
[root@c73 ~]# etcdctl  user grant-role user0 role0
Role role0 is granted to user user0

#开启密码认证，测试权限
[root@c73 ~]# etcdctl  auth enable
Authentication Enabled


[root@c73 ~]# etcdctl  --user root:root put key1 v1
OK
[root@c73 ~]# etcdctl  --user root:root get key1
key1
v1
[root@c73 ~]# etcdctl  --user=user0:user01 put foo bar
OK
[root@c73 ~]# etcdctl  --user=user0:user01 get foo
foo
bar
[root@c73 ~]# etcdctl  --user=user0:user01 put foo bar2
OK
[root@c73 ~]# etcdctl  --user=user0:user01 get foo
foo
bar2
[root@c73 ~]# etcdctl  --user=user0:user01 get key1
{"level":"warn","ts":"2022-07-07T10:39:36.151+0800","logger":"etcd-client","caller":"v3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0002a8fc0/#initially=[localhost:2379]","attempt":0,"error":"rpc error: code = PermissionDenied desc = etcdserver: permission denied"}
Error: etcdserver: permission denied
[root@c73 ~]# etcdctl  --user=root:root get foo
foo
bar2

1, 生成证书

#ca机构配置：有效期10年
cat > ca-config.json <<EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

#ca机构配置:　机构名称Comman Name，所在地Country国家, State省, Locality市
cat >  ca-csr.json <<EOF
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "ST": "Beijing",
            "L": "Beijing"
        }
    ]
}
EOF

#向ca机构申请：证书注册 (中国，北京省，北京市), 提供服务的ip
# Organization Name, Common Name
cat >  server-csr.json <<EOF
{
    "CN": "etcd",
    "hosts": [
    "192.168.56.180",
    "192.168.56.201"
    ], 
    "names": [
        {
            "C": "CN",
            "ST": "BeiJing",
            "L": "BeiJing",
            "O":"aa.com",
            "CN":"beijing.aa.com"
        }
    ]
}
EOF


#下载cfssl工具
curl -s -L -o /bin/cfssl https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 
curl -s -L -o /bin/cfssljson https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 
curl -s -L -o /bin/cfssl-certinfo https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 
chmod +x /bin/cfssl*

#用cfssl工具，生成证书：
mkdir ssl ; cd ssl
cfssl gencert -initca ../ca-csr.json | cfssljson -bare ca -
cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=../ca-config.json -profile=www ../server-csr.json | cfssljson -bare server
# ls *pem
#ca-key.pem  ca.pem  server-key.pem  server.pem

2, 安装etcd服务

etcd角色名	ip
etcd01	192.168.56.180
etcd02	192.168.56.201

下载etcd：https://github.com/etcd-io/etcd/releases/tag/v3.2.12

#下载etcd　tar包
cat >download-etcd.sh <<EOF
ETCD_VER=v3.2.12
GITHUB_URL=https://github.com/coreos/etcd/releases/download
DOWNLOAD_URL=${GITHUB_URL}
curl -L ${DOWNLOAD_URL}/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz  -o  etcd-${ETCD_VER}-linux-amd64.tar.gz
EOF
chmod +x download-etcd.sh
sh download-etcd.sh

#解压etcd　tar包，获取可执行文件：etcd, etcdctl
mkdir /opt/etcd/{bin,cfg,ssl}  -p
tar zxvf etcd-v3.2.12-linux-amd64.tar.gz
mv etcd-v3.2.12-linux-amd64/{etcd,etcdctl} /opt/etcd/bin/
rm -rf  etcd-v3.2.12*
#拷贝证书到etcd服务中
cp * /opt/etcd/ssl/

#编辑etcd配置文件
cat > /opt/etcd/cfg/etcd  <<EOF
#[Member]
ETCD_NAME="etcd01"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://192.168.56.180:2380"
ETCD_LISTEN_CLIENT_URLS="https://192.168.56.180:2379"
#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://192.168.56.180:2380"
ETCD_ADVERTISE_CLIENT_URLS="https://192.168.56.180:2379"
ETCD_INITIAL_CLUSTER="etcd01=https://192.168.56.180:2380,etcd02=https://192.168.56.201:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

#配置systemctl 管理etcd服务
cat  > /usr/lib/systemd/system/etcd.service <<EOF
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/opt/etcd/cfg/etcd
ExecStart=/opt/etcd/bin/etcd \
--name=${ETCD_NAME} \
--data-dir=${ETCD_DATA_DIR} \
--listen-peer-urls=${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/opt/etcd/ssl/server.pem \
--key-file=/opt/etcd/ssl/server-key.pem \
--peer-cert-file=/opt/etcd/ssl/server.pem \
--peer-key-file=/opt/etcd/ssl/server-key.pem \
--trusted-ca-file=/opt/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/opt/etcd/ssl/ca.pem
Restart=on-failure

同步/opt/etcd目录到其他主机

scp -r /opt/etcd  192.168.56.201:/opt/　# /opt/etcd/cfg/etcd 前5行配置, 改为本机ip的配置
scp  /usr/lib/systemd/system/etcd.service   192.168.56.201:/usr/lib/systemd/system/etcd.service  

依次启动所有主机上的etcd服务（第一台机器开始会处于等待状态，知道有新的节点启动，才会正常启动）

systemctl start etcd 
systemctl enable etcd 

ssh-copy-id 192.168.56.201 
ssh 192.168.56.201  "systemctl start etcd "
ssh 192.168.56.201  "systemctl enable etcd "

检查etcd集群：服务状态

#查看系统日志：
[root@jenkins ~]# tailf /var/log/messages
Sep 28 22:25:59 jenkins etcd: 96e8ad8d29182cfd is starting a new election at term 36
Sep 28 22:25:59 jenkins etcd: 96e8ad8d29182cfd became candidate at term 37
Sep 28 22:25:59 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from 96e8ad8d29182cfd at term 37
Sep 28 22:25:59 jenkins etcd: 96e8ad8d29182cfd [logterm: 1, index: 2] sent MsgVote request to bdc976e03235ad9b at term 37
Sep 28 22:26:00 jenkins etcd: 96e8ad8d29182cfd is starting a new election at term 37
Sep 28 22:26:00 jenkins etcd: 96e8ad8d29182cfd became candidate at term 38
Sep 28 22:26:00 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from 96e8ad8d29182cfd at term 38
Sep 28 22:26:00 jenkins etcd: 96e8ad8d29182cfd [logterm: 1, index: 2] sent MsgVote request to bdc976e03235ad9b at term 38
Sep 28 22:26:02 jenkins etcd: 96e8ad8d29182cfd is starting a new election at term 38
Sep 28 22:26:02 jenkins etcd: 96e8ad8d29182cfd became candidate at term 39
Sep 28 22:26:02 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from 96e8ad8d29182cfd at term 39
Sep 28 22:26:02 jenkins etcd: 96e8ad8d29182cfd [logterm: 1, index: 2] sent MsgVote request to bdc976e03235ad9b at term 39
Sep 28 22:26:03 jenkins etcd: 96e8ad8d29182cfd is starting a new election at term 39
Sep 28 22:26:03 jenkins etcd: 96e8ad8d29182cfd became candidate at term 40
Sep 28 22:26:03 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from 96e8ad8d29182cfd at term 40
Sep 28 22:26:03 jenkins etcd: 96e8ad8d29182cfd [logterm: 1, index: 2] sent MsgVote request to bdc976e03235ad9b at term 40
Sep 28 22:26:03 jenkins etcd: health check for peer bdc976e03235ad9b could not connect: dial tcp 192.168.56.201:2380: getsockopt: connection refused
Sep 28 22:26:03 jenkins etcd: peer bdc976e03235ad9b became active
Sep 28 22:26:03 jenkins etcd: established a TCP streaming connection with peer bdc976e03235ad9b (stream Message reader)
Sep 28 22:26:03 jenkins etcd: established a TCP streaming connection with peer bdc976e03235ad9b (stream MsgApp v2 reader)
Sep 28 22:26:03 jenkins etcd: established a TCP streaming connection with peer bdc976e03235ad9b (stream Message writer)
Sep 28 22:26:03 jenkins etcd: established a TCP streaming connection with peer bdc976e03235ad9b (stream MsgApp v2 writer)
Sep 28 22:26:03 jenkins etcd: 96e8ad8d29182cfd [term: 40] ignored a MsgVote message with lower term from bdc976e03235ad9b [term: 2]
Sep 28 22:26:04 jenkins etcd: publish error: etcdserver: request timed out
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd is starting a new election at term 40
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd became candidate at term 41
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from 96e8ad8d29182cfd at term 41
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd [logterm: 1, index: 2] sent MsgVote request to bdc976e03235ad9b at term 41
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd received MsgVoteResp from bdc976e03235ad9b at term 41
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd [quorum:2] has received 2 MsgVoteResp votes and 0 vote rejections
Sep 28 22:26:04 jenkins etcd: 96e8ad8d29182cfd became leader at term 41
Sep 28 22:26:04 jenkins etcd: raft.node: 96e8ad8d29182cfd elected leader 96e8ad8d29182cfd at term 41
Sep 28 22:26:04 jenkins etcd: setting up the initial cluster version to 3.2
Sep 28 22:26:04 jenkins etcd: published {Name:etcd01 ClientURLs:[https://192.168.56.180:2379]} to cluster 33793b7f6cbeb4e8
Sep 28 22:26:04 jenkins etcd: ready to serve client requests
Sep 28 22:26:04 jenkins etcd: ready to serve client requests
Sep 28 22:26:04 jenkins etcd: dialing to target with scheme: ""
Sep 28 22:26:04 jenkins etcd: could not get resolver for scheme: ""
Sep 28 22:26:04 jenkins etcd: dialing to target with scheme: ""
Sep 28 22:26:04 jenkins etcd: could not get resolver for scheme: ""
Sep 28 22:26:04 jenkins etcd: serving client requests on 192.168.56.180:2379
Sep 28 22:26:04 jenkins etcd: serving insecure client requests on 127.0.0.1:2379, this is strongly discouraged!
Sep 28 22:26:04 jenkins systemd: Started Etcd Server.
Sep 28 22:26:04 jenkins etcd: set the initial cluster version to 3.2
Sep 28 22:26:04 jenkins etcd: enabled capabilities for version 3.2

#检查服务
[root@jenkins ~]#  /opt/etcd/bin/etcdctl \
 --ca-file=/opt/etcd/ssl/ca.pem --cert-file=/opt/etcd/ssl/server.pem --key-file=/opt/etcd/ssl/server-key.pem \
 cluster-health
member 96e8ad8d29182cfd is healthy: got healthy result from https://192.168.56.180:2379
member bdc976e03235ad9b is healthy: got healthy result from https://192.168.56.201:2379