手动搭建Kubernetes1.8高可用集群（3）ETCD

最新推荐文章于 2024-08-27 17:53:42 发布

weixin_33716557

最新推荐文章于 2024-08-27 17:53:42 发布

阅读量332

点赞数

文章标签：运维

原文链接：http://blog.51cto.com/brave666/1963615

版权

Etcd 是 CoreOS 推出的高可用的键值存储系统，主要用于k8s集群的服务发现等，而本身 Etcd 也支持集群模式部署，从而实现自身高可用；我们使用容器启动etcd，systemd管理

一、准备

1、主机

Centos7 ，最好4台以上，越多越好
CPU2个
内存1.5G
主机名分别为node1,node2,node3,node4.......
关闭防火墙，selinux
时间同步
etcd搭建在node1,node2,node3

2、Docker

docker17.03
quay.io/coreos/etcd:v3.2.4

3、创建用户

groupadd -r kube-cert
useradd -r -g kube-cert -s /sbin/nologin kube

4、创建目录

/etc/ssl/etcd/ssl  属组kube 权限0700
/var/lib/etcd

5、设置/etc/hosts

[root@node1 ~]# cat /etc/hosts
127.0.0.1 localhost localhost.localdomain
::1 localhost6 localhost6.localdomain
192.168.1.121 node1 node1.cluster.local
192.168.1.122 node2 node2.cluster.local
192.168.1.123 node3 node3.cluster.local
192.168.1.124 node4 node3.cluster.local

二、自建etcd CA

etcd和k8s使用不同的CA以便管理

1、准备openssl.conf

[req]
req_extensions = v3_req
distinguished_name = req_distinguished_name

[req_distinguished_name]

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names

[ ssl_client ]
extendedKeyUsage = clientAuth, serverAuth
basicConstraints = CA:FALSE
subjectKeyIdentifier=hash
authorityKeyIdentifier=keyid,issuer
subjectAltName = @alt_names

[ v3_ca ]
basicConstraints = CA:TRUE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
subjectAltName = @alt_names
authorityKeyIdentifier=keyid:always,issuer

[alt_names]
DNS.1 = localhost
DNS.2 = node1
DNS.3 = node2
DNS.4 = node3
IP.1 = 192.168.1.121        #这里是3个节点的配置
IP.2 = 192.168.1.121
IP.3 = 192.168.1.122
IP.4 = 192.168.1.122
IP.5 = 192.168.1.123
IP.6 = 192.168.1.123
IP.7 = 127.0.0.1

2、自签CA

openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1

openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca"

3、签署 etcd 证书

openssl genrsa -out member-node1-key.pem 2048

openssl req -new -key member-node1-key.pem -out member-node1.csr -subj "/CN=etcd-member-node1" -config ./openssl.conf

openssl x509 -req -in member-node1.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-node1.pem -days 3650 -extensions ssl_client -extfile ./openssl.conf

以上命令生成node1的证书，把命令中的node1换成node2,node3，生成node2,node3的证书

4、分发证书

把生成的证书分发到每个节点/etc/ssl/etcd/ssl 证书属主kube 权限0600

三、准备配置文件

1、Unit文件/etc/systemd/system/etcd.service

[Unit]
Description=etcd docker wrapper
Wants=docker.socket
After=docker.service

[Service]
User=root
PermissionsStartOnly=true
EnvironmentFile=-/etc/etcd.env
ExecStart=/usr/local/bin/etcd
ExecStartPre=-/usr/bin/docker rm -f etcd1    #etcd1在不同的节点需要修改成 etcd1 etcd2 etcd3
ExecStop=/usr/bin/docker stop etcd1          #同上
Restart=always
RestartSec=15s
TimeoutStartSec=30s

[Install]
WantedBy=multi-user.target

2、etcd启动脚本/usr/local/bin/etcd 需要加执行权限

#!/bin/bash
/usr/bin/docker run \
  --restart=on-failure:5 \
  --env-file=/etc/etcd.env \
  --net=host \
  -v /etc/ssl/certs:/etc/ssl/certs:ro \
  -v /etc/ssl/etcd/ssl:/etc/ssl/etcd/ssl:ro \
  -v /var/lib/etcd:/var/lib/etcd:rw \
    --memory=512M \
    --oom-kill-disable \
      --blkio-weight=1000 \
    --name=etcd1 \                #这里的名字需要修改，同样是node1 , node2 ,node3
  quay.io/coreos/etcd:v3.2.4 \
  /usr/local/bin/etcd \
  "$@"

3、配置文件/etc/etcd.env 配置文件需要修改相关IP和证书名

ETCD_DATA_DIR=/var/lib/etcd
ETCD_ADVERTISE_CLIENT_URLS=https://192.168.1.121:2379                        #改
ETCD_INITIAL_ADVERTISE_PEER_URLS=https://192.168.1.121:2380                  #改
ETCD_INITIAL_CLUSTER_STATE=new
ETCD_METRICS=basic
ETCD_LISTEN_CLIENT_URLS=https://192.168.1.121:2379,https://127.0.0.1:2379    #改
ETCD_ELECTION_TIMEOUT=5000
ETCD_HEARTBEAT_INTERVAL=250
ETCD_INITIAL_CLUSTER_TOKEN=k8s_etcd
ETCD_LISTEN_PEER_URLS=https://192.168.1.121:2380                             #改
ETCD_NAME=etcd1                                                              #改
ETCD_PROXY=off
ETCD_INITIAL_CLUSTER=etcd1=https://192.168.1.121:2380,etcd2=https://192.168.1.122:2380,etcd3=https://192.168.1.123:2380
ETCD_AUTO_COMPACTION_RETENTION=8

# TLS settings
ETCD_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem                                
ETCD_CERT_FILE=/etc/ssl/etcd/ssl/member-node1.pem                            #改
ETCD_KEY_FILE=/etc/ssl/etcd/ssl/member-node1-key.pem                         #改
ETCD_PEER_TRUSTED_CA_FILE=/etc/ssl/etcd/ssl/ca.pem        
ETCD_PEER_CERT_FILE=/etc/ssl/etcd/ssl/member-node1.pem                       #改
ETCD_PEER_KEY_FILE=/etc/ssl/etcd/ssl/member-node1-key.pem                    #改
ETCD_PEER_CLIENT_CERT_AUTH=true

四、启动etcd集群

1、启动etcd

systemctl start etcd && systemctl enable etcd

1、测试etcd


[object Object]

五、需要注意问题

1、配置文件空格问题，以下报错就是由于空格导致的

Dec 06 00:59:59 node1 etcd[4684]: 2017-12-06 05:59:59.333113 C | etcdmain: couldn't find local name "etcd1                                                              " in the initial cluster 
Dec 06 00:59:59 node1 systemd[1]: etcd.service: main process exited, code=exited, status=1/FAILURE
etcdmain: couldn't find local name "etcd1                                                              " in the initial cluster

2、配置文件里的注释信息需要删除

etcd集群到此已经搭建好了，下面提供一个etcd自建CA脚本

#!/bin/bash

MASTERS="node1 node2 node3"
HOSTS="node1 node2 node3"

set -o errexit
set -o pipefail
usage()
{
cat << EOF
Create self signed certificates

Usage : $(basename $0) -f <config> [-d <ssldir>]
      -h | --help         : Show this message
      -f | --config       : Openssl configuration file
      -d | --ssldir       : Directory where the certificates will be installed

               ex :
               $(basename $0) -f openssl.conf -d /srv/ssl
EOF
}

# Options parsing
while (($#)); do
case "$1" in
-h | --help)   usage;   exit 0;;
-f | --config) CONFIG=${2}; shift 2;;
-d | --ssldir) SSLDIR="${2}"; shift 2;;
*)
usage
echo "ERROR : Unknown option"
exit 3
;;
esac
done

if [ -z ${CONFIG} ]; then
echo "ERROR: the openssl configuration file is missing. option -f"
exit 1
fi
if [ -z ${SSLDIR} ]; then
SSLDIR="/etc/ssl/etcd"
fi

tmpdir=$(mktemp -d /tmp/etcd_cacert.XXXXXX)
trap 'rm -rf "${tmpdir}"' EXIT
cd "${tmpdir}"

mkdir -p "${SSLDIR}"

# Root CA
if [ -e "$SSLDIR/ca-key.pem" ]; then
# Reuse existing CA
cp $SSLDIR/{ca.pem,ca-key.pem} .
else
openssl genrsa -out ca-key.pem 2048 > /dev/null 2>&1
openssl req -x509 -new -nodes -key ca-key.pem -days 10000 -out ca.pem -subj "/CN=etcd-ca" > /dev/null 2>&1
fi

# ETCD member
if [ -n "$MASTERS" ]; then
for host in $MASTERS; do
cn="${host%%.*}"
# Member key
openssl genrsa -out member-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key member-${host}-key.pem -out member-${host}.csr -subj "/CN=etcd-member-${cn}" -config ${CONFIG} > /dev/null 2>&1
openssl x509 -req -in member-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out member-${host}.pem -days 3650 -extensions ssl_client -extfile ${CONFIG} > /dev/null 2>&1

# Admin key
openssl genrsa -out admin-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key admin-${host}-key.pem -out admin-${host}.csr -subj "/CN=etcd-admin-${cn}" > /dev/null 2>&1
openssl x509 -req -in admin-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out admin-${host}.pem -days 3650 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
done
fi

# Node keys
#if [ -n "$HOSTS" ]; then
for host in $HOSTS; do
cn="${host%%.*}"
openssl genrsa -out node-${host}-key.pem 2048 > /dev/null 2>&1
openssl req -new -key node-${host}-key.pem -out node-${host}.csr -subj "/CN=etcd-node-${cn}" > /dev/null 2>&1
openssl x509 -req -in node-${host}.csr -CA ca.pem -CAkey ca-key.pem -CAcreateserial -out node-${host}.pem -days 3650 -extensions ssl_client  -extfile ${CONFIG} > /dev/null 2>&1
done
#fi

# Install certs
mv *.pem ${SSLDIR}/