K8S-Demo集群实践06：部署kube-apiserver到master节点（3个无状态实例）
- kube-apiserver是无状态的，可以通过kube-nginx进行代理访问，从而保证服务高用性
一、下载并分发二进制文件到3个master节点
[root@master1 ~]# cd /opt/install/
[root@master1 install]# wget https://dl.k8s.io/v1.18.5/kubernetes-server-linux-amd64.tar.gz
[root@master1 install]# tar -xzvf kubernetes-server-linux-amd64.tar.gz
[root@master1 install]# cd kubernetes
[root@master1 kubernetes]# tar -xzvf kubernetes-src.tar.gz
[root@master1 kubernetes]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp server/bin/{kube-apiserver,kube-controller-manager,kube-scheduler,kubeadm,mounter} root@${node_ip}:/opt/k8s/bin/
ssh root@${node_ip} "chmod +x /opt/k8s/bin/*"
done
二、创建加密配置文件
[root@master1 ~]# cd /opt/install/kubeconfig
[root@master1 kubeconfig]# export ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
[root@master1 kubeconfig]# cat > encryption-config.yaml <<EOF
kind: EncryptionConfig
apiVersion: v1
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: ${ENCRYPTION_KEY}
- identity: {}
EOF
[root@master1 kubeconfig]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp encryption-config.yaml root@${node_ip}:/opt/k8s/etc/encryption-config.yaml
done
三、创建并分发审计策略
1、创建审计策略文件 audit-policy.yaml
[root@master1 ~]# cd /opt/kubeconfig
[root@master1 kubeconfig]# cat > audit-policy.yaml <<EOF
apiVersion: audit.k8s.io/v1beta1
kind: Policy
rules:
# The following requests were manually identified as high-volume and low-risk, so drop them.
- level: None
resources:
- group: ""
resources:
- endpoints
- services
- services/status
users:
- 'k8s-demo-kube-proxy'
verbs:
- watch
- level: None
resources:
- group: ""
resources:
- nodes
- nodes/status
userGroups:
- 'system:nodes'
verbs:
- get
- level: None
namespaces:
- kube-system
resources:
- group: ""
resources:
- endpoints
users:
- 'k8s-demo-ctrl-mgr'
- 'k8s-demo-scheduler'
- 'system:serviceaccount:kube-system:endpoint-controller'
verbs:
- get
- update
- level: None
resources:
- group: ""
resources:
- namespaces
- namespaces/status
- namespaces/finalize
users:
- 'k8s-demo-apiserver'
verbs:
- get
# Don't log HPA fetching metrics.
- level: None
resources:
- group: metrics.k8s.io
users:
- 'k8s-demo-ctrl-mgr'
verbs:
- get
- list
# Don't log these read-only URLs.
- level: None
nonResourceURLs:
- '/healthz*'
- /version
- '/swagger*'
# Don't log events requests.
- level: None
resources:
- group: ""
resources:
- events
# node and pod status calls from nodes are high-volume and can be large, don't log responses
# for expected updates from nodes
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
users:
- kubelet
- 'system:node-problem-detector'
- 'system:serviceaccount:kube-system:node-problem-detector'
verbs:
- update
- patch
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- nodes/status
- pods/status
userGroups:
- 'system:nodes'
verbs:
- update
- patch
# deletecollection calls can be large, don't log responses for expected namespace deletions
- level: Request
omitStages:
- RequestReceived
users:
- 'system:serviceaccount:kube-system:namespace-controller'
verbs:
- deletecollection
# Secrets, ConfigMaps, and TokenReviews can contain sensitive & binary data,
# so only log at the Metadata level.
- level: Metadata
omitStages:
- RequestReceived
resources:
- group: ""
resources:
- secrets
- configmaps
- group: authentication.k8s.io
resources:
- tokenreviews
# Get repsonses can be large; skip them.
- level: Request
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
verbs:
- get
- list
- watch
# Default level for known APIs
- level: RequestResponse
omitStages:
- RequestReceived
resources:
- group: ""
- group: admissionregistration.k8s.io
- group: apiextensions.k8s.io
- group: apiregistration.k8s.io
- group: apps
- group: authentication.k8s.io
- group: authorization.k8s.io
- group: autoscaling
- group: batch
- group: certificates.k8s.io
- group: extensions
- group: metrics.k8s.io
- group: networking.k8s.io
- group: policy
- group: rbac.authorization.k8s.io
- group: scheduling.k8s.io
- group: settings.k8s.io
- group: storage.k8s.io
# Default level for all other requests.
- level: Metadata
omitStages:
- RequestReceived
EOF
2、分发审计策略文件到3个master节点
[root@master1 ~]# cd /opt/install/kubeconfig
[root@master1 kubeconfig]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp audit-policy.yaml root@${node_ip}:/opt/k8s/etc/audit-policy.yaml
done
四、部署kube-apiserver服务
1、创建kube-apiserver systemd unit模板文件
- 模板文件名称 apiserver.service.template
[root@master1 ~]# cd /opt/install/service
[root@master1 service]# cat > apiserver.service.template <<EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/GoogleCloudPlatform/kubernetes
After=network.target
[Service]
WorkingDirectory=${K8S_DIR}/kube-apiserver
ExecStart=/opt/k8s/bin/kube-apiserver \\
--advertise-address=##NODE_IP## \\
--default-not-ready-toleration-seconds=360 \\
--default-unreachable-toleration-seconds=360 \\
--feature-gates=DynamicAuditing=true \\
--max-mutating-requests-inflight=2000 \\
--max-requests-inflight=4000 \\
--default-watch-cache-size=200 \\
--delete-collection-workers=2 \\
--encryption-provider-config=/opt/k8s/etc/encryption-config.yaml \\
--etcd-cafile=/opt/k8s/etc/cert/ca.pem \\
--etcd-certfile=/opt/k8s/etc/cert/apiserver.pem \\
--etcd-keyfile=/opt/k8s/etc/cert/apiserver-key.pem \\
--etcd-servers=${ETCD_ENDPOINTS} \\
--bind-address=##NODE_IP## \\
--secure-port=6443 \\
--tls-cert-file=/opt/k8s/etc/cert/apiserver.pem \\
--tls-private-key-file=/opt/k8s/etc/cert/apiserver-key.pem \\
--insecure-port=0 \\
--audit-dynamic-configuration \\
--audit-log-maxage=15 \\
--audit-log-maxbackup=3 \\
--audit-log-maxsize=100 \\
--audit-log-truncate-enabled \\
--audit-log-path=${K8S_DIR}/kube-apiserver/audit.log \\
--audit-policy-file=/opt/k8s/etc/audit-policy.yaml \\
--profiling \\
--anonymous-auth=false \\
--client-ca-file=/opt/k8s/etc/cert/ca.pem \\
--enable-bootstrap-token-auth \\
--requestheader-allowed-names="k8s-demo-aggregator" \\
--requestheader-client-ca-file=/opt/k8s/etc/cert/ca.pem \\
--requestheader-extra-headers-prefix="X-Remote-Extra-" \\
--requestheader-group-headers=X-Remote-Group \\
--requestheader-username-headers=X-Remote-User \\
--service-account-key-file=/opt/k8s/etc/cert/ca.pem \\
--authorization-mode=Node,RBAC \\
--runtime-config=api/all=true \\
--enable-admission-plugins=NodeRestriction \\
--allow-privileged=true \\
--apiserver-count=3 \\
--event-ttl=168h \\
--kubelet-certificate-authority=/opt/k8s/etc/cert/ca.pem \\
--kubelet-client-certificate=/opt/k8s/etc/cert/apiserver.pem \\
--kubelet-client-key=/opt/k8s/etc/cert/apiserver-key.pem \\
--kubelet-https=true \\
--kubelet-timeout=10s \\
--proxy-client-cert-file=/opt/k8s/etc/cert/aggregator-client.pem \\
--proxy-client-key-file=/opt/k8s/etc/cert/aggregator-client-key.pem \\
--service-cluster-ip-range=${SERVICE_CIDR} \\
--service-node-port-range=${NODE_PORT_RANGE} \\
--logtostderr=true \\
--v=2
Restart=on-failure
RestartSec=10
Type=notify
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF
- –requestheader-allowed-names：不能为空，值用逗号分割，这里设置为"k8s-demo-aggregator"
- 在–proxy-client-cert-file证书中配置CN名称为"k8s-demo-aggregator"
- –secret-port：https监听端口
- –insecure-port=0：关闭监听 http 非安全端口(8080)
- –authorization-mode=Node,RBAC、–anonymous-auth=false： 开启Node和RBAC授权模式，拒绝未授权的请求
- –kubelet-*：如果指定，则使用https访问kubelet APIs；需要为证书对应的用户用户定义RBAC规则，否则访问kubelet API时提示未授权
- 如果kube-apiserver机器没有运行kube-proxy，则还需要添加–enable-aggregator-routing=true参数
2、分发kube-apiserver systemd unit文件到3个master节点
[root@master1 ~]# cd /opt/install/service
[root@master1 service]# for (( i=0; i < 3; i++ ))
do
sed -e "s/##NODE_NAME##/${MASTER_NAMES[i]}/" -e "s/##NODE_IP##/${MASTER_IPS[i]}/" apiserver.service.template > apiserver-${MASTER_IPS[i]}.service
done
[root@master1 service]# ll apiserver*.service
-rw-r--r-- 1 root root 2645 12月 14 21:58 apiserver-192.168.66.10.service
-rw-r--r-- 1 root root 2645 12月 14 21:58 apiserver-192.168.66.11.service
-rw-r--r-- 1 root root 2645 12月 14 21:58 apiserver-192.168.66.12.service
[root@master1 service]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
scp apiserver-${node_ip}.service root@${node_ip}:/etc/systemd/system/kube-apiserver.service
done
3、启动kube-apiserver服务，并检查服务状态
# 启动kube-apiserver服务
[root@master1 ~]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl daemon-reload && systemctl enable kube-apiserver && systemctl restart kube-apiserver"
done
# 查看进程和端口
[root@master1 ~]# ss -lnpt | grep kube-api
LISTEN 0 128 192.168.66.10:6443 *:* users:(("kube-apiserver",pid=1034,fd=5))
# 查看服务是否启动成功
[root@master1 ~]# for node_ip in ${MASTER_IPS[@]}
do
echo ">>> ${node_ip}"
ssh root@${node_ip} "systemctl status kube-apiserver |grep 'Active:'"
done
#查看集群信息
[root@master1 ~]# kubectl cluster-info
Kubernetes master is running at https://127.0.0.1:8443
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
#查看命名空间
[root@master1 ~]# kubectl get all --all-namespaces
NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
default service/kubernetes ClusterIP 10.8.0.1 <none> 443/TCP 62s
# 查看组件状态，可略写为 kubectl get cs
[root@master1 ~]# kubectl get componentstatuses
NAME STATUS MESSAGE ERROR
scheduler Unhealthy Get http://127.0.0.1:10251/healthz: dial tcp 127.0.0.1:10251: connect: connection refused
controller-manager Unhealthy Get http://127.0.0.1:10252/healthz: dial tcp 127.0.0.1:10252: connect: connection refused
etcd-1 Healthy {"health":"true"}
etcd-0 Healthy {"health":"true"}
etcd-2 Healthy {"health":"true"}
#如果上述状态异常，可以查看日志
[root@master1 ~]# journalctl -u kube-apiserver
- kube-apiserver审计日志存放路径 /opt/k8s/kubernetes/kube-apiserver/audit.log ，如下所示：
{“kind”:“Event”,“apiVersion”:“audit.k8s.io/v1”,“level”:“Metadata”,“auditID”:“73922efd-2fe5-49ab-9175-460895f44e5b”,“stage”:“ResponseComplete”,“requestURI”:"/metrics",“verb”:“get”,“user”:{“username”:“system:serviceaccount:monitoring:prometheus-k8s”,“uid”:“d67c91eb-9377-499f-b75a-e0c93040e37a”,“groups”:[“system:serviceaccounts”,“system:serviceaccounts:monitoring”,“system:authenticated”]},“sourceIPs”:[“192.168.66.15”],“userAgent”:“Prometheus/2.22.1”,“responseStatus”:{“metadata”:{},“code”:200},“requestReceivedTimestamp”:“2021-01-19T09:18:36.962586Z”,“stageTimestamp”:“2021-01-19T09:18:36.988119Z”,“annotations”:{“authorization.k8s.io/decision”:“allow”,“authorization.k8s.io/reason”:“RBAC: allowed by ClusterRoleBinding “prometheus-k8s” of ClusterRole “prometheus-k8s” to ServiceAccount “prometheus-k8s/monitoring””}}
参考
- https://github.com/kubernetes-sigs/apiserver-builder-alpha/blob/master/docs/concepts/auth.md
附：K8s-Demo集群版本信息
| 组件 | 版本 | 命令 |
|---|---|---|
| kubernetes | 1.18.5 | kubectl version |
| docker-ce | 19.03.11 | docker version 或者 rpm -qa | grep docker |
| etcd | 3.4.3 | etcdctl version |
| calico | 3.13.3 | calico -v |
| coredns | 1.7.0 | coredns -version |
附：专栏链接
K8S-Demo集群实践00：搭建镜像仓库Harbor+安全扫描
K8S-Demo集群实践01：准备VMware虚拟机模板
K8S-Demo集群实践02：准备VMware虚拟机3台Master+3台Node
K8S-Demo集群实践03：准备集群各组件间HTTPS通讯需要的x509证书
K8S-Demo集群实践04：部署etcd三节点高可用集群
K8S-Demo集群实践05：安装kubectl并配置集群管理员账户
K8S-Demo集群实践06：部署kube-apiserver到master节点（3个无状态实例）
K8S-Demo集群实践07：kube-apiserver高可用方案
K8S-Demo集群实践08：部署高可用kube-controller-manager集群
K8S-Demo集群实践09：部署高可用kube-scheduler集群
K8S-Demo集群实践10：部署ipvs模式的kube-proxy组件
K8S-Demo集群实践11：部署ipvs模式的kube-kubelet组件
K8S-Demo集群实践12：部署Calico网络
K8S-Demo集群实践13：部署集群CoreDNS
K8S-Demo集群实践14：部署集群监控服务Metrics Server
K8S-Demo集群实践15：部署Kubernetes Dashboard
K8S-Demo集群实践16：部署Kube-Prometheus
K8S-Demo集群实践17：部署私有云盘owncloud（10.6版本）
K8S-Demo集群实践18：构建宇宙中第一个基础容器镜像
- 先用起来，通过操作实践认识k8s，积累多了自然就理解了
- 把理解的知识分享出来，自造福田，自得福缘
- 追求简单，容易使人理解，知识的上下文也是知识的一部分，例如版本，时间等
- 欢迎留言交流，也可以提出问题，一般在周末回复和完善文档
- Jason@vip.qq.com 2021-1-19。
772
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
