二进制部署K8S集群

最新推荐文章于 2024-09-25 09:55:24 发布

Marco_pei

最新推荐文章于 2024-09-25 09:55:24 发布

阅读量2k

点赞数 1

分类专栏： k8s 文章标签： kubernetes

本文链接：https://blog.csdn.net/weixin_43707346/article/details/124175535

版权

k8s 专栏收录该内容

1 篇文章 0 订阅

订阅专栏

初始化设置服务器

关闭防火墙

]# systemctl stop firewalld 
]# systemctl disable firewalld

关闭 selinux

]# sed -i 's/enforcing/disabled/' /etc/selinux/config # 永久关闭
]# setenforce 0 # 临时关闭

关闭 swap

]# swapoff -a # 临时关闭
]# sed -ri 's/.*swap.*/#&/' /etc/fstab # 永久关闭

根据规划设置主机名

]# hostnamectl set-hostname <hostname>

内核参数优化

]# cat > /etc/sysctl.d/k8s.conf << EOF
net.ipv4.tcp_keepalive_time=600 #此参数表示TCP发送keepalive探测消息的间隔时间(秒)
net.ipv4.tcp_keepalive_intvl=30 #tcp检查间隔时间（keepalive探测包的发送间隔）
net.ipv4.tcp_keepalive_probes=10  #tcp检查次数（如果对方不予应答，探测包的发送次数）
net.ipv6.conf.all.disable_ipv6=1 #禁用IPv6，修为0为启用IPv6
net.ipv6.conf.default.disable_ipv6=1 #禁用IPv6，修为0为启用IPv6
net.ipv6.conf.lo.disable_ipv6=1 #禁用IPv6，修为0为启用IPv6
net.ipv4.neigh.default.gc_stale_time=120 #ARP缓存条目超时
net.ipv4.conf.all.rp_filter=0  #默认为1，系统会严格校验数据包的反向路径，可能导致丢包
net.ipv4.conf.default.rp_filter=0 #不开启源地址校验
net.ipv4.conf.default.arp_announce=2 #始终使用与目的IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.conf.lo.arp_announce=2 #始终使用与目的IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.conf.all.arp_announce=2 #始终使用与目的IP地址对应的最佳本地IP地址作为ARP请求的源IP地址
net.ipv4.ip_local_port_range= 45001 65000 # 定义网络连接可用作其源（本地）端口的最小和最大端口的限制，同时适用于TCP和UDP连接。
net.ipv4.ip_forward=1 # 其值为0,说明禁止进行IP转发；如果是1,则说明IP转发功能已经打开。
net.ipv4.tcp_max_tw_buckets=6000 #配置服务器 TIME_WAIT 数量
net.ipv4.tcp_syncookies=1 #此参数应该设置为1，防止SYN Flood
net.ipv4.tcp_synack_retries=2 #表示回应第二个握手包（SYN+ACK包）给客户端IP后，如果收不到第三次握手包（ACK包），进行重试的次数（默认为5）
net.bridge.bridge-nf-call-ip6tables=1 # 是否在ip6tables链中过滤IPv6包
net.bridge.bridge-nf-call-iptables=1 # 二层的网桥在转发包时也会被iptables的FORWARD规则所过滤，这样有时会出现L3层的iptables rules去过滤L2的帧的问题
net.netfilter.nf_conntrack_max=2310720 #连接跟踪表的大小，建议根据内存计算该值CONNTRACK_MAX = RAMSIZE (in bytes) / 16384 / (x / 32)，并满足nf_conntrack_max=4*nf_conntrack_buckets，默认262144

net.ipv6.neigh.default.gc_thresh1=8192
net.ipv6.neigh.default.gc_thresh2=32768
net.ipv6.neigh.default.gc_thresh3=65536

#gc_thresh3 是表大小的绝对限制
#gc_thresh2 设置为等于系统的最大预期邻居条目数的值
#在这种情况下，gc_thresh3 应该设置为一个比 gc_thresh2 值高的值，例如，比 gc_thresh2 高 25%-50%，将其视为浪涌容量。
#gc_thresh1 提高到较大的值；此设置的作用是，如果表包含的条目少于 gc_thresh1，内核将永远不会删除（超时）过时的条目。

net.core.netdev_max_backlog=16384 # 每CPU网络设备积压队列长度
net.core.rmem_max = 16777216 # 所有协议类型读写的缓存区大小
net.core.wmem_max = 16777216 # 最大的TCP数据发送窗口大小
net.ipv4.tcp_max_syn_backlog = 8096 # 第一个积压队列长度
net.core.somaxconn = 32768 # 第二个积压队列长度
fs.inotify.max_user_instances=8192 # 表示每一个real user ID可创建的inotify instatnces的数量上限，默认128.
fs.inotify.max_user_watches=524288 # 同一用户同时可以添加的watch数目，默认8192。
fs.file-max=52706963 # 文件描述符的最大值
fs.nr_open=52706963 #设置最大微博号打开数
kernel.pid_max = 4194303 #最大进程数
net.bridge.bridge-nf-call-arptables=1 #是否在arptables的FORWARD中过滤网桥的ARP包
vm.swappiness=0 # 禁止使用 swap 空间，只有当系统 OOM 时才允许使用它
vm.overcommit_memory=1 # 不检查物理内存是否够用
vm.panic_on_oom=0 # 开启 OOM
vm.max_map_count = 262144 
EOF

]# vim  /etc/security/limits.conf  
*	soft	nofile	65535
*	hard	nofile	65535

配置hosts

]# cat >>  /etc/hosts << EOF
172.16.100.30 k8s-master  #IP+hostname
172.16.100.31 k8s-node1  #IP+hostname
172.16.100.32 k8s-node2  #IP+hostname
EOF

二进制部署k8s集群

利用cfssl工具生成ssl证书

]# wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
]# wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
]# wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
]# mv cfssl_linux-amd64  /usr/local/bin/cfssl
]# mv cfssljson_linux-amd64 /usr/localbin/cfssljson
]# mv cfssl-certinfo_linux-amd64 /usr/local/bin/cfssl-certinfo
]# chmod +x /usr/local/bin/cfssl*

生成etcd的ssl证书

]# mkdir /data/TLS/{etcd,k8s,calico}
]# cd  /data/TLS/etcd/
]#cat > ca-config.json << EOF
{
"signing": {
"default": {
"expiry": "87600h"
},
"profiles": {
"kubernetes": {
"expiry": "87600h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}
EOF

]# cat > ca-csr.json << EOF
{
"CN": "kubernetes",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "Beijing",
"ST": "Beijing",
"O": "k8s", 
"OU": "System"
}
]
}
EOF

]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
]# cat > server-csr.json << EOF
{
"CN": "kubernetes",
"hosts": [
"10.0.0.1",
"127.0.0.1",
"172.16.100.30",   #集群IP
"172.16.100.31",   #集群IP
"172.16.100.32",   #集群IP(集群IP可以根据架构规划多写几个IP)
"kubernetes", 
"kubernetes.default", 
"kubernetes.default.svc", 
"kubernetes.default.svc.cluster", 
"kubernetes.default.svc.cluster.local"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"L": "BeiJing",
"ST": "BeiJing",
"O": "k8s", 
"OU": "System"
}
]
}
EOF

]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server

部署etcd集群 (先下载好etcd包)

]# mkdir  -p  /data/etcd/{bin,logs,cfg,ssl}
]# tar -xf etcd-v3.4.9-linux-amd64.tar.gz && cd etcd-v3.4.9-linux-amd64
]# cp -r  etcd etcdctl  /data/etcd/bin/
]# cp -r  /data/TLS/etcd/ca*.pem  /data/TLS/etcd/server*.pem   /data/etcd/ssl/
]# cd /data/etcd/cfg
]# cat > etcd.conf << EOF
#[Member] 
ETCD_NAME="etcd-1" 
ETCD_DATA_DIR="/data/etcd/default.etcd" 
ETCD_LISTEN_PEER_URLS="https://172.16.100.30:2380" 
ETCD_LISTEN_CLIENT_URLS="https://172.16.100.30:2379" 
#[Clustering] 
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.100.30:2380" 
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.100.30:2379" 
ETCD_INITIAL_CLUSTER="etcd-1=https://172.16.100.30:2380,etcd-2=https://172.16.100.31:2380,etcd-3=https://172.16.100.32:2380" 
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster" 
ETCD_INITIAL_CLUSTER_STATE="new"
EOF

]# cat > /usr/lib/systemd/system/etcd.service << EOF
[Unit] 
Description=Etcd 
Server After=network.target 
After=network-online.target 
Wants=network-online.target 
[Service] 
Type=notify 
EnvironmentFile=/data/etcd/cfg/etcd.conf 
ExecStart=/data/etcd/bin/etcd \
--cert-file=/data/etcd/ssl/server.pem \
--key-file=/data/etcd/ssl/server-key.pem \
--peer-cert-file=/data/etcd/ssl/server.pem \
--peer-key-file=/data/etcd/ssl/server-key.pem \
--trusted-ca-file=/data/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/data/etcd/ssl/ca.pem
Restart=on-failure 
LimitNOFILE=65536 
[Install] 
WantedBy=multi-user.target
EOF

]# systemctl daemon-reload && systemctl enable etcd  && systemctl start etcd
注：将/data/etcd目录和etcd.service之际拷贝到集群其他服务器，并修改对应IP和ETCD_NAME就好了
ETCD_NAME：节点名称，集群中唯一 
ETCD_DATA_DIR：数据目录 
ETCD_LISTEN_PEER_URLS：集群通信监听地址 
ETCD_LISTEN_CLIENT_URLS：客户端访问监听地址 
ETCD_INITIAL_ADVERTISE_PEER_URLS：集群通告地址 
ETCD_ADVERTISE_CLIENT_URLS：客户端通告地址 
ETCD_INITIAL_CLUSTER：集群节点地址 
ETCD_INITIAL_CLUSTER_TOKEN：集群 
Token ETCD_INITIAL_CLUSTER_STATE：加入集群的当前状态，new 是新集群，existing 表示加入 已有集群

查看验证etcd集群状态

]# ETCDCTL_API=3 etcdctl \
--cacert=/opt/etcd/ssl/ca.pem \
--cert=/opt/etcd/ssl/server.pem \
--key=/opt/etcd/ssl/server-key.pem \
--endpoints="https://192.168.31.71:2379,https://192.168.31.72:2379,https://192.16 8.31.73:2379" \
endpoint health
出现一下信息证明集群部署成功：
https://192.168.31.71:2379 is healthy: successfully committed proposal: took = 8.154404ms
https://192.168.31.73:2379 is healthy: successfully committed proposal: took = 9.044117ms 
https://192.168.31.72:2379 is healthy: successfully committed proposal: took = 10.000825ms
注：因为etcd开启了TLS,访问查询etcd的数据都要加上证书,建议在/etc/bashrc添加：
alias etcdctl='ETCDCTL_API=3 etcdctl \
--cacert=/data/etcd/ssl/ca.pem  \
--cert=/data/etcd/ssl/server.pem \
--key=/data/etcd/ssl/server-key.pem \
--endpoints="https://172.16.100.30:2379,https://172.16.100.31:2379,https://172.16.100.32:2379"'

部署docker环境

]# wget https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz
]# tar -xf docker-19.03.9.tgz
]# cp -r  docker/*  /usr/local/bin/
]# cat > /usr/lib/systemd/system/docker.service << EOF 
[Unit]
Description=Docker Application Container Engine 
Documentation=https://docs.docker.com 
After=network-online.target firewalld.service 
Wants=network-online.target
[Service]
Type=notify 
ExecStart=/usr/local/bin/dockerd
ExecReload=/bin/kill -s HUP $MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target 
EOF
]# mkdir /etc/docker  /data/docker
]# cat > /etcd/dokcer/daemon.json << EOF
{
  "registry-mirrors": ["https://b9pmyelo.mirror.aliyuncs.com"],
  "data-root": "/data/docker"
}
EOF
]# systemctl daemon-reload && systemctl enable docker && systemctl start docker
注：以上的docker操作在所有集群机器操作，有网络的同样也可以yum安装

在master部署kube-apiserver组件

生成kube-apiserver的ssl证书

]# cd /data/TLS/k8s
]# cat > ca-config.json << EOF
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

]# cat > ca-csr.json << EOF
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

]# cat > server-csr.json << EOF
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
      "172.16.100.30",
      "172.16.100.31",
      "172.16.100.32",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "BeiJing",
            "ST": "BeiJing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

]# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -
]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server
]# mkdir -p /data/kubernetes/{bin,logs,cfg,logs}
]# cp -r ca*.pem server*.pem  /data/kubernetes/ssl/
]# wget https://dl.k8s.io/v1.20.15/kubernetes-server-linux-amd64.tar.gz
]# tar -xf  kubernetes-server-linux-amd64.tar.gz
]# cp -r  kubernetes/server/bin/kube*  /data/kubernetes/bin/
]# cp -r  kubernetes/server/bin/kube*  /usr/local/bin/
]# cd /data/kubernetes/cfg
]# cat > kube-apiserver.conf << EOF
KUBE_APISERVER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--etcd-servers=https://172.16.100.30:2379,https://172.16.100.31:2379,https://172.16.100.32:2379 \
--bind-address=172.16.100.30 \
--secure-port=6443 \
--advertise-address=172.16.100.30 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth \
--token-auth-file=/data/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/data/kubernetes/ssl/server.pem \
--kubelet-client-key=/data/kubernetes/ssl/server-key.pem \
--tls-cert-file=/data/kubernetes/ssl/server.pem \
--tls-private-key-file=/data/kubernetes/ssl/server-key.pem \
--client-ca-file=/data/kubernetes/ssl/ca.pem \
--service-account-key-file=/data/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/data/etcd/ssl/ca.pem \
--etcd-certfile=/data/etcd/ssl/server.pem \
--etcd-keyfile=/data/etcd/ssl/server-key.pem \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/data/kubernetes/logs/kubernetes-audit.log"
EOF

]# head -c 16 /dev/urandom | od -An -t x | tr -d ' '   ##生成token
]# cat > token.csv << EOF        ##将刚刚生成的token字串复制到文件
c47ffb939f5ca36231d9e3121a252940,kubelet-bootstrap,10001,"system:node- bootstrapper"
EOF

]# cat > /usr/lib/systemd/system/kube-apiserver.service << EOF
[Unit] 
Description=Kubernetes API Server 
Documentation=https://github.com/kubernetes/kubernetes
 
[Service] 
EnvironmentFile=/data/kubernetes/cfg/kube-apiserver.conf 
ExecStart=/data/kubernetes/bin/kube-apiserver \$KUBE_APISERVER_OPTS 
Restart=on-failure
 
[Install] 
WantedBy=multi-user.target
EOF

]# systemctl daemon-reload && systemctl enable kube-apiserver && systemctl start kube-apiserver
]# kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

在master部署kube-controller-manager组件

]# cat > kube-controller-manager.conf << EOF
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--leader-elect=true \
--master=127.0.0.1:8080 \
--bind-address=127.0.0.1 \
--allocate-node-cidrs=true \
--cluster-cidr=10.244.0.0/16 \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-signing-cert-file=/data/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/data/kubernetes/ssl/ca-key.pem \
--root-ca-file=/data/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/data/kubernetes/ssl/ca-key.pem \
--experimental-cluster-signing-duration=87600h0m0s"
EOF

]# cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit] 
Description=Kubernetes Controller Manager 
Documentation=https://github.com/kubernetes/kubernetes 

[Service] 
EnvironmentFile=/data/kubernetes/cfg/kube-controller-manager.conf 
ExecStart=/data/kubernetes/bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure
 
[Install] 
WantedBy=multi-user.target
EOF

]# systemctl daemon-reload && systemctl enable kube-controller-manager && systemctl start kube-controller-manager

在master部署kube-scheduler组件

]# cat > kube-scheduler.conf << EOF
KUBE_SCHEDULER_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--leader-elect \
--master=127.0.0.1:8080 \
--bind-address=127.0.0.1"
EOF

]# cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler 
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/data/kubernetes/cfg/kube-scheduler.conf 
ExecStart=/data/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS 
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

]# systemctl daemon-reload && systemctl enable kube-scheduler && systemctl start kube-scheduler
]# kubectl get cs  #查看集群状态

在master部署kubelet组件

]# cat > kubelet.conf  << EOF
[root@k8s-master cfg]# cat kubelet.conf 
KUBELET_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--hostname-override=k8s-master \
--network-plugin=cni \
--kubeconfig=/data/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/data/kubernetes/cfg/bootstrap.kubeconfig \
--config=/data/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/data/kubernetes/ssl \
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"
EOF

]# cat > kubelet-config.yaml << EOF
kind: KubeletConfiguration 
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0 
port: 10250 
readOnlyPort: 10255 
cgroupDriver: cgroupfs 
clusterDNS:
- 10.0.0.2 
clusterDomain: cluster.local
failSwapOn: false 
authentication:
  anonymous:
    enabled: false 
  webhook: 
    cacheTTL: 2m0s 
    enabled: true 
  x509: 
    clientCAFile: /data/kubernetes/ssl/ca.pem 
authorization: 
  mode: Webhook 
  webhook: 
    cacheAuthorizedTTL: 5m0s 
    cacheUnauthorizedTTL: 30s 
evictionHard: 
imagefs.available: 15% 
memory.available: 100Mi 
nodefs.available: 10% 
nodefs.inodesFree: 5% 
maxOpenFiles: 1000000 
maxPods: 110
EOF

生成bootstrap.kubeconfig 文件

]# KUBE_APISERVER="https://172.16.100.30:6443"
]# TOKEN="5a7a05908e6fff5ea912d4a2758be58c"
]# kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem  --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=bootstrap.kubeconfig
]# kubectl config set-credentials "kubelet-bootstrap" --token=${TOKEN} --kubeconfig=bootstrap.kubeconfig
]# kubectl config set-context default --cluster=kubernetes --user="kubelet-bootstrap" --kubeconfig=bootstrap.kubeconfig
]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
]# cat > /usr/lib/systemd/system/kubelet.service << EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
[Service]
EnvironmentFile=/data/kubernetes/cfg/kubelet.conf
ExecStart=/data/kubernetes/bin/kubelet $KUBELET_OPTS
Restart=on-failure
LimitNOFILE=65536
[Install]
WantedBy=multi-user.target
EOF

]# systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet
]# kubectl get csr   #查看 kubelet 证书请求
]# kubectl certificate approve <csrNAME>
]# kuebctl get node   #查看node状态

在master部署kube-proxy组件
生成kube-proxy的ssl证书

]# cat > /data/TLS/k8s/kube-proxy-csr.json << EOF
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN", 
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF
]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy
]# cp -r  /data/TLS/k8s/kube-proxy*.pem  /data/kubernetes/ssl/
]# cat > kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--config=/data/kubernetes/cfg/kube-proxy-config.yml"
EOF
]# cat > kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
 kubeconfig: /data/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: k8s-master
clusterCIDR: 10.0.0.0/24
EOF

生成proxy.kubeconfig 文件

]# KUBE_APISERVER="https://172.16.100.30:6443"
]# kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem  --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kube-proxy.kubeconfig
]# kubectl config set-credentials kube-proxy --client-certificate=/data/kubernetes/ssl/kube-proxy.pem --client-key=/data/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
]# kubectl config set-context default --cluster=kubernetes --user="kube-proxy" --kubeconfig=kube-proxy.kubeconfig
]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

]# systemctl daemon-reload && systemctl enable kube-proxy && systemctl start kube-proxy

部署CNI网络calico

]# mkdir -p /data/calico/{ssl,logs,cfg}
]# cd  /data/calico/cfg
]# curl https://docs.projectcalico.org/manifests/calico-etcd.yaml -O calico-etcd.yml
]# vim calico-etcd.yml
---
apiVersion: v1
kind: Secret
type: Opaque
metadata:
  name: calico-etcd-secrets
  namespace: kube-system
data:
  #etcd的ssl证书:ca.pem\server.pem\server-key.pem, 通过base64 -w 0去转换,将结果输入以下：
  #例:cat ca.pem | base64 -w  0
  etcd-key: "LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcFFJQkFBS0NBUUVBNGNUY2Y4em9rQXdZRVFXSnhHQjFPNzVaVWF5U1hlTExCQm5DSmdhSUtJYy83VXJrCjkrWHFZbkJvUm1JNXQzUjFGMlZnemhnZ2N0NnZKUGhaUGpFRGh2anI1NEptaVZ4UGNFZTRPQlJjTHA3dTZJL1IKV3puRXVDZm5zWXBlamgxT0hsbXZ1RFlPN2JudGJGQllXQUxjT2F5NFdzMmNBazZuelFlNHZycFBZcG03ZWN1aQp2bmx4VkFFRU1lV05LWm93RWNzbis2S2FOMjNxSHpZdWRwNUJZbDBiTzZNVzREeEJSMVYxei8raWkrTFRGVnN0CmVWQmNlSGJoNVZrblloZ1lGV0tCU3ZxalRoeTFVbVhyZitDRXppSVhOT0U0VFlCNVdhTWhWZkZ2QmpBYzVEOHEKUG1MS3VLZ2tNSlo5dTRwV0pVTnpuN250UXQ5RG9BOHUrL0w2ZXdJREFRQUJBb0lCQVFDcTlRRHJIV1MxUHhNeQpRSGxUNUo2aFFNQXQ0bmxxOG9NOGRhellVblhrQ3BaVHZ6U21xc2pUQmI5UUhLMEx4L21xWDYvd0g4RGlldEV4Ck00V1FYRmtKYVpCbzNBdDgxQk9yT0FPOUkxMnlSOU1zODBwYXcrRzhlU3N1KzFJaVZ2cUNiUE5za0RLNXZPS1YKOURrUlhBa2EremtXT1Q5N1Y4Z2tyMFlyMXJTcU9jYmhxaWhNRWpBUTNiM3VSN1BDc01WMU9aSTJyV0xYeTlObAoxU0tIeDVvMXFRSFlaWHFaUnk0ZU9xOHFOdVhLeDZPSDZhdzJwcGVaYS9HcmJ6dm1GdWpvS0VQV1FsM0NlSng4CjdmSkU5WXg0dldvVlVxTkpxZHdxUjRiSlNZbWMxdVVkRzJNVkRCS0dHTnZiU0o4RVZCUzNUR0k2Z2QzWTN6eGQKbWVoT3hvcUJBb0dCQVBTRm40ZU51MjlsT1V3cWxkckorSlZHZUtlYlMrMEZTMVU2Wk4vTUpqUWFTTVJ5bVBhZwpZOHdNaEFvemlEQjBjSWJBS1ZGQWpBQXdZYWdSbWRSZm1vdEZVVjV2ZmwvM0hXcnNvOHhPamhlaFp0cml2ZHRvCmdrdlcyTUR4RU8vSDZFTkZ2VlI3RUtTbldNVCtFOXlzV0orZmlGVklFVjlsSDBZaUxIZjl6bnFiQW9HQkFPeGQKNHhJMm1aRU1ENi8xM2VzMWRsVnI3S1NXbStXZFJvVStrd0trN1kzZzdBMlJRaElrVks0SFlmOStMM1BRODhHbQoxQTVNdExxS0NGclFlTjhwWkc3VFYzMHVkeUtqOE5kMHpHbUY4Y0hySXFrNW9aSFhOcXBKTzR6b2tFd0pocjVwCnE1dTNZTUtSWS84Q2MwNThjb2ZIU25lL0ZtdCs4ei8yQWMwSmVnMmhBb0dBYW5na1ZtbW9TNERQeWhKZzNidEQKdWZ2TlhXMkpTZE1jVWlmeTlGOTM0d2Z1MTFydXI3UjJ2OHBUVS8zTU53ejhVakFwelc5RmhtK0tsaHZUMTEwcApkYXJoR3pXQTJWaElQdDU3RStMQWpCbURKNXZDLzE0cUhjdVc1YXdScTlabms2TXlKUzdRdUdFRmpnRHp0UXAyCkxFclNtZytmUU9KUEU4S2Rpa0hCUGpFQ2dZRUEwT0NMT040dFNVUEtYU28rYVl2K1BiQzVHQjNNT05hS3FsZEkKM081WXk2ZDNrdW5KNUhSY3JNbnpiUy9heVZOZkJjUGk0NXdmbmpVNit0Mzk0dUFXVStYS0MrTFMvemEzTC8rVQpZTEF3bTdpcUViZlBNeTFucm9ZMjdPZmNGSVhhb0V5TGpYazVOZGY3OFMvK0s5N0g2M3RQTUpFYVEvYVZDZkhoClY0dEhZK0VDZ1lFQXFCeW1qaW43ZWVlVHBMMU95OTR1bGVqTnNzZy8zcmQrTk90eHVxdG1TZ2hOTW5aS3ZtQWoKNWYzMElPbXNGNUhoQU9aRDlFNTIxcDNOMGpVeXRGS25wSVVPMmlxWTVwNTNDWFVMRUxZNHFhZDlZdGZ6SHEvOQp3cG1zOTB2ZlhQKy9Kd1NQWUpsM0prZ1g4THVQck9ibnQ4eFB5ckVjekZzZHh5bmZLcmpacFRzPQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo="
  etcd-cert: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURyekNDQXBlZ0F3SUJBZ0lVSEd0MDRQQUM2dnRuMkQwMHprVk9GTFZ4KzJJd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1F6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEVEQU9CZ05WQkFNVEIyVjBZMlFnUTBFd0hoY05Nakl3TXpJek1EWXdNREF3V2hjTk16SXdNekl3Ck1EWXdNREF3V2pCQU1Rc3dDUVlEVlFRR0V3SkRUakVRTUE0R0ExVUVDQk1IUW1WcFNtbHVaekVRTUE0R0ExVUUKQnhNSFFtVnBTbWx1WnpFTk1Bc0dBMVVFQXhNRVpYUmpaRENDQVNJd0RRWUpLb1pJaHZjTkFRRUJCUUFEZ2dFUApBRENDQVFvQ2dnRUJBT0hFM0gvTTZKQU1HQkVGaWNSZ2RUdStXVkdza2wzaXl3UVp3aVlHaUNpSFArMUs1UGZsCjZtSndhRVppT2JkMGRSZGxZTTRZSUhMZXJ5VDRXVDR4QTRiNDYrZUNab2xjVDNCSHVEZ1VYQzZlN3VpUDBWczUKeExnbjU3R0tYbzRkVGg1WnI3ZzJEdTI1N1d4UVdGZ0MzRG1zdUZyTm5BSk9wODBIdUw2NlQyS1p1M25Mb3I1NQpjVlFCQkRIbGpTbWFNQkhMSi91aW1qZHQ2aDgyTG5hZVFXSmRHenVqRnVBOFFVZFZkYy8vb292aTB4VmJMWGxRClhIaDI0ZVZaSjJJWUdCVmlnVXI2bzA0Y3RWSmw2My9naE00aUZ6VGhPRTJBZVZtaklWWHhid1l3SE9RL0tqNWkKeXJpb0pEQ1dmYnVLVmlWRGM1KzU3VUxmUTZBUEx2dnkrbnNDQXdFQUFhT0JuVENCbWpBT0JnTlZIUThCQWY4RQpCQU1DQmFBd0hRWURWUjBsQkJZd0ZBWUlLd1lCQlFVSEF3RUdDQ3NHQVFVRkJ3TUNNQXdHQTFVZEV3RUIvd1FDCk1BQXdIUVlEVlIwT0JCWUVGQm4zSUZ5eWFXWUMrVXJaOW5XUnF1YVJGd0dyTUI4R0ExVWRJd1FZTUJhQUZPYU0KTU0rZ0pnNFRqUUZ2dmNVOXJxY0FjNXAwTUJzR0ExVWRFUVFVTUJLSEJLd1FaQjZIQkt3UVpCK0hCS3dRWkNBdwpEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBR25Zalp0OWVFMTJXUlJyRVgxTVpHQWZ4NHNTVHlGa2FqeWQvYTlsCnNiSU42UHN2R1VERjFRbnlNaE1raVdubHc0UFdnZTN0b1pRdGpGYTYyMzlTdTdSVWI1d1FldWZWSkFXcmRwT08KTitOVjZJazVSWWpGdzFDUEFWaE5WN0IwVS9BUHVKOWhBR3N5Ui9VdHJ6ekZ4SWVIc25rTTY2RDN5M25QVFdTVgpFa1lZejdIcU5zb1lOSW1MckpHbmFCM2o5OUFLSG4zanJ4cXU4bDduYy9EcGpkNDhZRUM4WXBFejZJTDAzcnRWCkpZN2JuQUVScE9yYmJCbWZvck9wRWEzRUpYOEh6VStTSTBwVHA0dXQ3RUpsR3h3ZHgxbDhiU1kwakZFNkxzTDAKZS9pMmljdldnNlJFRU53emlPWkxlRXY3WmN2bHEzaktKcFMxWWwvN2NETk5QQmM9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K"
  etcd-ca: "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURWakNDQWo2Z0F3SUJBZ0lVVFcraEs4NmNoOGFhQzdVNFg2MCtVdmw3eXlBd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1F6RUxNQWtHQTFVRUJoTUNRMDR4RURBT0JnTlZCQWdUQjBKbGFXcHBibWN4RURBT0JnTlZCQWNUQjBKbAphV3BwYm1jeEVEQU9CZ05WQkFNVEIyVjBZMlFnUTBFd0hoY05Nakl3TXpJek1ETTFNREF3V2hjTk1qY3dNekl5Ck1ETTFNREF3V2pCRE1Rc3dDUVlEVlFRR0V3SkRUakVRTUE0R0ExVUVDQk1IUW1WcGFtbHVaekVRTUE0R0ExVUUKQnhNSFFtVnBhbWx1WnpFUU1BNEdBMVVFQXhNSFpYUmpaQ0JEUVRDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRApnZ0VQQURDQ0FRb0NnZ0VCQUxLV2pvWWR5M2NacUdDbk12blFQdGJPM2N2OUJXaGtVd0hrcENlZlM5cU5aenNECisrWUhhVElxL3V3U3cwL2g0aXRxc0RzZloxb0ZVYTd1dFU2Rzd2WEtWZXp2TC9ReTNETVQyY2lQWXNUWFJqVXEKbDQ1eWFVRnFGSkJTUklZWmcrci8ycmI1U3ROM2JTdlAvMnNvTVhHcUphaHM2aGdoKzVLRm5pRkwva3kwcVdGaQpLcFB2am1lUFpuTTJCTi9PaGdJbUczZW1wUytPdDBYS1NxNkNlNzkzS0hzSlJlRHAxRE5mTjRJZjFvUFFJclkrCnBTbjBRbVdFNUlqYXdjZEJlVWtPUFZEQWltVEY4V09wQjNVdU91SVFBMHdycHRQNnRidXVaQjZKUUlIaWtHT2kKSzhYMm91YTNHWkF6K1ByRGhYUElsaXpFTDBzT1FvYU8rdGVlVDQwQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFILwpCQVFEQWdFR01BOEdBMVVkRXdFQi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZPYU1NTStnSmc0VGpRRnZ2Y1U5CnJxY0FjNXAwTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBc21Pc1dEdWVvZEI2bEZ5RzRiRlp2eG1jWEJ5ZUMKVmlrZmdJQy9WVTNTQkZ2ZUk2M1JRY1ZoVVcvM3N0cWhPdmNCK0doRGMreklLY1hRMWJEQi85RWI1Y09XeUhOYgpoMDZmTE9mWTFJNjcwMDZqaVA3ZTdLNVlGSWdudjVzS2d1OENWelhjVUd2OXN3QjRnWGtzTEdYQVR4SURJSDBxCjNKQ3o5TEVrdkpKUDdnd0VHR0RBRHk2SEk0TXhDbmdKUDUvUEpvaXQ5NFhFdWI5TWwrdjY1b1gzT09WTVU3RG0KNlhWRFR2dzVCWWlVOTNFRG4rZ2xsWDJweFoybTgwbGpFc0Q0R1BZb2ptU0RlYXZpajA5K1U3cEVLN3JYRUJsdQpsMHl6eVIvNnNtK29OaHhPVUg0V0tiVmVpVWYvd3YyR1p6eldwN01tTWs5N2lQNlA3VVozVm5hbAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg=="  
  
---
kind: ConfigMap
apiVersion: v1
metadata:
  name: calico-config
  namespace: kube-system
data:
  etcd_endpoints: "https://172.16.100.30:2379,https://172.16.100.31:2379,https://172.16.100.32:2379" #ETCD集群
  etcd_ca: "/calico-secrets/etcd-ca"   # "/calico-secrets/etcd-ca"
  etcd_cert: "/calico-secrets/etcd-cert" # "/calico-secrets/etcd-cert"
  etcd_key: "/calico-secrets/etcd-key"  # "/calico-secrets/etcd-key"

  typha_service_name: "none"

  calico_backend: "bird"
  veth_mtu: "0"

  cni_network_config: |-
    {
      "name": "k8s-pod-network",
      "cniVersion": "0.3.1",
      "plugins": [
        {
          "type": "calico",
          "log_level": "info",
          "log_file_path": "/data/calico/logs/cni.log",
          "etcd_endpoints": "__ETCD_ENDPOINTS__",
          "etcd_key_file": "__ETCD_KEY_FILE__",
          "etcd_cert_file": "__ETCD_CERT_FILE__",
          "etcd_ca_cert_file": "__ETCD_CA_CERT_FILE__",
          "mtu": __CNI_MTU__,
          "ipam": {
              "type": "calico-ipam"
          },
          "policy": {
              "type": "k8s"
          },
          "kubernetes": {
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          }
        },
        {
          "type": "portmap",
          "snat": true,
          "capabilities": {"portMappings": true}
        },
        {
          "type": "bandwidth",
          "capabilities": {"bandwidth": true}
        }
      ]
    }

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
rules:
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
      - serviceaccounts
    verbs:
      - watch
      - list
      - get
  - apiGroups: ["networking.k8s.io"]
    resources:
      - networkpolicies
    verbs:
      - watch
      - list
	  
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-kube-controllers
subjects:
- kind: ServiceAccount
  name: calico-kube-controllers
  namespace: kube-system

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-node
rules:
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
    verbs:
      - get
  - apiGroups: ["discovery.k8s.io"]
    resources:
      - endpointslices
    verbs:
      - watch 
      - list
  - apiGroups: [""]
    resources:
      - endpoints
      - services
    verbs:
      - watch
      - list
  - apiGroups: [""]
    resources:
      - configmaps
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - nodes/status
    verbs:
      - patch

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: calico-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node
subjects:
- kind: ServiceAccount
  name: calico-node
  namespace: kube-system

---
kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: calico-node
  namespace: kube-system
  labels:
    k8s-app: calico-node
spec:
  selector:
    matchLabels:
      k8s-app: calico-node
  updateStrategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  template:
    metadata:
      labels:
        k8s-app: calico-node
    spec:
      nodeSelector:
        kubernetes.io/os: linux
      hostNetwork: true
      tolerations:
        - effect: NoSchedule
          operator: Exists
        - key: CriticalAddonsOnly
          operator: Exists
        - effect: NoExecute
          operator: Exists
      serviceAccountName: calico-node
      terminationGracePeriodSeconds: 0
      priorityClassName: system-node-critical
      initContainers:
        - name: install-cni
          image: docker.io/calico/cni:v3.22.1
          command: ["/opt/cni/bin/install"]
          envFrom:
          - configMapRef:
              name: kubernetes-services-endpoint
              optional: true
          env:
            - name: CNI_CONF_NAME
              value: "10-calico.conflist"
            - name: CNI_NETWORK_CONFIG
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: cni_network_config
            - name: ETCD_ENDPOINTS
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_endpoints
            - name: CNI_MTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            - name: SLEEP
              value: "false"
          volumeMounts:
            - mountPath: /host/opt/cni/bin
              name: cni-bin-dir
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
            - mountPath: /calico-secrets
              name: etcd-certs
          securityContext:
            privileged: true
        - name: flexvol-driver
          image: docker.io/calico/pod2daemon-flexvol:v3.22.1
          volumeMounts:
          - name: flexvol-driver-host
            mountPath: /host/driver
          securityContext:
            privileged: true
      containers:
        - name: calico-node
          image: docker.io/calico/node:v3.22.1
          envFrom:
          - configMapRef:
              name: kubernetes-services-endpoint
              optional: true
          env:
            - name: ETCD_ENDPOINTS
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_endpoints
            - name: ETCD_CA_CERT_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_ca
            - name: ETCD_KEY_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_key
            - name: ETCD_CERT_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_cert
            - name: CALICO_K8S_NODE_REF
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
            - name: CALICO_NETWORKING_BACKEND
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: calico_backend
            - name: CLUSTER_TYPE
              value: "k8s,bgp"
            - name: IP
              value: "autodetect"
            - name: CALICO_IPV4POOL_IPIP
              value: "Always"
            - name: CALICO_IPV4POOL_VXLAN
              value: "Never"
            - name: FELIX_IPINIPMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            - name: FELIX_VXLANMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
            - name: FELIX_WIREGUARDMTU
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: veth_mtu
			#修改k8s的IP
            - name: CALICO_IPV4POOL_CIDR
              value: "10.0.0.0/24"
            - name: CALICO_DISABLE_FILE_LOGGING
              value: "true"
            - name: FELIX_DEFAULTENDPOINTTOHOSTACTION
              value: "ACCEPT"
            - name: FELIX_IPV6SUPPORT
              value: "false"
            - name: FELIX_HEALTHENABLED
              value: "true"
			#新增以下：
            - name: KUBERNETES_SERVICE_HOST
              value: "172.16.100.30"
            - name: KUBERNETES_SERVICE_PORT
              value: "6443"
            - name: KUBERNETES_SERVICE_PORT_HTTPS
              value: "6443"
          securityContext:
            privileged: true
          resources:
            requests:
              cpu: 250m
          lifecycle:
            preStop:
              exec:
                command:
                - /bin/calico-node
                - -shutdown
          livenessProbe:
            exec:
              command:
              - /bin/calico-node
              - -felix-live
              - -bird-live
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
            timeoutSeconds: 10
          readinessProbe:
            exec:
              command:
              - /bin/calico-node
              - -felix-ready
              - -bird-ready
            periodSeconds: 10
            timeoutSeconds: 10
          volumeMounts:
            - mountPath: /host/etc/cni/net.d
              name: cni-net-dir
              readOnly: false
            - mountPath: /lib/modules
              name: lib-modules
              readOnly: true
            - mountPath: /run/xtables.lock
              name: xtables-lock
              readOnly: false
            - mountPath: /var/run/calico
              name: var-run-calico
              readOnly: false
            - mountPath: /var/lib/calico
              name: var-lib-calico
              readOnly: false
            - mountPath: /calico-secrets
              name: etcd-certs
            - name: policysync
              mountPath: /var/run/nodeagent
            - name: sysfs
              mountPath: /sys/fs/
              mountPropagation: Bidirectional
            - name: cni-log-dir
              mountPath: /var/log/calico/cni
              readOnly: true
      volumes:
        - name: lib-modules
          hostPath:
            path: /lib/modules
        - name: var-run-calico
          hostPath:
            path: /var/run/calico
        - name: var-lib-calico
          hostPath:
            path: /var/lib/calico
        - name: xtables-lock
          hostPath:
            path: /run/xtables.lock
            type: FileOrCreate
        - name: sysfs
          hostPath:
            path: /sys/fs/
            type: DirectoryOrCreate
        - name: cni-bin-dir
          hostPath:
            path: /opt/cni/bin
        - name: cni-net-dir
          hostPath:
            path: /etc/cni/net.d
        - name: cni-log-dir
          hostPath:
            path: /var/log/calico/cni
        - name: etcd-certs
          secret:
            secretName: calico-etcd-secrets
            defaultMode: 0400
        - name: policysync
          hostPath:
            type: DirectoryOrCreate
            path: /var/run/nodeagent
        - name: flexvol-driver-host
          hostPath:
            type: DirectoryOrCreate
            path: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds
			
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node
  namespace: kube-system

---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: calico-kube-controllers
  namespace: kube-system
  labels:
    k8s-app: calico-kube-controllers
spec:
  replicas: 1
  selector:
    matchLabels:
      k8s-app: calico-kube-controllers
  strategy:
    type: Recreate
  template:
    metadata:
      name: calico-kube-controllers
      namespace: kube-system
      labels:
        k8s-app: calico-kube-controllers
    spec:
      nodeSelector:
        kubernetes.io/os: linux
      tolerations:
        - key: CriticalAddonsOnly
          operator: Exists
        - key: node-role.kubernetes.io/master
          effect: NoSchedule
      serviceAccountName: calico-kube-controllers
      priorityClassName: system-cluster-critical
      hostNetwork: true
      containers:
        - name: calico-kube-controllers
          image: docker.io/calico/kube-controllers:v3.22.1
          env:
            - name: ETCD_ENDPOINTS
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_endpoints
            - name: ETCD_CA_CERT_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_ca
            - name: ETCD_KEY_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_key
            - name: ETCD_CERT_FILE
              valueFrom:
                configMapKeyRef:
                  name: calico-config
                  key: etcd_cert
            - name: ENABLED_CONTROLLERS
              value: policy,namespace,serviceaccount,workloadendpoint,node
          volumeMounts:
            - mountPath: /calico-secrets
              name: etcd-certs
          livenessProbe:
            exec:
              command:
              - /usr/bin/check-status
              - -l
            periodSeconds: 10
            initialDelaySeconds: 10
            failureThreshold: 6
            timeoutSeconds: 10
          readinessProbe:
            exec:
              command:
              - /usr/bin/check-status
              - -r
            periodSeconds: 10
      volumes:
        - name: etcd-certs
          secret:
            secretName: calico-etcd-secrets
            defaultMode: 0440

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-kube-controllers
  namespace: kube-system

---
apiVersion: policy/v1beta1
kind: PodDisruptionBudget
metadata:
  name: calico-kube-controllers
  namespace: kube-system
  labels:
    k8s-app: calico-kube-controllers
spec:
  maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: calico-kube-controllers


]# kubectl apply -f  calico-etcd.yml
]# kubectl get pods -n kube-system
]# kubectl get node

##授权 apiserver 访问 kubelet
]# cat > apiserver-to-kubelet-rbac.yml << EOF
apiVersion: rbac.authorization.k8s.io/v1 
kind: ClusterRole
metadata: 
  annotations: 
    rbac.authorization.kubernetes.io/autoupdate: "true" 
  labels: 
    kubernetes.io/bootstrapping: rbac-defaults 
  name: system:kube-apiserver-to-kubelet 
rules: 
  - apiGroups: 
      - "" 
    resources: 
      - nodes/proxy 
      - nodes/stats 
      - nodes/log 
      - nodes/spec 
      - nodes/metrics 
      - pods/log 
    verbs: 
      - "*" 
--- 
apiVersion: rbac.authorization.k8s.io/v1 
kind: ClusterRoleBinding 
metadata: 
  name: system:kube-apiserver 
  namespace: "" 
roleRef: 
  apiGroup: rbac.authorization.k8s.io 
  kind: ClusterRole 
  name: system:kube-apiserver-to-kubelet 
subjects:
  - apiGroup: rbac.authorization.k8s.io 
    kind: User 
    name: kubernetes
EOF
]# kubectl apply -f apiserver-to-kubelet-rbac.yml

在worker node部署kubelet、kube-proxy组件
##复制mater上的kubernetes过来,修改kubelet.conf和kube-proxy-config.yml的hostname
##生成新的kubelet证书和kubeconfig文件

]# KUBE_APISERVER="https://172.16.100.30:6443"
]# TOKEN="5a7a05908e6fff5ea912d4a2758be58c"
]# kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem  --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=bootstrap.kubeconfig
]# kubectl config set-credentials "kubelet-bootstrap" --token=${TOKEN} --kubeconfig=bootstrap.kubeconfig
]# kubectl config set-context default --cluster=kubernetes --user="kubelet-bootstrap" --kubeconfig=bootstrap.kubeconfig
]# kubectl config use-context default --kubeconfig=bootstrap.kubeconfig

]# KUBE_APISERVER="https://172.16.100.30:6443"
]# kubectl config set-cluster kubernetes --certificate-authority=/data/kubernetes/ssl/ca.pem  --embed-certs=true --server=${KUBE_APISERVER} --kubeconfig=kube-proxy.kubeconfig
]# kubectl config set-credentials kube-proxy --client-certificate=/data/kubernetes/ssl/kube-proxy.pem --client-key=/data/kubernetes/ssl/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
]# kubectl config set-context default --cluster=kubernetes --user="kube-proxy" --kubeconfig=kube-proxy.kubeconfig
]# kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
]# cat > kube-proxy.conf << EOF
KUBE_PROXY_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--config=/data/kubernetes/cfg/kube-proxy-config.yml"
EOF
]# cat > kube-proxy-config.yml << EOF
kind: KubeProxyConfiguration
apiVersion: kubeproxy.config.k8s.io/v1alpha1
bindAddress: 0.0.0.0
metricsBindAddress: 0.0.0.0:10249
clientConnection:
 kubeconfig: /data/kubernetes/cfg/kube-proxy.kubeconfig
hostnameOverride: k8s-node1
clusterCIDR: 10.0.0.0/24
EOF
]# cat > /usr/lib/systemd/system/kube-proxy.service << EOF
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/data/kubernetes/cfg/kube-proxy.conf
ExecStart=/data/kubernetes/bin/kube-proxy $KUBE_PROXY_OPTS 
Restart=on-failure LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOF
]# systemctl daemon-reload && systemctl enable kube-proxy && systemctl start kube-proxy

]# cat > kubelet.conf << EOF
KUBELET_OPTS="--logtostderr=false \
--v=2 \
--log-dir=/data/kubernetes/logs \
--hostname-override=k8s-node1 \  
--network-plugin=cni \
--kubeconfig=/data/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/data/kubernetes/cfg/bootstrap.kubeconfig \
--config=/data/kubernetes/cfg/kubelet-config.yml \
--cert-dir=/data/kubernetes/ssl \
--pod-infra-container-image=lizhenliang/pause-amd64:3.0"
EOF
]# cat > kubelet-config.yml << EOF
kind: KubeletConfiguration 
apiVersion: kubelet.config.k8s.io/v1beta1
address: 0.0.0.0 
port: 10250 
readOnlyPort: 10255 
cgroupDriver: cgroupfs 
clusterDNS:
- 10.0.0.2 
clusterDomain: cluster.local
failSwapOn: false 
authentication:
  anonymous:
    enabled: false 
  webhook: 
    cacheTTL: 2m0s 
    enabled: true 
  x509: 
    clientCAFile: /data/kubernetes/ssl/ca.pem 
authorization: 
  mode: Webhook 
  webhook: 
    cacheAuthorizedTTL: 5m0s 
    cacheUnauthorizedTTL: 30s 
evictionHard: 
imagefs.available: 15% 
memory.available: 100Mi 
nodefs.available: 10% 
nodefs.inodesFree: 5% 
maxOpenFiles: 1000000 
maxPods: 110
EOF
]# systemctl daemon-reload && systemctl enable kubelet && systemctl start kubelet