集群环境:C1(master01),C2(node02)
1、安装dokcer并启动
sudo apt-get install docker.io
sudo service docker start
2、关闭firewilld和selinux
查看selinux状态:/usr/sbin/sestatus -v
临时关闭:setenforce 0
永久关闭:修改/etc/selinux/config文件中设置SELINUX=disabled ,然后重启服务器。
显示防火墙和端口的侦听状态:ufw status
关闭的防火墙 :ufw disable
开启防火墙:ufw enable
3、配置hosts并同步时钟
hostnamectl set-hostname master01
hostnamectl set-hostname node01
echo -e "192.168.25.30 master01\n192.168.25.31 node01" >> /etc/hosts
4、待docker安装完成,配置docker.service(每个节点)
vi /lib/systemd/system/docker.service
#找到ExecStart=xxx,在这行上面加入一行,内容如下:(k8s的网络需要)
ExecStartPost=/sbin/iptables -I FORWARD -s 0.0.0.0/0 -j ACCEPT
5、重读systemctl并启动docker
systemctl daemon-reload
systemctl start docker
6、设置系统参数 - 允许路由转发,不对bridge的数据进行处理
#写入配置文件
cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
#生效配置文件
sysctl -p /etc/sysctl.d/k8s.conf
7、这里k8s采用二进制安装方式
#安装cfssl(所有节点) wget -q --timestamping \ https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 \ https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 mv cfssl_linux-amd64 /usr/local/bin/cfssl mv cfssljson_linux-amd64 /usr/local/bin/cfssljson cfssl version #生成根证书(主节点) mkdir -p /etc/kubernetes/ca vim /etc/kubernetes/ca/ca-config.json #输入一下内容 根据官网模板修改 { "signing": { "default": { "expiry": "87600h" }, "profiles": { "kubernetes": { "usages": [ "signing", "key encipherment", "server auth", "client auth" ], "expiry": "87600h" } } } vim /etc/kubernetes/ca/ca-csr.json #输入一下内容 根据官网模板修改 { "CN": "kubernetes", "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "k8s", "OU": "System" } ] } #生成证书和秘钥 cd /etc/kubernetes/ca cfssl gencert -initca ca-csr.json | cfssljson -bare ca ls ca-config.json ca.csr ca-csr.json ca-key.pem ca.pem #部署etcd #生成etcd证书 mkdir -p /var/lib/etcd #工作目录 mkdir -p /etc/kubernetes/ca/etcd vim /etc/kubernetes/ca/etcd/etcd-csr.json #输入一下内容 根据官网模板修改 { "CN": "etcd", "hosts": [ "127.0.0.1", "192.168.66.135" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "k8s", "OU": "System" } ] } cd /etc/kubernetes/ca/etcd/ cfssl gencert \ -ca=/etc/kubernetes/ca/ca.pem \ -ca-key=/etc/kubernetes/ca/ca-key.pem \ -config=/etc/kubernetes/ca/ca-config.json \ -profile=kubernetes etcd-csr.json | cfssljson -bare etcd ls etcd.csr etcd-csr.json etcd-key.pem etcd.pem #配置etcd为系统服务 vim /lib/systemd/system/etcd.service #输入如下内容,注意修改主机ip和etcd路径以及ca证书位置 [Unit] Description=Etcd Server After=network.target After=network-online.target Wants=network-online.target Documentation=https://github.com/coreos [Service] Type=notify WorkingDirectory=/var/lib/etcd/ ExecStart=/root/bin/etcd \ --name=192.168.66.135 \ --listen-client-urls=https://192.168.66.135:2379,http://127.0.0.1:2379 \ --advertise-client-urls=https://192.168.66.135:2379 \ --data-dir=/var/lib/etcd \ --listen-peer-urls=https://192.168.66.135:2380 \ --initial-advertise-peer-urls=https://192.168.66.135:2380 \ --cert-file=/etc/kubernetes/ca/etcd/etcd.pem \ --key-file=/etc/kubernetes/ca/etcd/etcd-key.pem \ --peer-cert-file=/etc/kubernetes/ca/etcd/etcd.pem \ --peer-key-file=/etc/kubernetes/ca/etcd/etcd-key.pem \ --trusted-ca-file=/etc/kubernetes/ca/ca.pem \ --peer-trusted-ca-file=/etc/kubernetes/ca/ca.pem Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target #启动服务 systemctl daemon-reload systemctl start etcd #验证etcd ETCDCTL_API=3 etcdctl \ --endpoints=https://192.168.66.135:2379 \ --cacert=/etc/kubernetes/ca/ca.pem \ --cert=/etc/kubernetes/ca/etcd/etcd.pem \ --key=/etc/kubernetes/ca/etcd/etcd-key.pem \ endpoint health #出现类似如下结果,说明配置成功 https://192.168.66.135:2379 is healthy: successfully committed proposal: took = 5.194485ms #如果有异常可以查看日志 journalctl -f -u etcd.service # 部署APIServer(主节点) #生成证书 mkdir -p /etc/kubernetes/ca/kubernetes #准备csr,类似etcd cat kubernetes-csr.json { "CN": "kubernetes", "hosts": [ "127.0.0.1", "192.168.66.135", "10.68.0.1", "kubernetes", "kubernetes.default", "kubernetes.default.svc", "kubernetes.default.svc.cluster", "kubernetes.default.svc.cluster.local" ], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "k8s", "OU": "System" } ] } cd /etc/kubernetes/ca/kubernetes/ #使用根证书(ca.pem)签发kubernetes证书 cfssl gencert \ -ca=/etc/kubernetes/ca/ca.pem \ -ca-key=/etc/kubernetes/ca/ca-key.pem \ -config=/etc/kubernetes/ca/ca-config.json \ -profile=kubernetes kubernetes-csr.json | cfssljson -bare kubernetes ls kubernetes.csr kubernetes-csr.json kubernetes-key.pem kubernetes.pem #生成随机token head -c 16 /dev/urandom | od -An -t x | tr -d ' ' 11f4d7eafcf06965e413a409d35c3893 #按照固定格式写入token.csv,注意替换token内容 echo "11f4d7eafcf06965e413a409d35c3893,kubelet-bootstrap,10001,\"system:kubelet-bootstrap\"" > /etc/kubernetes/ca/kubernetes/token.csv #配置apiserver为系统服务 vim /lib/systemd/system/kube-apiserver.service #输入如下内容,注意修改ip和相关路径 [Unit] Description=Kubernetes API Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] ExecStart=/root/bin/kube-apiserver \ --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota,NodeRestriction \ --insecure-bind-address=127.0.0.1 \ --kubelet-https=true \ --bind-address=192.168.66.135 \ --authorization-mode=Node,RBAC \ --runtime-config=rbac.authorization.k8s.io/v1 \ --enable-bootstrap-token-auth \ --token-auth-file=/etc/kubernetes/ca/kubernetes/token.csv \ --tls-cert-file=/etc/kubernetes/ca/kubernetes/kubernetes.pem \ --tls-private-key-file=/etc/kubernetes/ca/kubernetes/kubernetes-key.pem \ --client-ca-file=/etc/kubernetes/ca/ca.pem \ --service-account-key-file=/etc/kubernetes/ca/ca-key.pem \ --etcd-cafile=/etc/kubernetes/ca/ca.pem \ --etcd-certfile=/etc/kubernetes/ca/kubernetes/kubernetes.pem \ --etcd-keyfile=/etc/kubernetes/ca/kubernetes/kubernetes-key.pem \ --service-cluster-ip-range=10.68.0.0/16 \ --service-node-port-range=20000-40000 \ --etcd-servers=https://192.168.66.135:2379 \ --enable-swagger-ui=true \ --allow-privileged=true \ --audit-log-maxage=30 \ --audit-log-maxbackup=3 \ --audit-log-maxsize=100 \ --audit-log-path=/var/lib/audit.log \ --event-ttl=1h \ --v=2 Restart=on-failure RestartSec=5 Type=notify LimitNOFILE=65536 [Install] WantedBy=multi-user.target #启动服务 systemctl daemon-reload systemctl start kube-apiserver #部署CalicoNode(所有节点) #准备证书 vim /etc/kubernetes/ca/calico/calico-csr.json { "CN": "calico", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "k8s", "OU": "System" } ] cd /etc/kubernetes/ca/calico/ cfssl gencert \ -ca=/etc/kubernetes/ca/ca.pem \ -ca-key=/etc/kubernetes/ca/ca-key.pem \ -config=/etc/kubernetes/ca/ca-config.json \ -profile=kubernetes calico-csr.json | cfssljson -bare calico #配置为系统服务 vim /lib/systemd/system/kube-calico.service [Unit] Description=calico node After=docker.service Requires=docker.service [Service] User=root PermissionsStartOnly=true ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \ -e ETCD_ENDPOINTS=https://192.168.66.135:2379 \ -e ETCD_CA_CERT_FILE=/etc/kubernetes/ca/ca.pem \ -e ETCD_CERT_FILE=/etc/kubernetes/ca/calico/calico.pem \ -e ETCD_KEY_FILE=/etc/kubernetes/ca/calico/calico-key.pem \ -e CALICO_LIBNETWORK_ENABLED=true \ -e CALICO_NETWORKING_BACKEND=bird \ -e CALICO_DISABLE_FILE_LOGGING=true \ -e CALICO_IPV4POOL_CIDR=172.20.0.0/16 \ -e CALICO_IPV4POOL_IPIP=off \ -e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \ -e FELIX_IPV6SUPPORT=false \ -e FELIX_LOGSEVERITYSCREEN=info \ -e FELIX_IPINIPMTU=1440 \ -e FELIX_HEALTHENABLED=true \ -e IP= \ -v /etc/kubernetes/ca:/etc/kubernetes/ca \ -v /var/run/calico:/var/run/calico \ -v /lib/modules:/lib/modules \ -v /run/docker/plugins:/run/docker/plugins \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /var/log/calico:/var/log/calico \ registry.cn-hangzhou.aliyuncs.com/imooc/calico-node:v2.6.2 ExecStop=/usr/bin/docker rm -f calico-node Restart=always RestartSec=10 [Install] WantedBy=multi-user.target #启动服务 systemctl enable kube-calico.service systemctl start kube-calico.service #部署ControllerManager(主节点) #配置ControllerManager为系统服务 vim /lib/systemd/system/kube-controller-manager.service #输入如下内容,注意修改主机ip和相关路径 [Unit] Description=Kubernetes Controller Manager Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/root/bin/kube-controller-manager \ --address=127.0.0.1 \ --master=http://127.0.0.1:8080 \ --allocate-node-cidrs=true \ --service-cluster-ip-range=10.68.0.0/16 \ --cluster-cidr=172.20.0.0/16 \ --cluster-name=kubernetes \ --leader-elect=true \ --cluster-signing-cert-file=/etc/kubernetes/ca/ca.pem \ --cluster-signing-key-file=/etc/kubernetes/ca/ca-key.pem \ --service-account-private-key-file=/etc/kubernetes/ca/ca-key.pem \ --root-ca-file=/etc/kubernetes/ca/ca.pem \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target #启动服务 systemctl daemon-reload systemctl start kube-controller-manager # 部署Scheduler(主节点) #配置Scheduler为系统服务 vim /lib/systemd/system/kube-scheduler.service #输入如下内容,注意修改主机ip和相关路径 [Unit] Description=Kubernetes Scheduler Documentation=https://github.com/GoogleCloudPlatform/kubernetes [Service] ExecStart=/root/bin/kube-scheduler \ --address=127.0.0.1 \ --master=http://127.0.0.1:8080 \ --leader-elect=true \ --v=2 Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target #启动服务 systemctl enable kube-scheduler.service systemctl start kube-scheduler.service #配置kubectl mkdir -p /etc/kubernetes/ca/admin #准备csr.json,内容如下 { "CN": "admin", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "system:masters", "OU": "System" } ] } cd /etc/kubernetes/ca/admin/ cfssl gencert \ -ca=/etc/kubernetes/ca/ca.pem \ -ca-key=/etc/kubernetes/ca/ca-key.pem \ -config=/etc/kubernetes/ca/ca-config.json \ -profile=kubernetes admin-csr.json | cfssljson -bare admin #指定apiserver的地址和证书位置 kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/ca/ca.pem \ --embed-certs=true \ --server=https://192.168.66.135:6443 #设置客户端认证参数,指定admin证书和秘钥 kubectl config set-credentials admin \ --client-certificate=/etc/kubernetes/ca/admin/admin.pem \ --embed-certs=true \ --client-key=/etc/kubernetes/ca/admin/admin-key.pem #关联用户和集群 kubectl config set-context kubernetes \ --cluster=kubernetes --user=admin #设置当前上下文 kubectl config use-context kubernetes #部署kubelet #创建角色绑定(主节点) kubectl -n kube-system get clusterrole kubectl create clusterrolebinding kubelet-bootstrap \ --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap #创建bootstrap.kubeconfig(工作节点) #设置集群参数(注意替换ip) kubectl config set-cluster kubernetes \ --certificate-authority=/etc/kubernetes/ca/ca.pem \ --embed-certs=true \ --server=https://192.168.66.135:6443 \ --kubeconfig=bootstrap.kubeconfig #设置客户端认证参数(注意替换token) kubectl config set-credentials kubelet-bootstrap \ --token=11f4d7eafcf06965e413a409d35c3893\ --kubeconfig=bootstrap.kubeconfig #设置上下文 kubectl config set-context default \ --cluster=kubernetes \ --user=kubelet-bootstrap \ --kubeconfig=bootstrap.kubeconfig #选择上下文 kubectl config use-context default --kubeconfig=bootstrap.kubeconfig mv bootstrap.kubeconfig /etc/kubernetes/ #准备cni配置 [root@C2 ~]# cat /etc/cni/net.d/10-calico.conf { "name": "calico-k8s-network", "cniVersion": "0.1.0", "type": "calico", "etcd_endpoints": "https://192.168.66.135:2379", "etcd_key_file": "/etc/kubernetes/ca/calico/calico-key.pem", "etcd_cert_file": "/etc/kubernetes/ca/calico/calico.pem", "etcd_ca_cert_file": "/etc/kubernetes/ca/ca.pem", "log_level": "info", "ipam": { "type": "calico-ipam" }, "kubernetes": { "kubeconfig": "/etc/kubernetes/kubelet.kubeconfig" } } #配置为系统服务并启动 [root@C2 ~]# cat /lib/systemd/system/kubelet.service [Unit] Description=Kubernetes Kubelet Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=docker.service Requires=docker.service [Service] WorkingDirectory=/var/lib/kubelet ExecStart=/root/bin/kubelet \ --address=192.168.66.136 \ --hostname-override=192.168.66.136 \ --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/imooc/pause-amd64:3.0 \ --kubeconfig=/etc/kubernetes/kubelet.kubeconfig \ --experimental-bootstrap-kubeconfig=/etc/kubernetes/bootstrap.kubeconfig \ --cert-dir=/etc/kubernetes/ca \ --hairpin-mode hairpin-veth \ --network-plugin=cni \ --cni-conf-dir=/etc/cni/net.d \ --cni-bin-dir=/root/bin \ --cluster-dns=10.68.0.2 \ --cluster-domain=cluster.local. \ --allow-privileged=true \ --fail-swap-on=false \ --logtostderr=true \ --v=2 #kubelet cAdvisor 默认在所有接口监听 4194 端口的请求, 以下iptables限制内网访问 ExecStartPost=/sbin/iptables -A INPUT -s 10.0.0.0/8 -p tcp --dport 4194 -j ACCEPT ExecStartPost=/sbin/iptables -A INPUT -s 172.16.0.0/12 -p tcp --dport 4194 -j ACCEPT ExecStartPost=/sbin/iptables -A INPUT -s 192.168.0.0/16 -p tcp --dport 4194 -j ACCEPT ExecStartPost=/sbin/iptables -A INPUT -p tcp --dport 4194 -j DROP Restart=on-failure RestartSec=5 [Install] WantedBy=multi-user.target #启动kubelet之后到master节点允许worker加入(批准worker的tls证书请求) #--------*在主节点执行*--------- $ kubectl get csr|grep 'Pending' | awk '{print $1}'| xargs kubectl certificate approve #----------------------------- #部署kube-proxy(工作节点) #生成proxy证书 mkdir -p /etc/kubernetes/ca/kube-proxy #准备proxy证书配置 - proxy只需客户端证书,因此证书请求中 hosts 字段可以为空。 #CN 指定该证书的 User 为 system:kube-proxy,预定义的 ClusterRoleBinding system:node-proxy 将User system:kube-proxy 与 Role system:node-proxier 绑定,授予了调用 kube-api-server proxy的相关 API 的权限 cat /etc/kubernetes/ca/kube-proxy/kube-proxy-csr.json [root@C2 ~]# cat /etc/kubernetes/ca/kube-proxy/kube-proxy-csr.json { "CN": "system:kube-proxy", "hosts": [], "key": { "algo": "rsa", "size": 2048 }, "names": [ { "C": "CN", "ST": "Beijing", "L": "XS", "O": "k8s", "OU": "System" } ] } cd /etc/kubernetes/ca/kube-proxy/ cfssl gencert \ -ca=/etc/kubernetes/ca/ca.pem \ -ca-key=/etc/kubernetes/ca/ca-key.pem \ -config=/etc/kubernetes/ca/ca-config.json \ -profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy #同上配置系统服务并启动 [root@C2 ~]# cat /lib/systemd/system/kube kube-calico.service kubelet.service kube-proxy.service [root@C2 ~]# cat /lib/systemd/system/kube-proxy.service [Unit] Description=Kubernetes Kube-Proxy Server Documentation=https://github.com/GoogleCloudPlatform/kubernetes After=network.target [Service] WorkingDirectory=/var/lib/kube-proxy ExecStart=/root/bin/kube-proxy \ --bind-address=192.168.66.136 \ --hostname-override=192.168.66.136 \ --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig \ --logtostderr=true \ --v=2 Restart=on-failure RestartSec=5 LimitNOFILE=65536 [Install] WantedBy=multi-user.target ps:因为需要依赖conntrack-tools,所以在启动之前,执行yum install conntrack-tools -y #部署kube-dns(k8s app) #在主节点准备如下文件 cat kube-dns.yaml --- apiVersion: v1 kind: ConfigMap metadata: name: kube-dns namespace: kube-system labels: addonmanager.kubernetes.io/mode: EnsureExists --- apiVersion: v1 kind: ServiceAccount metadata: name: kube-dns namespace: kube-system labels: addonmanager.kubernetes.io/mode: Reconcile --- apiVersion: v1 kind: Service metadata: name: kube-dns namespace: kube-system labels: k8s-app: kube-dns addonmanager.kubernetes.io/mode: Reconcile kubernetes.io/name: "KubeDNS" spec: selector: k8s-app: kube-dns clusterIP: 10.68.0.2 ports: - name: dns port: 53 protocol: UDP - name: dns-tcp port: 53 protocol: TCP --- apiVersion: apps/v1 kind: Deployment metadata: name: kube-dns namespace: kube-system labels: k8s-app: kube-dns addonmanager.kubernetes.io/mode: Reconcile spec: strategy: rollingUpdate: maxSurge: 10% maxUnavailable: 0 selector: matchLabels: k8s-app: kube-dns template: metadata: labels: k8s-app: kube-dns annotations: scheduler.alpha.kubernetes.io/critical-pod: '' spec: tolerations: - key: "CriticalAddonsOnly" operator: "Exists" volumes: - name: kube-dns-config configMap: name: kube-dns optional: true containers: - name: kubedns image: registry.cn-hangzhou.aliyuncs.com/imooc/k8s-dns-kube-dns-amd64:1.14.5 resources: # TODO: Set memory limits when we've profiled the container for large # clusters, then set request = limit to keep this container in # guaranteed class. Currently, this container falls into the # "burstable" category so the kubelet doesn't backoff from restarting it. limits: memory: 170Mi requests: cpu: 100m memory: 70Mi livenessProbe: httpGet: path: /healthcheck/kubedns port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 readinessProbe: httpGet: path: /readiness port: 8081 scheme: HTTP # we poll on pod startup for the Kubernetes master service and # only setup the /readiness HTTP server once that's available. initialDelaySeconds: 3 timeoutSeconds: 5 args: - --domain=cluster.local. - --dns-port=10053 - --config-dir=/kube-dns-config - --v=2 env: - name: PROMETHEUS_PORT value: "10055" ports: - containerPort: 10053 name: dns-local protocol: UDP - containerPort: 10053 name: dns-tcp-local protocol: TCP - containerPort: 10055 name: metrics protocol: TCP volumeMounts: - name: kube-dns-config mountPath: /kube-dns-config - name: dnsmasq image: registry.cn-hangzhou.aliyuncs.com/imooc/k8s-dns-dnsmasq-nanny-amd64:1.14.5 livenessProbe: httpGet: path: /healthcheck/dnsmasq port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 args: - -v=2 - -logtostderr - -configDir=/etc/k8s/dns/dnsmasq-nanny - -restartDnsmasq=true - -- - -k - --cache-size=1000 - --log-facility=- - --server=/cluster.local./127.0.0.1#10053 - --server=/in-addr.arpa/127.0.0.1#10053 - --server=/ip6.arpa/127.0.0.1#10053 ports: - containerPort: 53 name: dns protocol: UDP - containerPort: 53 name: dns-tcp protocol: TCP # see: https://github.com/kubernetes/kubernetes/issues/29055 for details resources: requests: cpu: 150m memory: 20Mi volumeMounts: - name: kube-dns-config mountPath: /etc/k8s/dns/dnsmasq-nanny - name: sidecar image: registry.cn-hangzhou.aliyuncs.com/imooc/k8s-dns-sidecar-amd64:1.14.5 livenessProbe: httpGet: path: /metrics port: 10054 scheme: HTTP initialDelaySeconds: 60 timeoutSeconds: 5 successThreshold: 1 failureThreshold: 5 args: - --v=2 - --logtostderr - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.cluster.local.,5,A - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.cluster.local.,5,A ports: - containerPort: 10054 name: metrics protocol: TCP resources: requests: memory: 20Mi cpu: 10m dnsPolicy: Default # Don't use cluster DNS. serviceAccountName: kube-dns kubectl create -f ~/kube-dns.yaml