Kubeadm方式搭建K8S集群

Kubeadm方式搭建集群优缺点：

优点：
    简单优雅，支持高可用，升级方便
    
缺点：
    不易维护，文档不够细致

将master作为deploy节点，未指定节点时默认在master上进行操作。

建议deploy节点与其它节点配置ssh免密登录，配置过程参考：批量实现SSH免密登录 。

---

环境准备

环境准备工作请在所有节点进行。

• 主机说明：

系统	ip	角色	cpu	内存	hostname
CentOS 7.8	192.168.30.128	master、deploy	>=2	>=2G	master
CentOS 7.8	192.168.30.129	node	>=2	>=2G	node1
CentOS 7.8	192.168.30.130	node	>=2	>=2G	node2
CentOS 7.8	192.168.30.131	node	>=2	>=2G	node3

• 设置主机名：

以master为例，

hostnamectl set-hostname master

• 安装依赖包：

yum update -y

yum install -y curl git iptables conntrack ipvsadm ipset jq sysstat libseccomp

• 关闭防火墙、selinux和swap，重置iptables：

systemctl stop firewalld && systemctl disable firewalld

sed -i 's/=enforcing/=disabled/g' /etc/selinux/config && setenforce 0

iptables -F && iptables -X && iptables -F -t nat && iptables -X -t nat && iptables -P FORWARD ACCEPT

swapoff -a

sed -i '/swap/s/^\(.*\)$/#\1/g' /etc/fstab

• 系统参数设置：

cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_watches=89100
EOF

modprobe br_netfilter

sysctl -p /etc/sysctl.d/kubernetes.conf

• 安装docker：

curl http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -o /etc/yum.repos.d/docker.repo

yum makecache fast

yum install -y docker-ce

systemctl enable docker && systemctl start docker

cat > /etc/docker/daemon.json <<EOF
{  
    "registry-mirrors": ["http://f1361db2.m.daocloud.io"],
    "exec-opts":["native.cgroupdriver=systemd"]
}
EOF

systemctl restart docker

• 安装必要工具：

kubeadm     用于部署集群

bukelet     集群中各节点需要运行的组件，负责管理pod、容器的生命周期

kubectl     集群管理工具（master节点安装即可）

cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
       http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum install -y kubeadm-1.18.3 kubelet-1.18.3 kubectl-1.18.3 --disableexcludes=kubernetes

systemctl enable kubelet && systemctl start kubelet

---

集群初始化

• 集群初始化：

mkdir /software

vim /software/kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
kubernetesVersion: v1.18.3
controlPlaneEndpoint: 192.168.30.128:6443
networking:
    podSubnet: 172.10.0.0/16
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers

kubeadm config images pull --kubernetes-version=v1.18.3 --image-repository=registry.cn-hangzhou.aliyuncs.com/google_containers

kubeadm init --config=/software/kubeadm-config.yaml --upload-certs

初始化这一步如果报错：

error execution phase upload-config/kubelet: Error writing Crisocket information for the control-plane node: timed out waiting for the condition

解决：

swapoff -a

kubeadm reset -f

systemctl daemon-reload

systemctl restart kubelet

iptables -F && iptables -t nat -F && iptables -t mangle -F && iptables -X

mkdir ~/.kube

\cp /etc/kubernetes/admin.conf ~/.kube/config

kubectl get pods -n kube-system

NAME                             READY   STATUS    RESTARTS   AGE
coredns-546565776c-srxkq         0/1     Pending   0          60s
coredns-546565776c-w9fbs         0/1     Pending   0          60s
etcd-master                      1/1     Running   0          75s
kube-apiserver-master            1/1     Running   0          75s
kube-controller-manager-master   1/1     Running   0          75s
kube-proxy-qb7d5                 1/1     Running   0          60s
kube-scheduler-master            1/1     Running   0          75s

kubectl completion bash > ~/.kube/completion.bash.inc
 
echo 'source ~/.kube/completion.bash.inc' >> ~/.bash_profile

source ~/.bash_profile

注意备份上面初始化之后打印的join命令，这里分别是以master、node节点加入集群。

kubeadm join 192.168.30.128:6443 --token 1ndel7.xb623vep9pl5o6vl \
    --discovery-token-ca-cert-hash sha256:0e41f6020955c36970bf504cbfc0047941240dda57ebb9d85086706da14dcd1f \
    --control-plane --certificate-key 6518fe9f3eca5cb4a5860170d18c03109f54c94fba8ca7e5408a9aab5e598663

kubeadm join 192.168.30.128:6443 --token 1ndel7.xb623vep9pl5o6vl \
    --discovery-token-ca-cert-hash sha256:0e41f6020955c36970bf504cbfc0047941240dda57ebb9d85086706da14dcd1f

---

部署calico

• 部署calico：

mkdir /etc/kubernetes/addons

vim /etc/kubernetes/addons/calico-rbac.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-kube-controllers
  namespace: kube-system
  
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
rules:
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - watch
      - list
      - get
  - apiGroups: [""]
    resources:
      - pods
    verbs:
      - get
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
    verbs:
      - list
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - hostendpoints
    verbs:
      - get
      - list
      - create
      - update
      - delete
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - clusterinformations
    verbs:
      - get
      - create
      - update
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - kubecontrollersconfigurations
    verbs:
      - get
      - create
      - update
      - watch
      
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-kube-controllers
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-kube-controllers
subjects:
- kind: ServiceAccount
  name: calico-kube-controllers
  namespace: kube-system

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: calico-node
  namespace: kube-system

---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: calico-node
rules:
  - apiGroups: [""]
    resources:
      - pods
      - nodes
      - namespaces
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - endpoints
      - services
    verbs:
      - watch
      - list
      - get
  - apiGroups: [""]
    resources:
      - configmaps
    verbs:
      - get
  - apiGroups: [""]
    resources:
      - nodes/status
    verbs:
      - patch
      - update
  - apiGroups: ["networking.k8s.io"]
    resources:
      - networkpolicies
    verbs:
      - watch
      - list
  - apiGroups: [""]
    resources:
      - pods
      - namespaces
      - serviceaccounts
    verbs:
      - list
      - watch
  - apiGroups: [""]
    resources:
      - pods/status
    verbs:
      - patch
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - globalfelixconfigs
      - felixconfigurations
      - bgppeers
      - globalbgpconfigs
      - bgpconfigurations
      - ippools
      - ipamblocks
      - globalnetworkpolicies
      - globalnetworksets
      - networkpolicies
      - networksets
      - clusterinformations
      - hostendpoints
      - blockaffinities
    verbs:
      - get
      - list
      - watch
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ippools
      - felixconfigurations
      - clusterinformations
    verbs:
      - create
      - update
  - apiGroups: [""]
    resources:
      - nodes
    verbs:
      - get
      - list
      - watch
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - bgpconfigurations
      - bgppeers
    verbs:
      - create
      - update
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
      - ipamblocks
      - ipamhandles
    verbs:
      - get
      - list
      - create
      - update
      - delete
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - ipamconfigs
    verbs:
      - get
  - apiGroups: ["crd.projectcalico.org"]
    resources:
      - blockaffinities
    verbs:
      - watch
  - apiGroups: ["apps"]
    resources:
      - daemonsets
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: calico-node
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: calico-node
subjects:
- kind: ServiceAccount
  name: calico-node
  namespace: kube-system

vim /etc/kubernetes/addons/calico.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: calico-config
  namespace: kube-system
data:
  typha_service_name: "none"
  calico_backend: "bird"
  veth_mtu: "1440"
  cni_network_config: |-
    {
   
      "name": "k8s-pod-network",
      "cniVersion": "0.3.1",
      "plugins": [
        {
   
          "type": "calico",
          "log_level": "info",
          "datastore_type": "kubernetes",
          "nodename": "__KUBERNETES_NODE_NAME__",
          "mtu": __CNI_MTU__,
          "ipam": {
   
              "type": "calico-ipam"
          },
          "policy": {
   
              "type": "k8s"
          },
          "kubernetes": {
   
              "kubeconfig": "__KUBECONFIG_FILEPATH__"
          }
        },
        {
   
          "type": "portmap",
          "snat": true,
          "capabilities": {
   "portMappings": true}
        },
        {
   
          "type": "bandwidth",
          "capabilities": {
   "bandwidth": true}
        }
      ]
    }
  
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: bgpconfigurations.crd.projectcalico.org
spec:
  group: crd.projectcalico.org
  names:
    kind: BGPConfiguration
    listKind: BGPConfigurationList
    plural: bgpconfigurations
    singular: bgpconfiguration
  scope: Cluster
  versions:
  - name: v1
    schema:
      openAPIV3Schema:
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            properties:
              asNumber:
                format: int32
                type: integer
              logSeverityScreen:
                type: string
              nodeToNodeMeshEnabled:
                type: boolean
              serviceClusterIPs:
                items:
                  properties:
                    cidr:
                      type: string
                  type: object
                type: array
              serviceExternalIPs:
                items:
                  properties:
                    cidr:
                      type: string
                  type: object
                type: array
            type: object
        type: object
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: bgppeers.crd.projectcalico.org
spec:
  group: crd.projectcalico.org
  names:
    kind: BGPPeer
    listKind: BGPPeerList
    plural: bgppeers
    singular: bgppeer
  scope: Cluster
  versions:
  - name: v1
    schema:
      openAPIV3Schema:
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            properties:
              asNumber:
                format: int32
                type: integer
              node:
                type: string
              nodeSelector:
                type: string
              peerIP:
                type: string
              peerSelector:
                type: string
            required:
            - asNumber
            - peerIP
            type: object
        type: object
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: blockaffinities.crd.projectcalico.org
spec:
  group: crd.projectcalico.org
  names:
    kind: BlockAffinity
    listKind: BlockAffinityList
    plural: blockaffinities
    singular: blockaffinity
  scope: Cluster
  versions:
  - name: v1
    schema:
      openAPIV3Schema:
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            properties:
              cidr:
                type: string
              deleted:
                type: string
              node:
                type: string
              state:
                type: string
            required:
            - cidr
            - deleted
            - node
            - state
            type: object
        type: object
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []
  
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: clusterinformations.crd.projectcalico.org
spec:
  group: crd.projectcalico.org
  names:
    kind: ClusterInformation
    listKind: ClusterInformationList
    plural: clusterinformations
    singular: clusterinformation
  scope: Cluster
  versions:
  - name: v1
    schema:
      openAPIV3Schema:
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            properties:
              calicoVersion:
                type: string
              clusterGUID:
                type: string
              clusterType:
                type: string
              datastoreReady:
                type: boolean
              variant:
                type: string
            type: object
        type: object
    served: true
    storage: true
status:
  acceptedNames:
    kind: ""
    plural: ""
  conditions: []
  storedVersions: []

---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
  name: felixconfigurations.crd.projectcalico.org
spec:
  group: crd.projectcalico.org
  names:
    kind: FelixConfiguration
    listKind: FelixConfigurationList
    plural: felixconfigurations
    singular: felixconfiguration
  scope: Cluster
  versions:
  - name: v1
    schema:
      openAPIV3Schema:
        properties:
          apiVersion:
            type: string
          kind:
            type: string
          metadata:
            type: object
          spec:
            properties:
              bpfConnectTimeLoadBalancingEnabled:
                type: boolean
              bpfDataIfacePattern:
                type: string
              bpfDisableUnprivileged:
                type: boolean
              bpfEnabled:
                type: boolean
              bpfExternalServiceMode:
                type: string
              bpfKubeProxyEndpointSlicesEnabled:
                type: boolean
              bpfKubeProxyIptablesCleanupEnabled:
                type: boolean
              bpfKubeProxyMinSyncPeriod:
                type: string
              bpfLogLevel:
                type: string
              chainInsertMode:
                type: string
              dataplaneDriver:
                type: string
              debugDisableLogDropping:
                type: boolean
              debugMemoryProfilePath:
                type