【博客470】如何分别对k8s新集群和老集群开启新的feature-gates

如何对已有k8s集群开启新的feature-gates

场景：

k8s中有非常多的feature-gates，很多是默认不开启的，当有新的业务需求，需要开启某些feature-gates

以开启ServiceTopology为例

对新集群：

---
apiVersion: kubeadm.k8s.io/v1beta2
kind: ClusterConfiguration
apiServer:
  extraArgs:
    feature-gates: "ServiceTopology=true,EndpointSlice=true"
controllerManager:
  extraArgs:
    feature-gates: "ServiceTopology=true,EndpointSlice=true"
scheduler:
  extraArgs:
    feature-gates: "ServiceTopology=true,EndpointSlice=true"
networking:
   podSubnet: "10.244.0.0/16"

---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
featureGates:
  ServiceTopology: true
  EndpointSliceProxying: true

对已有集群：

1、对kube-proxy

// kubectl edit cm kube-proxy -n kube-system
apiVersion: v1
data:
  config.conf: |-
    apiVersion: kubeproxy.config.k8s.io/v1alpha1
    bindAddress: 0.0.0.0
    bindAddressHardFail: false
    # 在configmap的配置内容中加入开启ServiceTopology这个featureGates
    featureGates:
      ServiceTopology: true
    ...
    ...
    kind: KubeProxyConfiguration
    metricsBindAddress: ""
    mode: ""
    ...
kind: ConfigMap
metadata:
  labels:
    app: kube-proxy
  name: kube-proxy
  namespace: kube-system
  
2、修改apiserver配置

// cat kube-apiserver.yaml 
apiVersion: v1
kind: Pod
metadata:
  labels:
    component: kube-apiserver
    tier: control-plane
  name: kube-apiserver
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-apiserver
    - --allow-privileged=true
    - --authorization-mode=Node,RBAC
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --enable-admission-plugins=NodeRestriction
    - --enable-bootstrap-token-auth=true
    - --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
    - --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
    - --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
    - --etcd-servers=https://127.0.0.1:2379
    - --insecure-port=0
    - --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt
    - --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key
    - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
    - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
    - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
    - --requestheader-allowed-names=front-proxy-client
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --requestheader-extra-headers-prefix=X-Remote-Extra-
    - --requestheader-group-headers=X-Remote-Group
    - --requestheader-username-headers=X-Remote-User
    - --secure-port=6443
    - --service-account-issuer=https://kubernetes.default.svc.cluster.local
    - --service-account-key-file=/etc/kubernetes/pki/sa.pub
    - --service-account-signing-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=10.96.0.0/12
    - --tls-cert-file=/etc/kubernetes/pki/apiserver.crt
    - --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
    # 加入ServiceTopology这个feature-gate的开启
    - --feature-gates=ServiceTopology=true
    image: registry.aliyuncs.com/google_containers/kube-apiserver:v1.20.11
    imagePullPolicy: IfNotPresent
    ...
    ...
    
3、修改kube-controller-manager配置

apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-controller-manager
    tier: control-plane
  name: kube-controller-manager
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-controller-manager
    - --allocate-node-cidrs=true
    - --authentication-kubeconfig=/etc/kubernetes/controller-manager.conf
    - --authorization-kubeconfig=/etc/kubernetes/controller-manager.conf
    - --bind-address=127.0.0.1
    - --client-ca-file=/etc/kubernetes/pki/ca.crt
    - --cluster-cidr=10.244.0.0/16
    - --cluster-name=kubernetes
    - --cluster-signing-cert-file=/etc/kubernetes/pki/ca.crt
    - --cluster-signing-key-file=/etc/kubernetes/pki/ca.key
    - --controllers=*,bootstrapsigner,tokencleaner
    - --kubeconfig=/etc/kubernetes/controller-manager.conf
    - --leader-elect=true
    - --port=0
    - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
    - --root-ca-file=/etc/kubernetes/pki/ca.crt
    - --service-account-private-key-file=/etc/kubernetes/pki/sa.key
    - --service-cluster-ip-range=10.96.0.0/12
    - --use-service-account-credentials=true
    # 加入ServiceTopology这个feature-gate的开启
    - --feature-gates=ServiceTopology=true
    image: registry.aliyuncs.com/google_containers/kube-controller-manager:v1.20.11
    imagePullPolicy: IfNotPresent

4、修改scheduler配置

apiVersion: v1
kind: Pod
metadata:
  creationTimestamp: null
  labels:
    component: kube-scheduler
    tier: control-plane
  name: kube-scheduler
  namespace: kube-system
spec:
  containers:
  - command:
    - kube-scheduler
    - --authentication-kubeconfig=/etc/kubernetes/scheduler.conf
    - --authorization-kubeconfig=/etc/kubernetes/scheduler.conf
    - --bind-address=127.0.0.1
    - --kubeconfig=/etc/kubernetes/scheduler.conf
    - --leader-elect=true
    - --port=0
    # 加入ServiceTopology这个feature-gate的开启
    - --feature-gates=ServiceTopology=true
    image: registry.aliyuncs.com/google_containers/kube-scheduler:v1.20.11
    imagePullPolicy: IfNotPresent
    ...
    ...

开启其它feature-gates方法

1、查看官方文档，看某个feature-gates的修改需要更多k8s中哪些组件的启动参数

2、对kube-proxy这一类daemonset的，修改configmap，然后delete pod，重新创建

3、对apiserver，scheduler，kube-controller-manager等static pod，则修改/etc/kubernetes/manifests/下的yaml文件，kubelet会watch这些文件，并在他们发生变化的时候，自动重新创建pod

[root@ ~]# cd /etc/kubernetes/manifests/
[root@ manifests]# ls
etcd.yaml  kube-apiserver.yaml  kube-controller-manager.yaml  kube-scheduler.yaml