kubernetes【k8s】adm方式安装[超级详细]

集群规划【1master节点2node节点】

主机名	        IP地址	           推荐配置
k8s-master      172.16.115.237     1C2G40G
k8s-node1       172.16.115.238     1C2G40G
k8s-node2       172.16.115.239     1C2G40G

网络规划

network	       IP地址段	          解释
NodeIP         172.16.115.0/24    #对外提供用户访问
PodIP          10.2.0.0/16        #集群内部IP，可以动态感知后面的POD IP
ClusterIP      10.1.0.0/16        #POD的IP

1.k8s环境准备【机器标准化】所有节点执行

1.1配置hosts解析

cat >> /etc/hosts << EOF
172.16.115.237 k8s-master 
172.16.115.238 k8s-node1
172.16.115.239 k8s-node2
EOF

1.2关闭防火墙

systemctl  stop  firewalld   NetworkManager 
systemctl  disable  firewalld   NetworkManager

1.3关闭SELinux

setenforce 0
sed -i 's#SELINUX=disabled#SELINUX=disabled#g' /etc/selinux/config
getenforce

1.4关闭SWAP分区

swapoff -a
sed -i '/swap/d' /etc/fstab
free -h

1.5配置时间同步

yum install chrony -y
systemctl start chronyd
systemctl enable chronyd
date

1.6下载Docker CE Yum 源 

yum-config-manager \
    --add-repo https://download.docker.com/linux/centos/docker-ce.repo

1.7安装开源版Docker CE
 

yum install docker-ce-19.03.6 docker-ce-cli-19.03.6 containerd.io 

1.8启动Docker
 

service docker start 
chkconfig docker on		#开机启动

1.9查看Docker版本

docker version
docker info

1.10配置镜像加速器

mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://plqjafsr.mirror.aliyuncs.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker

2.Kubernetes 1.15 安装

2.1初始化工具安装

yum install net-tools vim wget lrzsz git -y

2.2设置免密码登录

yum install -y expect
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
export mypass=123456 
name=(k8s-master k8s-node1 k8s-node2)
for i in ${name[@]};do
expect -c "
spawn ssh-copy-id -i /root/.ssh/id_rsa.pub root@$i
  expect {
    \"*yes/no*\" {send \"yes\r\"; exp_continue}
    \"*password*\" {send \"$mypass\r\"; exp_continue}
    \"*Password*\" {send \"$mypass\r\";}
  }"
done

2.3连接测试

ssh k8s-node1
ssh k8s-node2

2.4优化内核参数

cat >>/etc/sysctl.conf<<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
fs.file-max=52706963
fs.nr_open=52706963
EOF

2.5应用内核配置

modprobe br_netfilter
sysctl -p

3配置证书

3.1下载自签名证书生成工具

mkdir /soft && cd /soft
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64
chmod +x cfssl_linux-amd64 cfssljson_linux-amd64 cfssl-certinfo_linux-amd64
mv cfssl_linux-amd64 /usr/local/bin/cfssl
mv cfssljson_linux-amd64 /usr/local/bin/cfssljson
mv cfssl-certinfo_linux-amd64 /usr/bin/cfssl-certinfo

3.2生成ETCD证书

CA 证书配置

mkdir /root/etcd && cd /root/etcd
cat << EOF | tee ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "www": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

创建CA证书请求文件

cat << EOF | tee ca-csr.json
{
    "CN": "etcd CA",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

创建ETCD证书请求文件

#可以把所有的master IP 加入到csr文件中（Master-1）

cat << EOF | tee server-csr.json
{
    "CN": "etcd",
    "hosts": [
    "k8s-master",
    "k8s-master2",
    "k8s-master3",
    "172.16.115.237",
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing"
        }
    ]
}
EOF

生成 ETCD CA 证书和ETCD公私钥（Master-1）

cd /root/etcd/
生成ca证书（Master-1）
cfssl gencert -initca ca-csr.json | cfssljson -bare ca –
ll
total 24
-rw-r--r-- 1 root root  287 Apr  5 11:23 ca-config.json      #ca 的配置文件
-rw-r--r-- 1 root root  956 Apr  5 11:26 ca.csr			  #ca 证书生成文件
-rw-r--r-- 1 root root  209 Apr  5 11:23 ca-csr.json		  #ca 证书请求文件
-rw------- 1 root root 1679 Apr  5 11:26 ca-key.pem		  #ca 证书key
-rw-r--r-- 1 root root 1265 Apr  5 11:26 ca.pem			  #ca 证书
-rw-r--r-- 1 root root  338 Apr  5 11:26 server-csr.json

生成etcd证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=www server-csr.json | cfssljson -bare server
ll
total 36
-rw-r--r-- 1 root root  287 Apr  5 11:23 ca-config.json
-rw-r--r-- 1 root root  956 Apr  5 11:26 ca.csr
-rw-r--r-- 1 root root  209 Apr  5 11:23 ca-csr.json
-rw------- 1 root root 1679 Apr  5 11:26 ca-key.pem
-rw-r--r-- 1 root root 1265 Apr  5 11:26 ca.pem
-rw-r--r-- 1 root root 1054 Apr  5 11:31 server.csr
-rw-r--r-- 1 root root  338 Apr  5 11:26 server-csr.json
-rw------- 1 root root 1675 Apr  5 11:31 server-key.pem	#etcd客户端使用
-rw-r--r-- 1 root root 1379 Apr  5 11:31 server.pem

创建 Kubernetes 相关证书

#此证书用于Kubernetes节点直接的通信, 与之前的ETCD证书不同. （Master-1）

配置ca 文件（Master-1）

mkdir /root/kubernetes/ && cd /root/kubernetes/
cat << EOF | tee ca-config.json
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
         "expiry": "87600h",
         "usages": [
            "signing",
            "key encipherment",
            "server auth",
            "client auth"
        ]
      }
    }
  }
}
EOF

创建ca证书申请文件（Master-1）

 cat << EOF | tee ca-csr.json
{
    "CN": "kubernetes",
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

生成API SERVER证书申请文件（Master-1）

 cat << EOF | tee server-csr.json
{
    "CN": "kubernetes",
    "hosts": [
      "10.0.0.1",
      "127.0.0.1",
"10.0.0.2",
"172.16.115.237",
"172.16.115.238",
"172.16.115.239",
"k8s-master",
"k8s-node1",
"k8s-node2",
      "kubernetes",
      "kubernetes.default",
      "kubernetes.default.svc",
      "kubernetes.default.svc.cluster",
      "kubernetes.default.svc.cluster.local"
    ],
    "key": {
        "algo": "rsa",
        "size": 2048
    },
    "names": [
        {
            "C": "CN",
            "L": "Beijing",
            "ST": "Beijing",
            "O": "k8s",
            "OU": "System"
        }
    ]
}
EOF

创建 Kubernetes Proxy 证书申请文件（Master-1）

cat << EOF | tee kube-proxy-csr.json
{
  "CN": "system:kube-proxy",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "Beijing",
      "ST": "Beijing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF

生成 kubernetes CA 证书和公私钥

生成ca证书（Master-1）

cfssl gencert -initca ca-csr.json | cfssljson -bare ca –

生成 api-server 证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

生成 kube-proxy 证书（Master-1）

cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json \
-profile=kubernetes kube-proxy-csr.json | cfssljson -bare kube-proxy

部署ETCD

下载etcd二进制安装文件

mkdir -p /soft && cd /soft
wget https://github.com/etcd-io/etcd/releases/download/v3.3.10/etcd-v3.3.10-linux-amd64.tar.gz
tar -xvf etcd-v3.3.10-linux-amd64.tar.gz
cd etcd-v3.3.10-linux-amd64/
cp etcd etcdctl /usr/local/bin/

编辑etcd配置文件

#注意修改每个节点的ETCD_NAME
#注意修改每个节点的监听地址

mkdir -p /etc/etcd/{cfg,ssl}
cat  >/etc/etcd/cfg/etcd.conf<<EOFL
#[Member]
ETCD_NAME="master-1"
ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
ETCD_LISTEN_PEER_URLS="https://172.16.115.237:2380"
ETCD_LISTEN_CLIENT_URLS="https://172.16.115.237:2379,http://172.16.115.237:2390"

#[Clustering]
ETCD_INITIAL_ADVERTISE_PEER_URLS="https://172.16.115.237:2380" #如果有多个可以用，号分割
ETCD_ADVERTISE_CLIENT_URLS="https://172.16.115.237:2379"
ETCD_INITIAL_CLUSTER="master-1=https://172.16.115.237:2380"
ETCD_INITIAL_CLUSTER_TOKEN="etcd-cluster"
ETCD_INITIAL_CLUSTER_STATE="new"
EOFL

创建ETCD的系统启动服务（所有master）

cat > /usr/lib/systemd/system/etcd.service<<EOFL
[Unit]
Description=Etcd Server
After=network.target
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/etc/etcd/cfg/etcd.conf
ExecStart=/usr/local/bin/etcd \
--name=\${ETCD_NAME} \
--data-dir=\${ETCD_DATA_DIR} \
--listen-peer-urls=\${ETCD_LISTEN_PEER_URLS} \
--listen-client-urls=\${ETCD_LISTEN_CLIENT_URLS},http://127.0.0.1:2379 \
--advertise-client-urls=\${ETCD_ADVERTISE_CLIENT_URLS} \
--initial-advertise-peer-urls=\${ETCD_INITIAL_ADVERTISE_PEER_URLS} \
--initial-cluster=\${ETCD_INITIAL_CLUSTER} \
--initial-cluster-token=\${ETCD_INITIAL_CLUSTER_TOKEN} \
--initial-cluster-state=new \
--cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--peer-cert-file=/etc/etcd/ssl/server.pem \
--peer-key-file=/etc/etcd/ssl/server-key.pem \
--trusted-ca-file=/etc/etcd/ssl/ca.pem \
--peer-trusted-ca-file=/etc/etcd/ssl/ca.pem
Restart=on-failure
LimitNOFILE=65536

[Install]
WantedBy=multi-user.target
EOFL

复制etcd证书到指定目录

mkdir -p /etc/etcd/ssl/
cp /root/etcd/*pem /etc/etcd/ssl/ -rf

复制etcd证书到每个节点

for i in  k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/etcd/{cfg,ssl};done
for i in  k8s-node1 k8s-node2;do scp /etc/etcd/ssl/* $i:/etc/etcd/ssl/;done
for i in  k8s-node1 k8s-node2;do echo $i "------>"; ssh $i ls /etc/etcd/ssl;done

启动etcd (所有节点)

chkconfig etcd on
service etcd start
service etcd status

检查etcd 集群是否运行正常

etcdctl --ca-file=/etc/etcd/ssl/ca.pem --cert-file=/etc/etcd/ssl/server.pem \
--key-file=/etc/etcd/ssl/server-key.pem --endpoints="https://k8s-master:2379"  cluster-health

创建Docker所需分配POD 网段 (任意master节点)

etcdctl --ca-file=/etc/etcd/ssl/ca.pem \
--cert-file=/etc/etcd/ssl/server.pem --key-file=/etc/etcd/ssl/server-key.pem \
--endpoints="https://k8s-master:2379" \
 set /coreos.com/network/config  \
 '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'

检查是否建立网段

etcdctl \
--endpoints=https://k8s-master:2379 \
--ca-file=/etc/etcd/ssl/ca.pem \
--key-file=/etc/etcd/ssl/server-key.pem \
--cert-file=/etc/etcd/ssl/server.pem \
get /coreos.com/network/config 

4部署Flannel

下载Flannel二进制包

所有的节点,下载到master-1

mkdir /soft ; cd /soft
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
tar xvf flannel-v0.10.0-linux-amd64.tar.gz
mv flanneld mk-docker-opts.sh /usr/local/bin/

复制flanneld到其他的所有节点

for i in k8s-node1 k8s-node2;do scp /usr/local/bin/flanneld $i:/usr/local/bin/;done
for i in k8s-node1 k8s-node2;do scp /usr/local/bin/mk-docker-opts.sh $i:/usr/local/bin/;done

配置Flannel (所有节点)

mkdir -p /etc/flannel
cat > /etc/flannel/flannel.cfg<<EOF
FLANNEL_OPTIONS="-etcd-endpoints=https://172.16.115.237:2379 -etcd-cafile=/etc/etcd/ssl/ca.pem -etcd-certfile=/etc/etcd/ssl/server.pem  -etcd-keyfile=/etc/etcd/ssl/server-key.pem  --healthz-ip=0.0.0.0 --healthz-port=7100"
EOF
#多个ETCD: -etcd-endpoints=https://172.16.115.237:2379,https://172.16.115.238:2379,https://172.16.115.239:2379

配置Flannel配置文件

cat > /usr/lib/systemd/system/flanneld.service <<EOF
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service

[Service]
Type=notify
EnvironmentFile=/etc/flannel/flannel.cfg
ExecStart=/usr/local/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/usr/local/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

启动Flannel

service flanneld start
chkconfig flanneld on
service flanneld status

所有的节点都需要有172.17.0.0/16 网段IP

ip a | grep flannel

节点停止flanneld

service flanneld stop

修改Docker启动文件

cat >/usr/lib/systemd/system/docker.service<<EOFL
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target

[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s

[Install]
WantedBy=multi-user.target
EOFL

重启Docker服务

systemctl daemon-reload
service flanneld restart
service docker restart

检查IP地址, docker 与flanneld 是同一个网段

ip a

 在每个Node节点Ping其他的节点, 网段都是通的。

[root@k8s-node1 ~]# ping 172.17.95.1
PING 172.17.95.1 (172.17.95.1) 56(84) bytes of data.
64 bytes from 172.17.95.1: icmp_seq=1 ttl=64 time=0.322 ms
64 bytes from 172.17.95.1: icmp_seq=2 ttl=64 time=0.225 ms
172.17.95.164 bytes from 172.17.95.1: icmp_seq=3 ttl=64 time=0.232 ms
64 bytes from 172.17.95.1: icmp_seq=4 ttl=64 time=0.245 ms

5安装Master 组件

Master端需要安装的组件如下:
kube-apiserver
kube-scheduler
kube-controller-manager

安装Api Server服务

下载Kubernetes二进制包(1.15.1)（master-1）

cd /soft
wget https://dl.k8s.io/v1.15.10/kubernetes-server-linux-amd64.tar.gz
tar xvf kubernetes-server-linux-amd64.tar.gz 
cd kubernetes/server/bin/
cp kube-scheduler kube-apiserver kube-controller-manager kubectl /usr/local/bin/

配置Kubernetes证书

#Kubernetes各个组件之间通信需要证书,需要复制个每个master节点（master-1）

mkdir -p /etc/kubernetes/{cfg,ssl}
cp /root/kubernetes/*.pem /etc/kubernetes/ssl/

复制到其他的节点

for i in k8s-node1 k8s-node2;do ssh $i mkdir -p /etc/kubernetes/{cfg,ssl};done
for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done
for i in k8s-node1 k8s-node2;do echo $i "---------->"; ssh $i ls /etc/kubernetes/ssl;done

创建 TLS Bootstrapping Token

# TLS bootstrapping 功能就是让 kubelet 先使用一个预定的低权限用户连接到 apiserver，
然后向 apiserver 申请证书，kubelet 的证书由 apiserver 动态签署
#Token可以是任意的包涵128 bit的字符串，可以使用安全的随机数发生器生成

head -c 16 /dev/urandom | od -An -t x | tr -d ' '
b78d1a70a76180a848090a178c806229

编辑Token 文件(master-1)

#b78d1a70a76180a848090a178c806229:随机字符串,自定义生成; kubelet-bootstrap:用户名; 10001:UID; system:kubelet-bootstrap：用户组

echo b78d1a70a76180a848090a178c806229,kubelet-bootstrap,10001,"system:kubelet-bootstrap" /etc/kubernetes/cfg/token.csv
#如有其他msater节点请复制到其他的master节点

创建Apiserver配置文件(所有的master节点)

#配置文件内容基本相同, 如果有多个节点, 那么需要修改IP地址即可

cat >/etc/kubernetes/cfg/kube-apiserver.cfg <<EOFL
KUBE_APISERVER_OPTS="--logtostderr=true \
--v=4 \
--insecure-bind-address=0.0.0.0 \
--insecure-port=8080 \
--etcd-servers=https://172.16.115.237:2379 \
--bind-address=0.0.0.0 \
--secure-port=6443 \
--advertise-address=0.0.0.0 \
--allow-privileged=true \
--service-cluster-ip-range=10.0.0.0/24 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth \
--token-auth-file=/etc/kubernetes/cfg/token.csv \
--service-node-port-range=30000-50000 \
--tls-cert-file=/etc/kubernetes/ssl/server.pem  \
--tls-private-key-file=/etc/kubernetes/ssl/server-key.pem \
--client-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-key-file=/etc/kubernetes/ssl/ca-key.pem \
--etcd-cafile=/etc/etcd/ssl/ca.pem \
--etcd-certfile=/etc/etcd/ssl/server.pem \
--etcd-keyfile=/etc/etcd/ssl/server-key.pem"
EOFL

#参数说明
--logtostderr 							启用日志 
---v 									日志等级
--etcd-servers 						etcd 集群地址 
--etcd-servers=https://172.16.115.237:2379
--bind-address 						监听地址 
--secure-port https						安全端口 
--advertise-address     				集群通告地址 
--allow-privileged      				启用授权 
--service-cluster-ip-range Service 			虚拟IP地址段 
--enable-admission-plugins 				准入控制模块 
--authorization-mode  					认证授权,启用RBAC授权
--enable-bootstrap-token-auth 		    启用TLS bootstrap功能
--token-auth-file 						token 文件 
--service-node-port-range 				Service Node类型默认分配端口范围

配置kube-apiserver 启动文件(所有的master节点)

cat >/usr/lib/systemd/system/kube-apiserver.service<<EOFL
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-apiserver.cfg
ExecStart=/usr/local/bin/kube-apiserver \$KUBE_APISERVER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOFL

启动kube-apiserver服务

service kube-apiserver start 
chkconfig kube-apiserver on
service kube-apiserver status

查看加密的端口是否已经启动

netstat -anltup | grep 6443

查看加密的端口是否已经启动（node节点）

telnet 172.16.115.237 6443

部署kube-scheduler 服务

#创建kube-scheduler配置文件（所有的master节点）

cat >/etc/kubernetes/cfg/kube-scheduler.cfg<<EOFL
KUBE_SCHEDULER_OPTS="--logtostderr=true --v=4 --bind-address=0.0.0.0 --master=127.0.0.1:8080 --leader-elect"
EOFL

查看配置文件

cat  /etc/kubernetes/cfg/kube-scheduler.cfg

创建kube-scheduler 启动文件

#创建kube-scheduler systemd unit 文件（所有的master节点）

cat >/usr/lib/systemd/system/kube-scheduler.service<<EOFL
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-scheduler.cfg
ExecStart=/usr/local/bin/kube-scheduler \$KUBE_SCHEDULER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOFL

启动kube-scheduler服务（所有的master节点）

service kube-scheduler restart
chkconfig kube-scheduler on

查看Master节点组件状态（任意一台master）

kubectl get cs

部署kube-controller-manager

创建kube-controller-manager配置文件(所有节点)

cat >/etc/kubernetes/cfg/kube-controller-manager.cfg<<EOFL
KUBE_CONTROLLER_MANAGER_OPTS="--logtostderr=true \
--v=4 \
--master=127.0.0.1:8080 \
--leader-elect=true \
--address=0.0.0.0 \
--service-cluster-ip-range=10.0.0.0/24 \
--cluster-name=kubernetes \
--cluster-signing-cert-file=/etc/kubernetes/ssl/ca.pem \
--cluster-signing-key-file=/etc/kubernetes/ssl/ca-key.pem  \
--root-ca-file=/etc/kubernetes/ssl/ca.pem \
--service-account-private-key-file=/etc/kubernetes/ssl/ca-key.pem"
EOFL

参数说明

--master=127.0.0.1:8080  #指定Master地址
--leader-elect 			 #竞争选举机制产生一个 leader 节点，其它节点为阻塞状态。
--service-cluster-ip-range #kubernetes service 指定的IP地址范围。

创建kube-controller-manager 启动文件

cat  >/usr/lib/systemd/system/kube-controller-manager.service<<EOFL
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-controller-manager.cfg
ExecStart=/usr/local/bin/kube-controller-manager \$KUBE_CONTROLLER_MANAGER_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOFL

启动kube-controller-manager服务

chkconfig kube-controller-manager on
service kube-controller-manager start
service kube-controller-manager status

查看Master 节点组件状态

必须要在各个节点组件正常的情况下, 才去部署Node节点组件.（master节点）

kubectl get cs
NAME                 STATUS    MESSAGE             ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   {"health":"true"}

6部署Node节点组件

部署 kubelet 组件

从Master节点复制Kubernetes 文件到Node

cd /soft
for i in k8s-node1 k8s-node2;do scp kubernetes/server/bin/kubelet kubernetes/server/bin/kube-proxy $i:/usr/local/bin/;done

创建kubelet bootstrap.kubeconfig 文件

Maste-1节点

mkdir /root/config ; cd /root/config
cat >environment.sh<<EOFL
# 创建kubelet bootstrapping kubeconfig
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"
# 设置集群参数
kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=\${KUBE_APISERVER} \
  --kubeconfig=bootstrap.kubeconfig
# 设置客户端认证参数
kubectl config set-credentials kubelet-bootstrap \
  --token=\${BOOTSTRAP_TOKEN} \
  --kubeconfig=bootstrap.kubeconfig
# 设置上下文参数
kubectl config set-context default \
  --cluster=kubernetes \
  --user=kubelet-bootstrap \
  --kubeconfig=bootstrap.kubeconfig
# 设置默认上下文
kubectl config use-context default --kubeconfig=bootstrap.kubeconfig
#通过 bash environment.sh获取 bootstrap.kubeconfig 配置文件。
EOFL

执行脚本

sh environment.sh

创建kube-proxy kubeconfig文件 （master-1）

cat  >env_proxy.sh<<EOF
# 创建kube-proxy kubeconfig文件
BOOTSTRAP_TOKEN=b78d1a70a76180a848090a178c806229
KUBE_APISERVER="https://172.16.115.237:6443"

kubectl config set-cluster kubernetes \
  --certificate-authority=/etc/kubernetes/ssl/ca.pem \
  --embed-certs=true \
  --server=\${KUBE_APISERVER} \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-credentials kube-proxy \
  --client-certificate=/etc/kubernetes/ssl/kube-proxy.pem \
  --client-key=/etc/kubernetes/ssl/kube-proxy-key.pem \
  --embed-certs=true \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config set-context default \
  --cluster=kubernetes \
  --user=kube-proxy \
  --kubeconfig=kube-proxy.kubeconfig

kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig
EOF

执行脚本

sh env_proxy.sh

复制kubeconfig文件与证书到所有Node节点

将bootstrap kubeconfig kube-proxy.kubeconfig 文件复制到所有Node节点
远程创建目录 (master-1)
 

for i in k8s-node1 k8s-node2;do ssh $i "mkdir -p /etc/kubernetes/{cfg,ssl}";done

复制证书文件ssl  (master-1)

for i in k8s-node1 k8s-node2;do scp /etc/kubernetes/ssl/* $i:/etc/kubernetes/ssl/;done

复制kubeconfig文件  (master-1)

for i in k8s-node1 k8s-node2;do scp -rp bootstrap.kubeconfig kube-proxy.kubeconfig $i:/etc/kubernetes/cfg/;done

创建kubelet参数配置文件

不同的Node节点, 需要修改IP地址 （node节点操作）

cat >/etc/kubernetes/cfg/kubelet.config<<EOF
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
address: 172.16.115.238
port: 10250
readOnlyPort: 10255
cgroupDriver: cgroupfs
clusterDNS: ["10.0.0.2"]
clusterDomain: cluster.local.
failSwapOn: false
authentication:
  anonymous:
    enabled: true
EOF

创建kubelet配置文件

不同的Node节点, 需要修改IP地址

/etc/kubernetes/cfg/kubelet.kubeconfig 文件自动生成

cat >/etc/kubernetes/cfg/kubelet<<EOF
KUBELET_OPTS="--logtostderr=true \
--v=4 \
--hostname-override=172.16.115.239 \
--kubeconfig=/etc/kubernetes/cfg/kubelet.kubeconfig \
--bootstrap-kubeconfig=/etc/kubernetes/cfg/bootstrap.kubeconfig \
--config=/etc/kubernetes/cfg/kubelet.config \
--cert-dir=/etc/kubernetes/ssl \
--pod-infra-container-image=docker.io/kubernetes/pause:latest"
EOF

创建kubelet系统启动文件(node节点)

cat >/usr/lib/systemd/system/kubelet.service<<EOF
[Unit]
Description=Kubernetes Kubelet
After=docker.service
Requires=docker.service

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kubelet
ExecStart=/usr/local/bin/kubelet \$KUBELET_OPTS
Restart=on-failure
KillMode=process

[Install]
WantedBy=multi-user.target
EOF

将kubelet-bootstrap用户绑定到系统集群角色

master-1节点操作

kubectl create clusterrolebinding kubelet-bootstrap \
  --clusterrole=system:node-bootstrapper \
  --user=kubelet-bootstrap

启动kubelet服务（node节点）

chkconfig kubelet on 
service kubelet start 
service kubelet status

服务端批准与查看CSR请求

查看CSR请求
Maste-1节点操作
 

kubectl get csr
NAME                                                   AGE   REQUESTOR           CONDITION
node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM   32s   kubelet-bootstrap   Pending
node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM   30s   kubelet-bootstrap   Pending

批准请求

Master节点操作

kubectl certificate approve node-csr-elVwxGEknQrBQ3IowikGxUfygraO3MIwlaibMkfMFoM
kubectl certificate approve node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM

节点重名处理

如果出现节点重名, 可以先删除证书, 然后重新申请
Master节点删除csr
 

kubectl delete csr node-csr-pAIbikYclOa8vVu7aXesSPh-fmo62rztJH9kMHV9hXM

Node节点删除kubelet.kubeconfig

客户端重启kubelet服务, 再重新申请证书

rm -rf /etc/kubernetes/cfg/kubelet.kubeconfig

查看节点状态 

所有的Node节点状态必须为Ready （master）

kubectl get nodesNAME             
STATUS   ROLES    AGE     VERSION
172.16.115.238   Ready    <none>   2m37s   v1.15.10
172.16.115.239   Ready    <none>   2m23s   v1.15.10

部署kube-proxy 组件

kube-proxy 运行在所有Node节点上, 监听Apiserver 中 Service 和 Endpoint 的变化情况，创建路由规则来进行服务负载均衡。

创建kube-proxy配置文件

注意修改hostname-override地址, 不同的节点则不同。

cat >/etc/kubernetes/cfg/kube-proxy<<EOF
KUBE_PROXY_OPTS="--logtostderr=true \
--v=4 \
--metrics-bind-address=0.0.0.0 \
--hostname-override=172.16.115.239 \
--cluster-cidr=10.0.0.0/24 \
--kubeconfig=/etc/kubernetes/cfg/kube-proxy.kubeconfig"
EOF

创建kube-proxy systemd unit 文件

cat >/usr/lib/systemd/system/kube-proxy.service<<EOF
[Unit]
Description=Kubernetes Proxy
After=network.target

[Service]
EnvironmentFile=/etc/kubernetes/cfg/kube-proxy
ExecStart=/usr/local/bin/kube-proxy \$KUBE_PROXY_OPTS
Restart=on-failure

[Install]
WantedBy=multi-user.target
EOF

启动kube-proxy 服务

chkconfig kube-proxy on 
service kube-proxy start 
service kube-proxy status

运行Demo项目

kubectl run nginx --image=nginx --replicas=2
kubectl run --generator=deployment/apps.v1 is DEPRECATED and will be removed in a future version. Use kubectl run --generator=run-pod/v1 or kubectl create instead.
deployment.apps/nginx created

获取容器IP与运行节点

kubectl get pods -o wide

创建容器svc端口

kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort

查看容器状态

kubectl describe pod nginx-7bb7cd8db5-7ndfk

查看SVC

kubectl get svc
NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)        AGE
kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP        3h14m
nginx        NodePort    10.0.0.129   <none>        88:34947/TCP   94s

访问web

curl http://172.16.115.238:34947
<!DOCTYPE html>
<html>
<head>
<title>Welcome to nginx!</title>
<style>
html { color-scheme: light dark; }
body { width: 35em; margin: 0 auto;
font-family: Tahoma, Verdana, Arial, sans-serif; }
</style>
</head>
<body>
<h1>Welcome to nginx!</h1>
<p>If you see this page, the nginx web server is successfully installed and
working. Further configuration is required.</p>

<p>For online documentation and support please refer to
<a href="http://nginx.org/">nginx.org</a>.<br/>
Commercial support is available at
<a href="http://nginx.com/">nginx.com</a>.</p>

<p><em>Thank you for using nginx.</em></p>
</body>
</html>

删除项目

kubectl delete deployment nginx 
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx

服务启动顺序

启动Master节点

service keepalived start
service etcd start
service kube-scheduler start
service kube-controller-manager start
service kube-apiserver  restart
kubectl get cs

启动Node节点

service flanneld start
service docker start
service kubelet start
service kube-proxy start

停止Node节点

service kubelet stop
service kube-proxy stop
service docker stop
service flanneld stop

停止Master 节点

service kube-controller-manager stop
service kube-scheduler stop
service etcd stop
service keepalived stop

7部署DNS

部署coredns

cd /soft/kubernetes
tar xf kubernetes-src.tar.gz
mkdir /root/dns && cd /root/dns
cp /soft/kubernetes/cluster/addons/dns/coredns/coredns.yaml.sed /root/dns/coredns.yaml
修改一些参数：
修改参数有3个地方，一个是ip6.arpa 指定，一个是更改成国内镜像源，一个是定义clusterIP，具体如下
ip6.arpa修改为kubernetes cluster.local. in-addr.arpa ip6.arpa
国内镜像修改为 coredns/coredns:1.2.6
clusterIP修改为自己集群设置的IP范围内的，我集群的是 10.0.0.0/24,所以设置为10.0.0.2（且不为已使用的IP）
修改resources:
          limits:
            memory: 1Gi
          requests:
            cpu: 1028m
            memory: 500Mi

具体的yaml是(我们只需修改clusterIP的IP为自己集群的IP范围内，且不重复的)

作者：凤非飞
链接：https://www.jianshu.com/p/d655e35bf54a
来源：简书
著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
kubectl apply -f coredns.yaml

查询所有ns中的pod

kubectl get pod -A

查询指定ns中的pod

kubectl get pod -n kube-system

查看启动进程

kubectl get pod -n kube-system coredns-76fb6cf764-c5tln -n kube-system

查看SVC

kubectl get svc -o wide -n=kube-system
NAME       TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE   SELECTOR
kube-dns   ClusterIP   10.0.0.254   <none>        53/UDP,53/TCP,9153/TCP   87s   k8s-app=kube-dns

验证DNS是否有效

删除之前创建的nginx demo

kubectl delete deployment nginx 
kubectl delete pods nginx
kubectl delete svc -l run=nginx
kubectl delete deployment.apps/nginx

启动新容器

kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools

#出现错误
error: unable to upgrade connection: Forbidden (user=system:anonymous, verb=create, resource=nodes, subresource=proxy)
#解决方法

kubectl create clusterrolebinding system:anonymous --clusterrole=cluster-admin --user=system:anonymous
kubectl delete pod  dnstools
kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools

创建Nginx 容器

kubectl run nginx --image=nginx --replicas=2

创建svc (cluster IP)

kubectl expose deployment nginx --port=88 --target-port=80 --type=NodePort

查看SVC

kubectl get svc -A
NAMESPACE     NAME         TYPE        CLUSTER-IP   EXTERNAL-IP   PORT(S)                  AGE
default       kubernetes   ClusterIP   10.0.0.1     <none>        443/TCP                  4h50m
default       nginx        NodePort    10.0.0.221   <none>        88:42616/TCP             32m
kube-system   kube-dns     ClusterIP   10.0.0.2     <none>        53/UDP,53/TCP,9153/TCP   4s

测试解析Nginx

dns 解析的名称是svc (service 名称, 非pod名称)

kubectl run -it --rm --restart=Never --image=infoblox/dnstools:latest dnstools
If you don't see a command prompt, try pressing enter.
dnstools# nslookup nginx
Server:         10.0.0.2
Address:        10.0.0.2#53

Name:   nginx.default.svc.cluster.local
Address: 10.0.0.221

案例:容器的网络访问不区分命名空间（kubernetes ns）

在default ns 可以访问到kube-system ns 服务nginx

kubectl run nginx-n1 --image=nginx --replicas=1 -n kube-system

查看容器状态（指定命名空间）

kubectl get pods -n kube-system

查看容器状态（显示所有的命名空间）

kubectl get pod,svc -A
kubectl expose deployment nginx-n1 --port=99 --target-port=80 -n kube-system

跨ns访问服务

kubectl get svc -n kube-system | grep nginx-n1  
nginx-n1   ClusterIP   10.0.0.68    <none>        99/TCP                   10s

访问服务

curl 10.0.0.68

解析不成功

nslookup nginx-n1
Server:         10.0.0.2
Address:        10.0.0.2#53

** server can't find nginx-n1: NXDOMAIN

解决方法(默认解析为default空间)

nslookup nginx-n1.kube-system.svc.cluster.local
Server:         10.0.0.2
Address:        10.0.0.2#53

Name:   nginx-n1.kube-system.svc.cluster.local
Address: 10.0.0.68

部署Dashboard

下载文件

创建目录(master-1节点)

mkdir /root/dashboard
wget https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.1/src/deploy/recommended/kubernetes-dashboard.yaml

修改端口

修改为nodeport端口50000
注意镜像地址无法下载, 使用另外的镜像替换
 

mirrorgooglecontainers/kubernetes-dashboard-amd64:v1.10.1
sed -i '/targetPort:/a\ \ \ \ \ \ nodePort: 50000\n\ \ type: NodePort' kubernetes-dashboard.1.10.yaml

部署

kubectl apply -f kubernetes-dashboard.yaml

查看服务端口

kubectl get services -n kube-system

创建用户授权

kubectl create serviceaccount  dashboard-admin -n kube-system
kubectl create clusterrolebinding  \
dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin

获取Token

kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
Name:         dashboard-admin-token-ptxn8
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: 2c17e5ee-c11b-4fde-bd0b-02235f433ee0

Type:  kubernetes.io/service-account-token

Data
====
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW
4tdG9rZW4tcHR4bjgiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiMmMxN2U1ZWUtYzExYi00ZmRlLWJkMGItMDIyMzVmNDMzZWUwIiwic3ViIjoic
3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.i2728HQZ5NRYdwXpSVNnTXK0UEQJ1va6XFyrJMdJ--2QHvV0xzUfF4uEM0HSMnXUFVoLjRZMarGYJ6_4eptoIW7Px0vKUoWY2--SU-ET-pPXKLFBMQ-c8FSKjuMa1W6-fz1Z4oJyt_4VNRbJrzQbhYcQm8EeiVBs-N0pd4qvoUnJqR2
z_TgdctONPKsQCCy7BadzApoGT00TFGvVEjC-4kOD4ZoWXURItK89VgMAmgxxGmxr3WrD_e-lad9Za_CElA_n90eBPOP5JB3yV0iAxsV6uyo2HZagrvHzbfj7hzmCaHPLlvQc4tApDhnN1QY-RHaJKitZiebQ8TUsmT9DRQ
ca.crt:     1359 bytes
namespace:  11 bytes

登录系统

http://nodeip:50000 访问

8.部署kuboard

kubectl apply -f https://kuboard.cn/install-script/kuboard.yaml

获取token

kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')

获取nodeport端口

kubectl get svc -n kube-system
kuboard                NodePort    10.0.0.95    <none>        80:32567/TCP             15h

登录系统

http://nodeip:32567 访问

9部署Ingress 

服务反向代理
部署Traefik 2.0版本
创建 traefik-crd.yaml 文件 (master-1)

cd /root/ingress

cat >/root/ingress/traefik-crd.yaml<<EOF
## IngressRoute
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingressroutes.traefik.containo.us
spec:
  scope: Namespaced
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: IngressRoute
    plural: ingressroutes
    singular: ingressroute
---
## IngressRouteTCP
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: ingressroutetcps.traefik.containo.us
spec:
  scope: Namespaced
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: IngressRouteTCP
    plural: ingressroutetcps
    singular: ingressroutetcp
---
## Middleware
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: middlewares.traefik.containo.us
spec:
  scope: Namespaced
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: Middleware
    plural: middlewares
    singular: middleware
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
  name: tlsoptions.traefik.containo.us
spec:
  scope: Namespaced
  group: traefik.containo.us
  version: v1alpha1
  names:
    kind: TLSOption
    plural: tlsoptions
    singular: tlsoption
EOF

创建Traefik CRD资源(master-1)

cd /root/ingress
kubectl create -f traefik-crd.yaml -n kube-system
kubectl get CustomResourceDefinition

创建Traefik  RABC文件(master-1)

cat >/root/ingress/traefik-rbac.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  namespace: kube-system
  name: traefik-ingress-controller
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik-ingress-controller
rules:
  - apiGroups: [""]
    resources: ["services","endpoints","secrets"]
    verbs: ["get","list","watch"]
  - apiGroups: ["extensions"]
    resources: ["ingresses"]
    verbs: ["get","list","watch"]
  - apiGroups: ["extensions"]
    resources: ["ingresses/status"]
    verbs: ["update"]
  - apiGroups: ["traefik.containo.us"]
    resources: ["middlewares"]
    verbs: ["get","list","watch"]
  - apiGroups: ["traefik.containo.us"]
    resources: ["ingressroutes"]
    verbs: ["get","list","watch"]
  - apiGroups: ["traefik.containo.us"]
    resources: ["ingressroutetcps"]
    verbs: ["get","list","watch"]
  - apiGroups: ["traefik.containo.us"]
    resources: ["tlsoptions"]
    verbs: ["get","list","watch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
  name: traefik-ingress-controller
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: traefik-ingress-controller
subjects:
  - kind: ServiceAccount
    name: traefik-ingress-controller
    namespace: kube-system
EOF

创建RABC 资源

kubectl create -f traefik-rbac.yaml -n kube-system

创建Traefik ConfigMap (master-1)

cat >/root/ingress/traefik-config.yaml<<EOF 
kind: ConfigMap
apiVersion: v1
metadata:
  name: traefik-config
data:
  traefik.yaml: |-
    serversTransport:
      insecureSkipVerify: true
    api:
      insecure: true
      dashboard: true
      debug: true
    metrics:
      prometheus: ""
    entryPoints:
      web:
        address: ":80"
      websecure:
        address: ":443"
    providers:
      kubernetesCRD: ""
    log:
      filePath: ""
      level: error
      format: json
    accessLog:
      filePath: ""
      format: json
      bufferingSize: 0
      filters:
        retryAttempts: true
        minDuration: 20
      fields:
        defaultMode: keep
        names:
          ClientUsername: drop
        headers:
          defaultMode: keep
          names:
            User-Agent: redact
            Authorization: drop
            Content-Type: keep
EOF 

创建Traefik ConfigMap资源配置

kubectl apply -f traefik-config.yaml -n kube-system

设置节点标签

设置节点label

kubectl label nodes 172.16.115.238 IngressProxy=true
kubectl label nodes 172.16.115.239 IngressProxy=true

查看节点标签

检查是否成功

kubectl get nodes --show-labels
NAME             STATUS   ROLES    AGE   VERSION    LABELS
172.16.115.238   Ready    <none>   44h   v1.15.10   IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.238,kubernetes.io/os=linux
172.16.115.239   Ready    <none>   44h   v1.15.10   IngressProxy=true,beta.kubernetes.io/arch=amd64,beta.kubernetes.io/os=linux,kubernetes.io/arch=amd64,kubernetes.io/hostname=172.16.115.239,kubernetes.io/os=linux

创建 traefik 部署文件

注意每个Node节点的80与443端口不能被占用

netstat -antupl | grep -E "80|443"

cat >/root/ingress/traefik-deploy.yaml<<EOF
apiVersion: v1
kind: Service
metadata:
  name: traefik
spec:
  ports:
    - name: web
      port: 80
    - name: websecure
      port: 443
    - name: admin
      port: 8080
  selector:
    app: traefik
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: traefik-ingress-controller
  labels:
    app: traefik
spec:
  selector:
    matchLabels:
      app: traefik
  template:
    metadata:
      name: traefik
      labels:
        app: traefik
    spec:
      serviceAccountName: traefik-ingress-controller
      terminationGracePeriodSeconds: 1
      containers:
        - image: traefik:v2.0.5
          name: traefik-ingress-lb
          ports:
            - name: web
              containerPort: 80
              hostPort: 80           #hostPort方式，将端口暴露到集群节点
            - name: websecure
              containerPort: 443
              hostPort: 443          #hostPort方式，将端口暴露到集群节点
            - name: admin
              containerPort: 8080
          resources:
            limits:
              cpu: 300m
              memory: 500Mi
            requests:
              cpu: 100m
              memory: 102Mi
          securityContext:
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
          args:
            - --configfile=/config/traefik.yaml
          volumeMounts:
            - mountPath: "/config"
              name: "config"
      volumes:
        - name: config
          configMap:
            name: traefik-config 
      tolerations:              #设置容忍所有污点，防止节点被设置污点
        - operator: "Exists"
      nodeSelector:             #设置node筛选器，在特定label的节点上启动
        IngressProxy: "true"

EOF

部署 Traefik 资源

kubectl apply -f traefik-deploy.yaml -n kube-system

kubectl get DaemonSet -A
NAMESPACE     NAME                         DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR       AGE
kube-system   traefik-ingress-controller   2         2         0       2            0           IngressProxy=true   6s

Traefik 路由配置
配置Traefik Dashboard
 

cat >/root/ingress/traefik-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: traefik-dashboard-route
spec:
  entryPoints:
    - web
  routes:
    - match: Host(`traefik.lateautumn4lin.dashboard`)
      kind: Rule
      services:
        - name: traefik
          port: 8080
EOF

创建Ingress (traefik)

kubectl apply -f traefik-dashboard-route.yaml -n kube-system

客户端访问Traefik Dashboard

绑定物理主机Hosts文件或者域名解析
/etc/hosts
172.16.115.239 ingress.hostscc.com

访问web

ingress.hostscc.com

部署访问服务(http)

创建https服务
代理dashboard https 服务
 创建自签名证书

cd /root/ingress
openssl req -x509 -nodes -days 3650 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=cloud.lateautumn4lin.dashboard"

将证书存储到 Kubernetes Secret中

kubectl create secret generic cloud-mydlq-tls --from-file=tls.crt --from-file=tls.key -n kube-system

查看系统secret

kubectl get secret -n kube-system
dashboard-tls                            kubernetes.io/tls                     2      41s

创建路由文件

先查询kuberbentes dashboard 的命名空间

cat >/root/ingress/kubernetes-dashboard-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: kubernetes-dashboard-route
spec:
  entryPoints:
    - websecure
  tls:
    secretName: cloud-mydlq-tls
  routes:
    - match: Host(`cloud.lateautumn4lin.dashboard`) 
      kind: Rule
      services:
        - name: kubernetes-dashboard
          port: 443
EOF

创建 Kubernetes Dashboard 路由规则对象

kubectl apply  -f kubernetes-dashboard-route.yaml -n kube-system

查看创建的路由

kubectl get IngressRoute -A      

绑定hosts 访问

172.16.115.239 cloud.lateautumn4lin.dashboard

获取token

kubectl -n kube-system describe secret $(kubectl -n kube-system get secret | grep kuboard-user | awk '{print $1}')

9部署监控系统

监控方案

Prometheus 与 Grafana

安装NFS服务端master节点安装nfs

yum -y install nfs-utils

创建nfs目录

mkdir -p /ifs/kubernetes

修改权限

 chmod -R 777 /ifs/kubernetes

编辑export文件

cat >/etc/exports<<EOF
/ifs/kubernetes *(rw,no_root_squash,sync)
EOF

修改配置启动文件
#修改配置文件

 cat >/etc/systemd/system/sockets.target.wants/rpcbind.socket<<EOFL
[Unit]
Description=RPCbind Server Activation Socket
[Socket]
ListenStream=/var/run/rpcbind.sock
ListenStream=0.0.0.0:111
ListenDatagram=0.0.0.0:111
[Install]
WantedBy=sockets.target
EOFL

启动rpcbind、nfs服务

systemctl restart rpcbind 
systemctl enable rpcbind
systemctl restart nfs 
systemctl enable nfs

配置生效

 exportfs -f

showmount测试(master-1)

showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *

所有node节点安装客户端

yum -y install nfs-utils

所有的Node检查

showmount -e k8s-master
Export list for k8s-master:
/ifs/kubernetes *

部署PVC 
Nfs服务端地址需要修改

mkdir /root/nfs

cat >/root/nfs/nfs-deployment.yaml<<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
---
kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-client-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          imagePullPolicy: IfNotPresent
          image: quay.io/external_storage/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: fuseim.pri/ifs
            - name: NFS_SERVER
              value: 172.16.115.237
            - name: NFS_PATH
              value: /ifs/kubernetes
      volumes:
        - name: nfs-client-root
          nfs:
            server: 172.16.115.237
            path: /ifs/kubernetes
EOF

cat >/root/nfs/nfs-class.yaml<<EOF
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: managed-nfs-storage
provisioner: fuseim.pri/ifs # or choose another name, must match deployment's env PROVISIONER_NAME'
parameters:
  archiveOnDelete: "true"
EOF

cat >/root/nfs/nfs-rabc.yaml<<EOF
kind: ServiceAccount
apiVersion: v1
metadata:
  name: nfs-client-provisioner
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    # replace with namespace where provisioner is deployed
    namespace: default
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io
EOF

kubectl apply  -f  nfs-class.yaml 
kubectl apply  -f  nfs-deployment.yaml
kubectl apply  -f  nfs-rabc.yaml

查看nfs pod状态

kubectl get pods
NAME                                     READY   STATUS    RESTARTS   AGE
nfs-client-provisioner-9bf85b8fc-fcj7k   1/1     Running   0          23s

查看是否部署成功

kubectl get StorageClass
NAME                  PROVISIONER      AGE
managed-nfs-storage   fuseim.pri/ifs   61s

部署监控系统

注意需要修改的配置文件
配置configmap文件

cat >prometheus-configmap.yaml<<EOF
apiVersion: v1
kind: ConfigMap
metadata:
  name: prometheus-config
  namespace: kube-system
data:
  prometheus.yml: |
    global:
      scrape_interval:     15s
      evaluation_interval: 15s
    rule_files:
    - /etc/prometheus/rules.yml
    alerting:
      alertmanagers:
        - static_configs:
          - targets: ["alertmanager:9093"]
    scrape_configs:

    - job_name: 'kubernetes-apiservers'
      kubernetes_sd_configs:
      - role: endpoints
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - source_labels: [__meta_kubernetes_namespace, __meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name]
        action: keep
        regex: default;kubernetes;https

    - job_name: 'kubernetes-cadvisor'
      kubernetes_sd_configs:
      - role: node
      scheme: https
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      relabel_configs:
      - action: labelmap
        regex: __meta_kubernetes_node_label_(.+)
      - target_label: __address__
        replacement: kubernetes.default.svc:443
      - source_labels: [__meta_kubernetes_node_name]
        regex: (.+)
        target_label: __metrics_path__
        replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor

    - job_name: 'kubernetes-service-endpoints'
      kubernetes_sd_configs:
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
        action: keep
        regex: true
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        action: replace
        target_label: kubernetes_name

    - job_name: 'kubernetes-services'
      kubernetes_sd_configs:
      - role: service
      metrics_path: /probe
      params:
        module: [http_2xx]
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_probe]
        action: keep
        regex: true
      - source_labels: [__address__]
        target_label: __param_target
      - target_label: __address__
        replacement: blackbox-exporter.example.com:9115
      - source_labels: [__param_target]
        target_label: instance
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_service_name]
        target_label: kubernetes_name

    - job_name: 'kubernetes-ingresses'
      kubernetes_sd_configs:
      - role: ingress
      relabel_configs:
      - source_labels: [__meta_kubernetes_ingress_annotation_prometheus_io_probe]
        action: keep
        regex: true
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
        action: replace
        regex: ([^:]+)(?::\d+)?;(\d+)
        replacement: $1:$2
        target_label: __address__
      - action: labelmap
        regex: __meta_kubernetes_pod_label_(.+)
      - source_labels: [__meta_kubernetes_namespace]
        action: replace
        target_label: kubernetes_namespace
      - source_labels: [__meta_kubernetes_pod_name]
        action: replace
        target_label: kubernetes_pod_name
        
    - job_name: 'kubernetes_node'
      tls_config:
        ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
      bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
      kubernetes_sd_configs:
      # 基于endpoint的服务发现，不再经过service代理层面
      - role: endpoints
      relabel_configs:
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape, __meta_kubernetes_endpoint_port_name]
        regex: true;prometheus-node-exporter
        action: keep
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
        action: replace
        target_label: __scheme__
        regex: (https?)
      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
        action: replace
        target_label: __metrics_path__
        regex: (.+)
      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
        action: replace
        target_label: __address__
        regex: (.+)(?::\d+);(\d+)
        replacement: $1:$2
      # 去掉label name中的前缀__meta_kubernetes_service_label_
      - action: labelmap
        regex: __meta_kubernetes_service_label_(.+)
      # 为了区分所属node,把instance 从node-exporter ep的实例，替换成ep所在node的ip
      - source_labels: [__meta_kubernetes_pod_host_ip]
        regex: '(.*)'
        replacement: '${1}'
        target_label: instance
EOF

部署

kubectl apply -f prometheus-configmap.yaml

部署prometheus工作主程序，注意挂载上面的configmap：

cat >prometheus.deploy.yml<<EOF
apiVersion: apps/v1beta2
kind: Deployment
metadata:
  labels:
    name: prometheus-deployment
  name: prometheus
  namespace: kube-system
spec:
  replicas: 1
  selector:
    matchLabels:
      app: prometheus
  template:
    metadata:
      labels:
        app: prometheus
    spec:
      containers:
      - image: prom/prometheus:v2.0.0
        name: prometheus
        command:
        - "/bin/prometheus"
        args:
        - "--config.file=/etc/prometheus/prometheus.yml"
        - "--storage.tsdb.path=/prometheus"
        - "--storage.tsdb.retention=24h"
        ports:
        - containerPort: 9090
          protocol: TCP
        volumeMounts:
        - mountPath: "/prometheus"
          name: data
        - mountPath: "/etc/prometheus"
          name: config-volume
        resources:
          requests:
            cpu: 100m
            memory: 100Mi
          limits:
            cpu: 500m
            memory: 2500Mi
      serviceAccountName: prometheus    
      volumes:
      - name: data
        emptyDir: {}
      - name: config-volume
        configMap:
          name: prometheus-config  

EOF

部署

kubectl apply -f prometheus.deploy.yml

部署svc、ingress、rbac授权。
注意：在本地是使用traefik做对外服务代理的，因此修改了默认的NodePort的svc.type为ClusterIP的方式，Ingress添加后，可以以域名方式直接访问。若不做代理，可以无需部署ingress，svc.type使用默认的NodePort，然后通过node ip+port的形式访问

cat >prometheus.svc.yaml<<EOF
kind: Service
apiVersion: v1
metadata:
  labels:
    app: prometheus
  name: prometheus
  namespace: kube-system
spec:
  type: ClusterIP
  ports:
  - port: 80
    protocol: TCP
    targetPort: 9090
  selector:
    app: prometheus
EOF

部署

kubectl apply -f prometheus.svc.yaml

prometheus-route.yaml文件

cat >prometheus-route.yaml<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: prometheus
  namespace: kube-system
spec:
  entryPoints:
    - web
  routes:
    - match: Host(`prometheus.ygbx.com`)
      kind: Rule
      services:
        - name: prometheus
          port: 80
EOF

部署

kubectl apply -f prometheus-route.yaml

rbac-setup.yam文件

cat >rbac-setup.yaml<<EOF
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: prometheus
rules:
- apiGroups: [""]
  resources:
  - nodes
  - nodes/proxy
  - services
  - endpoints
  - pods
  verbs: ["get", "list", "watch"]
- apiGroups:
  - extensions
  resources:
  - ingresses
  verbs: ["get", "list", "watch"]
- nonResourceURLs: ["/metrics"]
  verbs: ["get"]
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: prometheus
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: prometheus
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: prometheus
subjects:
- kind: ServiceAccount
  name: prometheus
  namespace: kube-system

EOF

部署

kubectl apply -f prometheus-route.yaml

配置好dns记录

 访问http://prometheus.ygbx.com/随便选取一个metric，点击execute，查看是否能正常获取结果输出。点击status—target，可以看到metrics的数据来源，即各exporter，点击相应exporter上的链接可查看这个exporter提供的metrics明细。

 为了更好的展示图形效果，需要部署grafana，因此前已经部署有grafana，这里不再部署，贴一个all-in-one.yaml部署文件。

cat >grafana-all-in-one.yaml<<EOF
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: grafana-core
  namespace: kube-system
  labels:
    app: grafana
    component: core
spec:
  replicas: 1
  template:
    metadata:
      labels:
        app: grafana
        component: core
    spec:
      containers:
      - image: grafana/grafana:4.2.0
        name: grafana-core
        imagePullPolicy: IfNotPresent
        # env:
        resources:
          # keep request = limit to keep this container in guaranteed class
          limits:
            cpu: 100m
            memory: 100Mi
          requests:
            cpu: 100m
            memory: 100Mi
        env:
          # The following env variables set up basic auth twith the default admin user and admin password.
          - name: GF_AUTH_BASIC_ENABLED
            value: "true"
          - name: GF_AUTH_ANONYMOUS_ENABLED
            value: "false"
          # - name: GF_AUTH_ANONYMOUS_ORG_ROLE
          #   value: Admin
          # does not really work, because of template variables in exported dashboards:
          # - name: GF_DASHBOARDS_JSON_ENABLED
          #   value: "true"
        readinessProbe:
          httpGet:
            path: /login
            port: 3000
          # initialDelaySeconds: 30
          # timeoutSeconds: 1
        volumeMounts:
        - name: grafana-persistent-storage
          mountPath: /var
      volumes:
      - name: grafana-persistent-storage
        emptyDir: {}

---
apiVersion: v1
kind: Service
metadata:
  name: grafana
  namespace: kube-system
  labels:
    app: grafana
    component: core
spec:
  type: NodePort
  ports:
    - port: 3000
  selector:
    app: grafana
    component: core


EOF

创建ingress访问

cat >grafana-route.ymal<<EOF
apiVersion: traefik.containo.us/v1alpha1
kind: IngressRoute
metadata:
  name: grafana
  namespace: kube-system
spec:
  entryPoints:
    - web
  routes:
    - match: Host(`grafana.ygbx.com`)
      kind: Rule
      services:
        - name: grafana
          port: 3000
EOF

配置好dns记录

访问http://grafana.ygbx.com/

默认管理账号密码为admin admin

 选择资源类型，填入prometheus的服务地址及端口号，点击保存

 导入展示模板：
点击dashboard，点击import dashboard，在弹出框内填写数字315，会自动加载官方提供的315号模板，然后选择数据源为刚添加的数据源，模板就创建好了.

 

 

 

 基本部署到这里就结束了

10容器日志收集方案

把log-agent打包至业务镜像
日志落地至物理节点
每个物理节点启动日志容器

本例中在每个node 节点部署一个pod 收集日志

安装日志组件

设置serviceAccount

kubectl create serviceaccount admin -n kube-system

配置权限

mkdir es

cat >es-rbac.yaml<<EOF 
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: es-rbac
subjects:
  - kind: ServiceAccount
    name: admin 
    namespace: kube-system
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
EOF

创建权限

kubectl apply -f  es-rbac.yaml

安装Elasticsearch

docker pull registry.cn-hangzhou.aliyuncs.com/cqz/elasticsearch:5.5.1

wget https://acs-logging.oss-cn-hangzhou.aliyuncs.com/elasticsearch.yml