关闭selinux
永久关闭命令：
sed -i 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config
grep SELINUX=disabled /etc/selinux/config
临时关闭命令：
setenforce 0
getenforce
关闭防火墙
systemctl stop firewalld
systemctl disable firewalld
临时修改命令：
ulimit -n 65535
永久修改命令：
echo '* - nofile 65535 ' >>/etc/security/limits.conf
查看配置文件是否修改正确
tail -1 /etc/security/limits.conf
centos7.4以上
DNS服务安装部署
- 创建主机域host.com
- 创建业务域od.com
- 主辅同步(10.4.7.11主、10.4.7.12辅)
- 客户端配置指向自建DNS
下载安装bind9
yum install bind -y1.修改配置文件
vim /etc/named.conf
2.
vim /etc/named.rfc1912.zones
在文章最后添加
zone "host.com" IN {
type master;
file "host.com.zone";
allow-update {10.4.7.11; };
};
zone "od.com" IN {
type master;
file "od.com.zone";
allow-update {10.4.7.11; };
};
3.
vim /var/named/host.com.zone
$TTL 600 ; 10 minutes
@ IN SOA dns.host.com. dnsadmin.host.com. (
2020051601 ; serial
10800 ; refresh (3 hours)
900 ; retry (15 minutes)
604800 ; expire (1 week)
86400 ; minimum (1 day)
)
NS dns.host.com.
$TTL 60 ; 1 minute
dns A 10.4.7.11
HDSS7-11 A 10.4.7.11
HDSS7-12 A 10.4.7.12
HDSS7-21 A 10.4.7.21
HDSS7-22 A 10.4.7.22
HDSS7-200 A 10.4.7.200
4.
vim /var/named/od.com.zone
$ORIGIN od.com.
$TTL 600 ; 10 minutes
@ IN SOA dns.od.com. dnsadmin.od.com. (
2020051601 ; serial
10800 ; refresh (3 hours)
900 ; retry (15 minutes)
604800 ; expire (1 week)
86400 ; minimum (1 day)
)
NS dns.od.com.
$TTL 60 ; 1 minute
dns A 10.0.0.115.
systemctl restart named6.
配置个主机网卡指向自建DNS
vim /etc/sysctl.conf
net.ipv4.ip_forward = 17.
systemctl restart network9.
修改/etc/resolv.conf
添加以下代码,可以解析省略host.com的域名
search host.com准备签发证书环境
运维主机HDSS7-200.host.com上：
安装CFSSL
- 证书签发工具CFSSL: R1.2
wget https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -O /usr/bin/cfssl
wget https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -O /usr/bin/cfssl-json
wget https://pkg.cfssl.org/R1.2/cfssl-certinfo_linux-amd64 -O /usr/bin/cfssl-certinfo
chmod +x /usr/bin/cfssl*
vim /opt/certs/ca-csr.json
创建生成CA证书签名请求（csr）的JSON配置文件
复制/opt/certs/ca-csr.json
{
"CN": "kubernetes-ca",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
],
"ca": {
"expiry": "175200h"
}
}
CN: Common Name，浏览器使用该字段验证网站是否合法，一般写的是域名。非常重要。浏览器使用该字段验证网站是否合法
C: Country， 国家
ST: State，州，省
L: Locality，地区，城市
O: Organization Name，组织名称，公司名称
OU: Organization Unit Name，组织单位名称，公司部门
生成CA证书和私钥
复制/opt/certs
[root@hdss7-200 certs]# cfssl gencert -initca ca-csr.json | cfssl-json -bare ca -
2019/01/18 09:31:19 [INFO] generating a new CA key and certificate from CSR
2019/01/18 09:31:19 [INFO] generate received request
2019/01/18 09:31:19 [INFO] received CSR
2019/01/18 09:31:19 [INFO] generating key: rsa-2048
2019/01/18 09:31:19 [INFO] encoded CSR
2019/01/18 09:31:19 [INFO] signed certificate with serial number 345276964513449660162382535043012874724976422200
生成ca.pem、ca.csr、ca-key.pem(CA私钥,需妥善保管)
复制/opt/certs
[root@hdss7-200 certs]# ls -l
-rw-r--r-- 1 root root 836 Jan 16 11:04 ca-config.json
-rw-r--r-- 1 root root 332 Jan 16 11:10 ca-csr.json
-rw------- 1 root root 1675 Jan 16 11:17 ca-key.pem
-rw-r--r-- 1 root root 1001 Jan 16 11:17 ca.csr
-rw-r--r-- 1 root root 1354 Jan 16 11:17 ca.pem部署docker
HDSS7-200.host.com,HDSS7-21.host.com,HDSS7-22.host.com上
安装yum源
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo安装epel源
mv /etc/yum.repos.d/epel.repo /etc/yum.repos.d/epel.repo.backup
mv /etc/yum.repos.d/epel-testing.repo /etc/yum.repos.d/epel-testing.repo.backupwget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo安装docker-ce
yum install -y yum-utils device-mapper-persistent-data lvm2yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repoyum makecache fastyum -y install docker-ceservice docker start
systemctl enable docker简单安装
curl -fsSL https://get.docker.com|bash -s docker --mirror Aliyun配置docker配置文件
复制/etc/docker/daemon.json
注意：这里bip复制/etc/docker/daemon.json
vi /etc/docker/daemon.json
{
"graph": "/data/docker",
"storage-driver": "overlay2",
"insecure-registries": ["registry.access.redhat.com","quay.io","harbor.od.com"],
"bip": "172.7.21.1/24",
"exec-opts": ["native.cgroupdriver=systemd"],
"live-restore": true
}要根据宿主机ip变化每台docker都要修改
安装harbor
下载harbor私有仓库安装包
下载地址
https://storage.googleapis.com/harbor-releases/release-1.7.0/harbor-offline-installer-v1.7.1.tgz
传到/opt
解压
mv harbor harbor-v1.8.3
创建软连接便于后续升级管理
ln -s /opt/harbor-v1.8.3 /opt/harbor
vim harbor.yml
#修改为自家DNS的业务域
hostname: harbor.od.com
#修改映射端口
port: 180
#数据目录
data_volume: /data/harbor
#logs
location: /data/harbor/logs
mkdir /data/harbor/logs -p安装docker-compose
docker-compose
https://github.com/docker/compose/
centos7.6安装epel源后
yum install docker-compose -y安装harbor
./install.sh安装nginx反代harbor
yum install nginx -y
修改nginx配置
vi /etc/nginx/conf.d/harbor.od.com.conf
server {
listen 80;
server_name harbor.od.com;
client_max_body_size 1000m;
location / {
proxy_pass http://127.0.0.1:180;
}
}修改dns配置hdss7-11主机
$ORIGIN od.com.
$TTL 600 ; 10 minutes
@ IN SOA dns.od.com. dnsadmin.od.com. (
2020051602 ; serial
10800 ; refresh (3 hours)
900 ; retry (15 minutes)
604800 ; expire (1 week)
86400 ; minimum (1 day)
)
NS dns.od.com.
$TTL 60 ; 1 minute
dns A 10.4.7.11
harbor A 10.4.7.200重启DNS服务
systemctl restart named
dig -t A harbor.od.com +short
检查是否解析成功在hdss7-200上curl harbor.od.com是否成功
curl -v harbor.od.com启动nginx
systemclt start nginx自己笔记本配置自建DNS访问harbor.od.com查看是否成功
登录harbor
admin/Harbor123456 
下载一个nginx的1.7.9的镜像
docker pull nginx:1.7.9上传私有仓库镜像测试
docker tag nginx:1.7.9 harbor.od.com/public/nginx:v1.7.9
docker login harbor.od.com
docker push harbor.od.com/public/nginx:v1.7.9
部署Master节点服务
安装etcd
部署安装服务图解
集群规划
| 主机名 | 角色 | ip |
|---|---|---|
| HDSS7-12.host.com | etcd lead | 10.4.7.12 |
| HDSS7-21.host.com | etcd follow | 10.4.7.21 |
| HDSS7-22.host.com | etcd follow | 10.4.7.22 |
注意：这里部署文档以HDSS7-12.host.com主机为例，另外两台主机安装部署方法类似
创建生成CA证书的JSON配置文件
/opt/certs/ca-config.json
{
"signing": {
"default": {
"expiry": "175200h"
},
"profiles": {
"server": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth"
]
},
"client": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"client auth"
]
},
"peer": {
"expiry": "175200h",
"usages": [
"signing",
"key encipherment",
"server auth",
"client auth"
]
}
}
}
}创建生成证书签名请求（csr）的JSON配置文件
运维主机HDSS7-200.host.com上：
复制/opt/certs/etcd-peer-csr.json
{
"CN": "etcd-peer",
"hosts": [
"10.4.7.11",
"10.4.7.12",
"10.4.7.21",
"10.4.7.22"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}签发etcd的证书cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer etcd-peer-csr.json | cfssl-json -bare etcd-peer
2019/01/18 09:35:09 [INFO] generate received request
2019/01/18 09:35:09 [INFO] received CSR
2019/01/18 09:35:09 [INFO] generating key: rsa-2048
2019/01/18 09:35:09 [INFO] encoded CSR
2019/01/18 09:35:10 [INFO] signed certificate with serial number 324191491384928915605254764031096067872154649010
2019/01/18 09:35:10 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
检查生成的证书、私钥
复制/opt/certs
[root@hdss7-200 certs]# ls -l|grep etcd
-rw-r--r-- 1 root root 387 Jan 18 12:32 etcd-peer-csr.json
-rw------- 1 root root 1679 Jan 18 12:32 etcd-peer-key.pem
-rw-r--r-- 1 root root 1074 Jan 18 12:32 etcd-peer.csr
-rw-r--r-- 1 root root 1432 Jan 18 12:32 etcd-peer.pem创建etcd用户
HDSS7-12.host.com\HDSS7-21.host.com\HDSS7-22.host.com上：
复制
[root@hdss7-12 ~]# useradd -s /sbin/nologin -M etcd下载软件，解压，做软连接
etcd下载地址HDSS7-12.host.com上：
复制/opt/src
[root@hdss7-12 src]# ls -l
total 9604
-rw-r--r-- 1 root root 9831476 Jan 18 10:45 etcd-v3.1.18-linux-amd64.tar.gz
[root@hdss7-12 opt]# tar xf etcd-v3.1.20-linux-amd64.tar.gz
[root@hdss7-12 opt]# mv etcd-v3.1.20-linux-amd64 etcd-v3.1.20
[root@hdss7-12 opt]# mv etcd-v3.1.20-linux-amd64 etcd-v3.1.20
ln -s etcd-v3.1.20 etcd
创建目录，拷贝证书、私钥
HDSS7-12.host.com上：
复制
[root@hdss7-12 src]# mkdir -p /data/etcd /data/logs/etcd-server /opt/etcd/certs /data/etcd/etcd-server
[root@hdss7-12 src]# mkdir -p /opt/etcd/certs
将运维主机上生成的ca.pem、etcd-peer-key.pem、etcd-peer.pem拷贝到/opt/etcd/certs目录中，注意私钥文件权限600
复制/opt/etcd/certs
[root@hdss7-12 certs]# chmod 600 etcd-peer-key.pem
[root@hdss7-12 certs]# chown -R etcd.etcd /opt/etcd/certs/
[root@hdss7-12 certs]# ls -l
total 12
-rw-r--r-- 1 etcd etcd 1354 Jan 18 14:45 ca.pem
-rw------- 1 etcd etcd 1679 Jan 18 17:00 etcd-peer-key.pem
-rw-r--r-- 1 etcd etcd 1444 Jan 18 17:02 etcd-peer.pem
创建etcd服务启动脚本
HDSS7-12.host.com上：
复制/opt/etcd/etcd-server-startup.sh
#!/bin/sh
./etcd --name etcd-server-7-12 \
--data-dir /data/etcd/etcd-server \
--listen-peer-urls https://10.4.7.12:2380 \
--listen-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
--quota-backend-bytes 8000000000 \
--initial-advertise-peer-urls https://10.4.7.12:2380 \
--advertise-client-urls https://10.4.7.12:2379,http://127.0.0.1:2379 \
--initial-cluster etcd-server-7-12=https://10.4.7.12:2380,etcd-server-7-21=https://10.4.7.21:2380,etcd-server-7-22=https://10.4.7.22:2380 \
--ca-file ./certs/ca.pem \
--cert-file ./certs/etcd-peer.pem \
--key-file ./certs/etcd-peer-key.pem \
--client-cert-auth \
--trusted-ca-file ./certs/ca.pem \
--peer-ca-file ./certs/ca.pem \
--peer-cert-file ./certs/etcd-peer.pem \
--peer-key-file ./certs/etcd-peer-key.pem \
--peer-client-cert-auth \
--peer-trusted-ca-file ./certs/ca.pem \
--log-output stdout
注意：etcd集群各主机的启动脚本略有不同，部署其他节点时注意修改。
调整权限和目录
HDSS7-12.host.com上：
[root@hdss7-12 certs]# chmod +x /opt/etcd/etcd-server-startup.sh
[root@hdss7-12 certs]# chown -R etcd.etcd /data/etcd /data/logs/etcd-server/ /opt/etcd
安装supervisor软件使etcd后台运行
HDSS7-12.host.com上：
[root@hdss7-12 certs]# yum install supervisor -y
[root@hdss7-12 certs]# systemctl start supervisord
[root@hdss7-12 certs]# systemctl enable supervisord
创建etcd-server的启动配置
HDSS7-12.host.com上：
复制/etc/supervisord.d/etcd-server.ini
[program:etcd-server-7-12]
command=/opt/etcd/etcd-server-startup.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/etcd ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=etcd ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/etcd-server/etcd.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/etcd-server/etcd.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
注意：etcd集群各主机启动配置略有不同，配置其他节点时注意修改。
启动etcd服务并检查
HDSS7-12.host.com上：
复制
[root@hdss7-12 certs]# supervisorctl update
etcd-server-7-12: started
[root@hdss7-12 certs]# supervisorctl status
etcd-server-7-12 RUNNING pid 6692, uptime 0:00:05
安装部署启动检查所有集群规划主机上的etcd服务
略
检查集群状态
3台均启动后，检查集群状态
复制
[root@hdss7-12 ~]# /opt/etcd/etcdctl cluster-health
member 988139385f78284 is healthy: got healthy result from http://127.0.0.1:2379
member 5a0ef2a004fc4349 is healthy: got healthy result from http://127.0.0.1:2379
member f4a0cb0a765574a8 is healthy: got healthy result from http://127.0.0.1:2379
cluster is healthy
[root@hdss7-12 ~]# /opt/etcd/etcdctl member list
988139385f78284: name=etcd-server-7-22 peerURLs=https://10.4.7.22:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.22:2379 isLeader=false
5a0ef2a004fc4349: name=etcd-server-7-21 peerURLs=https://10.4.7.21:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.21:2379 isLeader=false
f4a0cb0a765574a8: name=etcd-server-7-12 peerURLs=https://10.4.7.12:2380 clientURLs=http://127.0.0.1:2379,https://10.4.7.12:2379 isLeader=true部署kube-apiserver集群
集群规划
| 主机名 | 角色 | ip |
|---|---|---|
| HDSS7-21.host.com | kube-apiserver | 10.4.7.21 |
| HDSS7-22.host.com | kube-apiserver | 10.4.7.22 |
| HDSS7-11.host.com | 4层负载均衡 | 10.4.7.11 |
| HDSS7-12.host.com | 4层负载均衡 | 10.4.7.12 |
注意：这里10.4.7.11和10.4.7.12使用nginx做4层负载均衡器，用keepalived跑一个vip：10.4.7.10，代理两个kube-apiserver，实现高可用
这里部署文档以HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似
下载软件，解压，做软连接
HDSS7-21.host.com上：
kubernetes下载地址
复制/opt/src
[root@hdss7-21 src]# ls -l|grep kubernetes
-rw-r--r-- 1 root root 417761204 Jan 17 16:46 kubernetes-server-linux-amd64.tar.gz
[root@hdss7-21 src]# tar xf kubernetes-server-linux-amd64.tar.gz -C /opt
[root@hdss7-21 src]# mv /opt/kubernetes /opt/kubernetes-v1.15.2
[root@hdss7-21 src]# ln -s /opt/kubernetes-v1.15.2 /opt/kubernetes
[root@hdss7-21 src]# mkdir /opt/kubernetes/server/bin/{cert,conf}
[root@hdss7-21 src]# ls -l /opt|grep kubernetes
lrwxrwxrwx 1 root root 31 Jan 18 10:49 kubernetes -> kubernetes-v1.13.2-linux-amd64/
drwxr-xr-x 4 root root 50 Jan 17 17:40 kubernetes-v1.13.2-linux-amd64签发client证书
运维主机HDSS7-200.host.com上：
创建生成证书签名请求（csr）的JSON配置文件
复制/opt/certs/client-csr.json
{
"CN": "k8s-node",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}
复制生成client证书和私钥
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client-csr.json | cfssl-json -bare client
2019/01/18 14:02:50 [INFO] generate received request
2019/01/18 14:02:50 [INFO] received CSR
2019/01/18 14:02:50 [INFO] generating key: rsa-2048
2019/01/18 14:02:51 [INFO] encoded CSR
2019/01/18 14:02:51 [INFO] signed certificate with serial number 423108651040279300242366884100637974155370861448
2019/01/18 14:02:51 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
复制检查生成的证书、私钥
[root@hdss7-200 certs]# ls -l|grep client
-rw------- 1 root root 1679 Jan 21 11:13 client-key.pem
-rw-r--r-- 1 root root 989 Jan 21 11:13 client.csr
-rw-r--r-- 1 root root 1367 Jan 21 11:13 client.pem
签发kube-apiserver证书
运维主机HDSS7-200.host.com上：
创建生成证书签名请求（csr）的JSON配置文件
复制/opt/certs/apiserver-csr.json
{
"CN": "apiserver",
"hosts": [
"127.0.0.1",
"192.168.0.1",
"kubernetes.default",
"kubernetes.default.svc",
"kubernetes.default.svc.cluster",
"kubernetes.default.svc.cluster.local",
"10.4.7.10",
"10.4.7.21",
"10.4.7.22",
"10.4.7.23"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}
复制生成kube-apiserver证书和私钥
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server apiserver-csr.json | cfssl-json -bare apiserver
2019/01/18 14:05:44 [INFO] generate received request
2019/01/18 14:05:44 [INFO] received CSR
2019/01/18 14:05:44 [INFO] generating key: rsa-2048
2019/01/18 14:05:46 [INFO] encoded CSR
2019/01/18 14:05:46 [INFO] signed certificate with serial number 633406650960616624590510576685608580490218676227
2019/01/18 14:05:46 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
复制检查生成的证书、私钥
[root@hdss7-200 certs]# ls -l|grep apiserver
total 72
-rw-r--r-- 1 root root 406 Jan 21 14:10 apiserver-csr.json
-rw------- 1 root root 1675 Jan 21 14:11 apiserver-key.pem
-rw-r--r-- 1 root root 1082 Jan 21 14:11 apiserver.csr
-rw-r--r-- 1 root root 1599 Jan 21 14:11 apiserver.pemHDSS7-21.host.com上：
拷贝证书至各运算节点，并创建配置
拷贝证书、私钥，注意私钥文件属性600
复制/opt/kubernetes/server/bin/cert
cd cert/
ls
scp -rp hdss7-200:/opt/certs/ca.pem .
scp -rp hdss7-200:/opt/certs/ca-key.pem .
scp -rp hdss7-200:/opt/certs/client.pem .
scp -rp hdss7-200:/opt/certs/client-key.pem .
scp -rp hdss7-200:/opt/certs/apiserver.pem .
scp -rp hdss7-200:/opt/certs/apiserver-key.pem .
[root@hdss7-21 cert]# ls -l /opt/kubernetes/server/bin/cert
total 40
-rw------- 1 root root 1676 Jan 21 16:39 apiserver-key.pem
-rw-r--r-- 1 root root 1599 Jan 21 16:36 apiserver.pem
-rw------- 1 root root 1675 Jan 21 13:55 ca-key.pem
-rw-r--r-- 1 root root 1354 Jan 21 13:50 ca.pem
-rw------- 1 root root 1679 Jan 21 13:53 client-key.pem
-rw-r--r-- 1 root root 1368 Jan 21 13:53 client.pem创建配置
复制/opt/kubernetes/server/bin/conf/audit.yaml
apiVersion: audit.k8s.io/v1beta1 # This is required.
kind: Policy
# Don't generate audit events for all requests in RequestReceived stage.
omitStages:
- "RequestReceived"
rules:
# Log pod changes at RequestResponse level
- level: RequestResponse
resources:
- group: ""
# Resource "pods" doesn't match requests to any subresource of pods,
# which is consistent with the RBAC policy.
resources: ["pods"]
# Log "pods/log", "pods/status" at Metadata level
- level: Metadata
resources:
- group: ""
resources: ["pods/log", "pods/status"]
# Don't log requests to a configmap called "controller-leader"
- level: None
resources:
- group: ""
resources: ["configmaps"]
resourceNames: ["controller-leader"]
# Don't log watch requests by the "system:kube-proxy" on endpoints or services
- level: None
users: ["system:kube-proxy"]
verbs: ["watch"]
resources:
- group: "" # core API group
resources: ["endpoints", "services"]
# Don't log authenticated requests to certain non-resource URL paths.
- level: None
userGroups: ["system:authenticated"]
nonResourceURLs:
- "/api*" # Wildcard matching.
- "/version"
# Log the request body of configmap changes in kube-system.
- level: Request
resources:
- group: "" # core API group
resources: ["configmaps"]
# This rule only applies to resources in the "kube-system" namespace.
# The empty string "" can be used to select non-namespaced resources.
namespaces: ["kube-system"]
# Log configmap and secret changes in all other namespaces at the Metadata level.
- level: Metadata
resources:
- group: "" # core API group
resources: ["secrets", "configmaps"]
# Log all other resources in core and extensions at the Request level.
- level: Request
resources:
- group: "" # core API group
- group: "extensions" # Version of group should NOT be included.
# A catch-all rule to log all other requests at the Metadata level.
- level: Metadata
# Long-running requests like watches that fall under this rule will not
# generate an audit event in RequestReceived.
omitStages:
- "RequestReceived"创建启动脚本
HDSS7-21.host.com上：
复制/opt/kubernetes/server/bin/kube-apiserver.sh
#!/bin/bash
./kube-apiserver \
--apiserver-count 2 \
--audit-log-path /data/logs/kubernetes/kube-apiserver/audit-log \
--audit-policy-file ./conf/audit.yaml \
--authorization-mode RBAC \
--client-ca-file ./cert/ca.pem \
--requestheader-client-ca-file ./cert/ca.pem \
--enable-admission-plugins NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota \
--etcd-cafile ./cert/ca.pem \
--etcd-certfile ./cert/client.pem \
--etcd-keyfile ./cert/client-key.pem \
--etcd-servers https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
--service-account-key-file ./cert/ca-key.pem \
--service-cluster-ip-range 192.168.0.0/16 \
--service-node-port-range 3000-29999 \
--target-ram-mb=1024 \
--kubelet-client-certificate ./cert/client.pem \
--kubelet-client-key ./cert/client-key.pem \
--log-dir /data/logs/kubernetes/kube-apiserver \
--tls-cert-file ./cert/apiserver.pem \
--tls-private-key-file ./cert/apiserver-key.pem \
--v 2调整权限和目录
HDSS7-21.host.com上：
复制/opt/kubernetes/server/bin
[root@hdss7-21 bin]# chmod +x /opt/kubernetes/server/bin/kube-apiserver.sh
[root@hdss7-21 bin]# mkdir -p /data/logs/kubernetes/kube-apiserver创建supervisor配置
HDSS7-21.host.com上：
复制/etc/supervisord.d/kube-apiserver.ini
[program:kube-apiserver-7-21]
command=/opt/kubernetes/server/bin/kube-apiserver.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-apiserver/apiserver.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
启动服务并检查
HDSS7-21.host.com上：
复制
[root@hdss7-21 bin]# supervisorctl update
kube-apiserverr: added process group
[root@hdss7-21 bin]# supervisorctl status
etcd-server-7-21 RUNNING pid 6661, uptime 1 day, 8:41:13
kube-apiserver RUNNING pid 43765, uptime 2:09:41
安装部署启动检查所有集群规划主机上的kube-apiserver
略(注意需要更改supervitor的配置文件的文件名)
配4层反向代理
HDSS7-11.host.com,HDSS7-12.host.com上：
nginx配置
复制/etc/nginx/nginx.conf
stream {
upstream kube-apiserver {
server 10.4.7.21:6443 max_fails=3 fail_timeout=30s;
server 10.4.7.22:6443 max_fails=3 fail_timeout=30s;
}
server {
listen 7443;
proxy_connect_timeout 2s;
proxy_timeout 900s;
proxy_pass kube-apiserver;
}
}keepalived配置
yum install keepalived -y
check_port.sh
复制/etc/keepalived/check_port.sh
#!/bin/bash
#keepalived 监控端口脚本
#使用方法：
#在keepalived的配置文件中
#vrrp_script check_port {#创建一个vrrp_script脚本,检查配置
# script "/etc/keepalived/check_port.sh 6379" #配置监听的端口
# interval 2 #检查脚本的频率,单位（秒）
#}
CHK_PORT=$1
if [ -n "$CHK_PORT" ];then
PORT_PROCESS=`ss -lnt|grep $CHK_PORT|wc -l`
if [ $PORT_PROCESS -eq 0 ];then
echo "Port $CHK_PORT Is Not Used,End."
exit 1
fi
else
echo "Check Port Cant Be Empty!"
fichmod +x /etc/keepalived/check_port.shHDSS7-11.host.com上keepalived主
复制
[root@hdss7-11 ~]# rpm -qa keepalived
keepalived-1.3.5-6.el7.x86_64
复制/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 10.4.7.11
}
vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 251
priority 100
advert_int 1
mcast_src_ip 10.4.7.11
nopreempt
authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
10.4.7.10
}
}HDSS7-12.host.com上keepalived备
复制
[root@hdss7-12 ~]# rpm -qa keepalived keepalived-1.3.5-6.el7.x86_64
复制/etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id 10.4.7.12
}
vrrp_script chk_nginx {
script "/etc/keepalived/check_port.sh 7443"
interval 2
weight -20
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 251
mcast_src_ip 10.4.7.12
priority 90
advert_int 1
authentication {
auth_type PASS
auth_pass 11111111
}
track_script {
chk_nginx
}
virtual_ipaddress {
10.4.7.10
}
}
启动代理并检查
HDSS7-11.host.com,HDSS7-12.host.com上：
-
启动
复制[root@hdss7-11 ~]# systemctl start keepalived [root@hdss7-11 ~]# systemctl enable keepalived [root@hdss7-11 ~]# nginx -s reload [root@hdss7-12 ~]# systemctl start keepalived [root@hdss7-12 ~]# systemctl enable keepalived [root@hdss7-12 ~]# nginx -s reload -
检查
复制[root@hdss7-11 ~]## netstat -luntp|grep 7443 tcp 0 0 0.0.0.0:7443 0.0.0.0:* LISTEN 17970/nginx: master [root@hdss7-12 ~]## netstat -luntp|grep 7443 tcp 0 0 0.0.0.0:7443 0.0.0.0:* LISTEN 17970/nginx: master [root@hdss7-11 ~]# ip add|grep 10.4.9.10 inet 10.9.7.10/32 scope global vir0 [root@hdss7-11 ~]# ip add|grep 10.4.9.10部署controller-manager
集群规划
主机名 角色 ip HDSS7-21.host.com controller-manager 10.4.7.21 HDSS7-22.host.com controller-manager 10.4.7.22 注意：这里部署文档以
HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似创建启动脚本
HDSS7-21.host.com上：复制/opt/kubernetes/server/bin/kube-controller-manager.sh
#!/bin/sh ./kube-controller-manager \ --cluster-cidr 172.7.0.0/16 \ --leader-elect true \ --log-dir /data/logs/kubernetes/kube-controller-manager \ --master http://127.0.0.1:8080 \ --service-account-private-key-file ./cert/ca-key.pem \ --service-cluster-ip-range 192.168.0.0/16 \ --root-ca-file ./cert/ca.pem \ --v 2
调整文件权限，创建目录HDSS7-21.host.com上：复制/opt/kubernetes/server/bin
[root@hdss7-21 bin]# chmod +x /opt/kubernetes/server/bin/kube-controller-manager.sh [root@hdss7-21 bin]# mkdir -p /data/logs/kubernetes/kube-controller-manager
创建supervisor配置HDSS7-21.host.com上：复制/etc/supervisord.d/kube-conntroller-manager.ini
[program:kube-controller-manager-7-21] command=/opt/kubernetes/server/bin/kube-controller-manager.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=22 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=false ; redirect proc stderr to stdout (default false) stdout_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) stderr_logfile=/data/logs/kubernetes/kube-controller-manager/controll.stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=4 ; # of stderr logfile backups (default 10) stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stderr_events_enabled=false ; emit events on stderr writes (default false)
启动服务并检查HDSS7-21.host.com上：复制
[root@hdss7-21 bin]# supervisorctl update kube-controller-manager: added process group [root@hdss7-21 bin]# supervisorctl status etcd-server-7-21 RUNNING pid 6661, uptime 1 day, 8:41:13 kube-apiserver RUNNING pid 43765, uptime 2:09:41 kube-controller-manager RUNNING pid 44230, uptime 2:05:01安装部署启动检查所有集群规划主机上的kube-controller-manager服务
略
署kube-scheduler
集群规划
主机名 角色 ip HDSS7-21.host.com kube-scheduler 10.4.7.21 HDSS7-22.host.com kube-scheduler 10.4.7.22 注意：这里部署文档以
HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似创建启动脚本
HDSS7-21.host.com上：复制/opt/kubernetes/server/bin/kube-scheduler.sh
#!/bin/sh ./kube-scheduler \ --leader-elect \ --log-dir /data/logs/kubernetes/kube-scheduler \ --master http://127.0.0.1:8080 \ --v 2
调整文件权限，创建目录HDSS7-21.host.com上：复制/opt/kubernetes/server/bin
[root@hdss7-21 bin]# chmod +x /opt/kubernetes/server/bin/kube-scheduler.sh [root@hdss7-21 bin]# mkdir -p /data/logs/kubernetes/kube-scheduler
创建supervisor配置HDSS7-21.host.com上：复制/etc/supervisord.d/kube-scheduler.ini
[program:kube-scheduler-7-21] command=/opt/kubernetes/server/bin/kube-scheduler.sh ; the program (relative uses PATH, can take args) numprocs=1 ; number of processes copies to start (def 1) directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd) autostart=true ; start at supervisord start (default: true) autorestart=true ; retstart at unexpected quit (default: true) startsecs=22 ; number of secs prog must stay running (def. 1) startretries=3 ; max # of serial start failures (default 3) exitcodes=0,2 ; 'expected' exit codes for process (default 0,2) stopsignal=QUIT ; signal used to kill process (default TERM) stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10) user=root ; setuid to this UNIX account to run the program redirect_stderr=false ; redirect proc stderr to stdout (default false) stdout_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stdout.log ; stdout log path, NONE for none; default AUTO stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stdout_logfile_backups=4 ; # of stdout logfile backups (default 10) stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stdout_events_enabled=false ; emit events on stdout writes (default false) stderr_logfile=/data/logs/kubernetes/kube-scheduler/scheduler.stderr.log ; stderr log path, NONE for none; default AUTO stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB) stderr_logfile_backups=4 ; # of stderr logfile backups (default 10) stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0) stderr_events_enabled=false ; emit events on stderr writes (default false)
启动服务并检查HDSS7-21.host.com上：复制
[root@hdss7-21 bin]# supervisorctl update kube-scheduler: added process group [root@hdss7-21 bin]# supervisorctl status etcd-server-7-21 RUNNING pid 6661, uptime 1 day, 8:41:13 kube-apiserver RUNNING pid 43765, uptime 2:09:41 kube-controller-manager RUNNING pid 44230, uptime 2:05:01 kube-scheduler RUNNING pid 44779, uptime 2:02:27
安装部署启动检查所有集群规划主机上的kube-scheduler服务略
软连接kubectl到/usr/bin
ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl
查看集群状态:
kubectl get cs部署Node节点服务
部署kubelet
集群规划
| 主机名 | 角色 | ip |
|---|---|---|
| HDSS7-21.host.com | kubelet | 10.4.7.21 |
| HDSS7-22.host.com | kubelet | 10.4.7.22 |
注意：这里部署文档以HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似
签发kubelet证书
运维主机HDSS7-200.host.com上：
创建生成证书签名请求（csr）的JSON配置文件
复制kubelet-csr.json
{
"CN": "k8s-kubelet",
"hosts": [
"127.0.0.1",
"10.4.7.10",
"10.4.7.21",
"10.4.7.22",
"10.4.7.23",
"10.4.7.24",
"10.4.7.25",
"10.4.7.26",
"10.4.7.27",
"10.4.7.28"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}
复制/opt/certs生成kubelet证书和私钥
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server kubelet-csr.json | cfssl-json -bare kubelet
2019/01/18 17:51:16 [INFO] generate received request
2019/01/18 17:51:16 [INFO] received CSR
2019/01/18 17:51:16 [INFO] generating key: rsa-2048
2019/01/18 17:51:17 [INFO] encoded CSR
2019/01/18 17:51:17 [INFO] signed certificate with serial number 48870268157415133698067712395152321546974943470
2019/01/18 17:51:17 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").
复制/opt/certs检查生成的证书、私钥
[root@hdss7-200 certs]# ls -l|grep kubelet
total 88
-rw-r--r-- 1 root root 415 Jan 22 16:58 kubelet-csr.json
-rw------- 1 root root 1679 Jan 22 17:00 kubelet-key.pem
-rw-r--r-- 1 root root 1086 Jan 22 17:00 kubelet.csr
-rw-r--r-- 1 root root 1456 Jan 22 17:00 kubelet.pem
HDSS7-21.host.com上：拷贝证书至各运算节点，并创建配置
拷贝证书、私钥，注意私钥文件属性600
复制/opt/kubernetes/server/bin/cert
[root@hdss7-21 cert]# ls -l /opt/kubernetes/server/bin/cert
total 40
-rw------- 1 root root 1676 Jan 21 16:39 apiserver-key.pem
-rw-r--r-- 1 root root 1599 Jan 21 16:36 apiserver.pem
-rw------- 1 root root 1675 Jan 21 13:55 ca-key.pem
-rw-r--r-- 1 root root 1354 Jan 21 13:50 ca.pem
-rw------- 1 root root 1679 Jan 21 13:53 client-key.pem
-rw-r--r-- 1 root root 1368 Jan 21 13:53 client.pem
-rw------- 1 root root 1679 Jan 22 17:00 kubelet-key.pem
-rw-r--r-- 1 root root 1456 Jan 22 17:00 kubelet.pemHDSS7-21.host.com上：创建配置
给kubectl创建软连接
复制/opt/kubernetes/server/bin
[root@hdss7-21 bin]# ln -s /opt/kubernetes/server/bin/kubectl /usr/bin/kubectl
[root@hdss7-21 bin]# which kubectl
/usr/bin/kubectl
注意：在conf目录下set-cluster
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
--embed-certs=true \
--server=https://10.4.7.10:7443 \
--kubeconfig=kubelet.kubeconfig
Cluster "myk8s" set.
set-credentials
注意：在conf目录下
复制/opt/kubernetes/bin/server/conf
[root@hdss7-21 conf]# kubectl config set-credentials k8s-node --client-certificate=/opt/kubernetes/server/bin/cert/client.pem --client-key=/opt/kubernetes/server/bin/cert/client-key.pem --embed-certs=true --kubeconfig=kubelet.kubeconfig
User "k8s-node" set.
set-context
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=k8s-node \
--kubeconfig=kubelet.kubeconfig
Context "myk8s-context" created.
use-context
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config use-context myk8s-context --kubeconfig=kubelet.kubeconfig
Switched to context "myk8s-context".
k8s-node.yaml
- 创建资源配置文件
复制/opt/kubernetes/server/bin/conf/k8s-node.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: k8s-node
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:node
subjects:
- apiGroup: rbac.authorization.k8s.io
kind: User
name: k8s-node
- 应用资源配置文件
复制/opt/kubernetes/server/conf
[root@hdss7-21 conf]# kubectl create -f k8s-node.yaml
clusterrolebinding.rbac.authorization.k8s.io/k8s-node created
- 检查
复制/opt/kubernetes/server/conf
[root@hdss7-21 conf]# kubectl get clusterrolebinding k8s-node
NAME AGE
k8s-node 3m
运维主机HDSS7-200.host.com上：准备infra_pod基础镜像
下载
复制
[root@hdss7-200 ~]# docker pull kubernetes/pause
Using default tag: latest
latest: Pulling from kubernetes/pause
4f4fb700ef54: Download complete
b9c8ec465f6b: Extracting 65.54kB/71.32kB
4f4fb700ef54: Pull complete
b9c8ec465f6b: Pull complete
Digest: sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105
Status: Downloaded newer image for kubernetes/pause:latest
docker.io/kubernetes/pause:latest
配置主机登录私有仓库提交至私有仓库（harbor）中
复制/root/.docker/config.json
{
"auths": {
"harbor.od.com": {
"auth": "YWRtaW46SGFyYm9yMTIzNDU="
}
}
}
注意：也可以在各运算节点使用docker login harbor.od.com，输入用户名，密码这里代表：用户名admin，密码Harbor12345
[root@hdss7-200 ~]# echo YWRtaW46SGFyYm9yMTIzNDU=|base64 -d
admin:Harbor12345
- 给镜像打tag
复制
[root@hdss7-200 ~]# docker tag f9d5de079539 harbor.od.com/public/pause:latest
- push到harbor
复制
docker push harbor.od.com/public/pause:latest
The push refers to repository [harbor.od.com/public/pause]
5f70bf18a086: Mounted from public/nginx
e16a89738269: Pushed
latest: digest: sha256:b31bfb4d0213f254d361e0079deaaebefa4f82ba7aa76ef82e90b4935ad5b105 size: 938
HDSS7-21.host.com上：创建kubelet启动脚本
复制/opt/kubernetes/server/bin/kubelet.sh
#!/bin/sh
./kubelet \
--anonymous-auth=false \
--cgroup-driver systemd \
--cluster-dns 192.168.0.2 \
--cluster-domain cluster.local \
--runtime-cgroups=/systemd/system.slice --kubelet-cgroups=/systemd/system.slice \
--fail-swap-on="false" \
--client-ca-file ./cert/ca.pem \
--tls-cert-file ./cert/kubelet.pem \
--tls-private-key-file ./cert/kubelet-key.pem \
--hostname-override 10.4.7.21 \
--image-gc-high-threshold 20 \
--image-gc-low-threshold 10 \
--kubeconfig ./conf/kubelet.kubeconfig \
--log-dir /data/logs/kubernetes/kube-kubelet \
--pod-infra-container-image harbor.od.com/public/pause:latest \
--root-dir /data/kubelet
注意：kubelet集群各主机的启动脚本略有不同，部署其他节点时注意修改。
检查配置，权限，创建日志目录
HDSS7-21.host.com上：
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# ls -l|grep kubelet.kubeconfig
-rw------- 1 root root 6471 Jan 22 17:33 kubelet.kubeconfig
[root@hdss7-21 conf]# chmod +x /opt/kubernetes/server/bin/kubelet.sh
[root@hdss7-21 conf]# mkdir -p /data/logs/kubernetes/kube-kubelet /data/kubelet
创建supervisor配置
HDSS7-21.host.com上：
复制/etc/supervisord.d/kube-kubelet.ini
[program:kube-kubelet]
command=/opt/kubernetes/server/bin/kubelet.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-kubelet/kubelet.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
启动服务并检查
HDSS7-21.host.com上：
复制
[root@hdss7-21 bin]# supervisorctl update
kube-kubelet: added process group
[root@hdss7-21 bin]# supervisorctl status
etcd-server-7-21 RUNNING pid 9507, uptime 22:44:48
kube-apiserver RUNNING pid 9770, uptime 21:10:49
kube-controller-manager RUNNING pid 10048, uptime 18:22:10
kube-kubelet STARTING
kube-scheduler RUNNING pid 10041, uptime 18:22:13
检查运算节点
HDSS7-21.host.com上：
复制
[root@hdss7-21 bin]# kubectl get node
NAME STATUS ROLES AGE VERSION
10.4.7.21 Ready <none> 3m v1.15.2
非常重要！
安装部署启动检查所有集群规划主机上的kubelet服务
略
部署kube-proxy
集群规划
| 主机名 | 角色 | ip |
|---|---|---|
| HDSS7-21.host.com | kube-proxy | 10.4.7.21 |
| HDSS7-22.host.com | kube-proxy | 10.4.7.22 |
注意：这里部署文档以HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似
签发kube-proxy证书
运维主机HDSS7-200.host.com上：
创建生成证书签名请求（csr）的JSON配置文件
复制/opt/certs/kube-proxy-csr.json
{
"CN": "system:kube-proxy",
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}设置集群参数和客户端认证参数时 –embed-certs 都为 true，这会将 certificate-authority、client-certificate 和client-key 指向的证书文件内容写入到生成的 kube-proxy.kubeconfig 文件中；
kube-proxy.pem 证书中 CN 为 system:kube-proxy，kube-apiserver 预定义的 RoleBinding cluster-admin 将User system:kube-proxy 与 Role system:node-proxier 绑定，该 Role 授予了调用 kube-apiserver proxy 相关 API 的权限
生成kube-proxy证书和私钥
复制/opt/certs
[root@hdss7-200 certs]# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client kube-proxy-csr.json | cfssl-json -bare kube-proxy-client
2019/01/18 18:14:23 [INFO] generate received request
2019/01/18 18:14:23 [INFO] received CSR
2019/01/18 18:14:23 [INFO] generating key: rsa-2048
2019/01/18 18:14:23 [INFO] encoded CSR
2019/01/18 18:14:23 [INFO] signed certificate with serial number 375797145588654714099258750873820528127028390681
2019/01/18 18:14:23 [WARNING] This certificate lacks a "hosts" field. This makes it unsuitable for
websites. For more information see the Baseline Requirements for the Issuance and Management
of Publicly-Trusted Certificates, v.1.1.6, from the CA/Browser Forum (https://cabforum.org);
specifically, section 10.2.3 ("Information Requirements").检查生成的证书、私钥
复制/opt/certs
[root@hdss7-200 certs]# ls -l|grep kube-proxy
-rw------- 1 root root 1679 Jan 22 17:31 kube-proxy-client-key.pem
-rw-r--r-- 1 root root 1005 Jan 22 17:31 kube-proxy-client.csr
-rw-r--r-- 1 root root 1383 Jan 22 17:31 kube-proxy-client.pem
-rw-r--r-- 1 root root 268 Jan 22 17:23 kube-proxy-csr.json拷贝证书至各运算节点，并创建配置
HDSS7-21.host.com上：
拷贝证书、私钥，注意私钥文件属性600
复制/opt/kubernetes/server/bin/cert
[root@hdss7-21 cert]# ls -l /opt/kubernetes/server/bin/cert
total 40
-rw------- 1 root root 1676 Jan 21 16:39 apiserver-key.pem
-rw-r--r-- 1 root root 1599 Jan 21 16:36 apiserver.pem
-rw------- 1 root root 1675 Jan 21 13:55 ca-key.pem
-rw-r--r-- 1 root root 1354 Jan 21 13:50 ca.pem
-rw------- 1 root root 1679 Jan 21 13:53 client-key.pem
-rw-r--r-- 1 root root 1368 Jan 21 13:53 client.pem
-rw------- 1 root root 1679 Jan 22 17:00 kubelet-key.pem
-rw-r--r-- 1 root root 1456 Jan 22 17:00 kubelet.pem
-rw------- 1 root root 1679 Jan 22 17:31 kube-proxy-client-key.pem
-rw-r--r-- 1 root root 1383 Jan 22 17:31 kube-proxy-client.pem创建配置
set-cluster
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config set-cluster myk8s \
--certificate-authority=/opt/kubernetes/server/bin/cert/ca.pem \
--embed-certs=true \
--server=https://10.4.7.10:7443 \
--kubeconfig=kube-proxy.kubeconfig
Cluster "myk8s" set.set-credentials
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config set-credentials kube-proxy \
--client-certificate=/opt/kubernetes/server/bin/cert/kube-proxy-client.pem \
--client-key=/opt/kubernetes/server/bin/cert/kube-proxy-client-key.pem \
--embed-certs=true \
--kubeconfig=kube-proxy.kubeconfig
User "kube-proxy" set.set-context
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config set-context myk8s-context \
--cluster=myk8s \
--user=kube-proxy \
--kubeconfig=kube-proxy.kubeconfig
Context "myk8s-context" created.use-context
注意：在conf目录下
复制/opt/kubernetes/server/bin/conf
[root@hdss7-21 conf]# kubectl config use-context myk8s-context --kubeconfig=kube-proxy.kubeconfig
Switched to context "myk8s-context".
创建kube-proxy启动脚本
HDSS7-21.host.com上：
复制/opt/kubernetes/server/bin/kube-proxy.sh
#!/bin/sh
./kube-proxy \
--cluster-cidr 172.7.0.0/16 \
--hostname-override 10.4.7.21 \
--proxy-mode=ipvs \
--ipvs-scheduler=nq \
--kubeconfig ./conf/kube-proxy.kubeconfig注意：kube-proxy集群各主机的启动脚本略有不同，部署其他节点时注意修改。
检查配置，权限，创建日志目录
HDSS7-21.host.com上：
复制/opt/kubernetes/server/conf
[root@hdss7-21 conf]# ls -l|grep kube-proxy.kubeconfig
-rw------- 1 root root 6471 Jan 22 17:33 kube-proxy.kubeconfig
[root@hdss7-21 conf]# chmod +x /opt/kubernetes/server/bin/kube-proxy.sh
[root@hdss7-21 conf]# mkdir -p /data/logs/kubernetes/kube-proxy创建supervisor配置
HDSS7-21.host.com上：
复制/etc/supervisord.d/kube-proxy.ini
[program:kube-proxy-7-21]
command=/opt/kubernetes/server/bin/kube-proxy.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/kubernetes/server/bin ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=22 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/kubernetes/kube-proxy/proxy.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/kubernetes/kube-proxy/proxy.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false 启动服务并检查
HDSS7-21.host.com上：
复制
[root@hdss7-21 bin]# supervisorctl update
kube-proxy: added process group
[root@hdss7-21 bin]# supervisorctl status
etcd-server-7-21 RUNNING pid 9507, uptime 22:44:48
kube-apiserver RUNNING pid 9770, uptime 21:10:49
kube-controller-manager RUNNING pid 10048, uptime 18:22:10
kube-kubelet RUNNING pid 14597, uptime 0:32:59
kube-proxy STARTING
kube-scheduler RUNNING pid 10041, uptime 18:22:13安装部署启动检查所有集群规划主机上的kube-proxy服务
开启ipvs的脚本
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir|grep -o "^[^.]*")
do
/sbin/modinfo -F filename $i &>/dev/null
if [ $? -eq 0 ];then
/sbin/modprobe $i
fi
done略
验证集群:
在任意一个运算节点，创建一个资源配置清单
这里我们选择HDSS7-21.host.com主机
复制/root/nginx-ds.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: nginx-ds
spec:
template:
metadata:
labels:
app: nginx-ds
spec:
containers:
- name: my-nginx
image: harbor.od.com/public/nginx:v1.7.9
ports:
- containerPort: 80（1.16版本yaml文件格式，注意变化）
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: nginx-ds
spec:
selector:
matchLabels:
app: nginx-ds
template:
metadata:
labels:
app: nginx-ds
spec:
containers:
- name: my-nginx
image: harbor.kcwl.com/public/nginx:1.7.9
ports:
- containerPort: 80
kubectl命令
kubectl get namespace
查看命名空间
kubectl get ns
kubectl get all (-n default)
[root@hdss7-21 ~]# kubectl create ns app
namespace/app created
[root@hdss7-21 ~]# kubectl get ns
NAME STATUS AGE
app Active 5s
default Active 13d
kube-node-lease Active 13d
kube-public Active 13d
kube-system Active 13d
[root@hdss7-21 ~]# kubectl delete namespace app
namespace "app" deleted
[root@hdss7-21 ~]# kubectl get ns
NAME STATUS AGE
default Active 13d
kube-node-lease Active 13d
kube-public Active 13d
kube-system Active 13d
kubectl create deployment nginx-dp --image=harbor.od.com/public/nginx:v1.7.9 -n kube-public
kubectl get deploy -n kube-public
kubectl get pods -n kube-public -o wide
kubectl describe deployment nginx-dp -n kube-public
kubectl delete pod nginx-dp-5dfc689474-xkddb -n kube-public
kubectl delete deploy nginx-dp -n kube-public
kubectl expose deployment nginx-dp --port=80 -n kube-public
kubectl get all -n kube-public
kubectl scale deployment nginx-dp --replicas=2 -n kube-public
ipvsadm -Ln
kubectl scale deployment nginx-dp --replicas=1 -n kube-public
kubectl get pods -n kube-public
kubectl describe svc nginx-dp -n kube-public
kubectl get pods nginx-dp-5dfc689474-9f4nj -o yaml -n kube-public
kubectl cluster-info #查看集群信息
Kubernetes master is running at http://localhost:8080
CoreDNS is running at http://localhost:8080/api/v1/namespaces/kube-system/services/coredns:dns/proxy
kubernetes-dashboard is running at http://localhost:8080/api/v1/namespaces/kube-system/services/https:kubernetes-dashboard:/proxy
To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'.
[root@hdss7-21 bin]# kubectl -s http://localhost:8080 get componentstatuses #查看组件状态
# kubectl api-resources #查看api资源，有特别多资源
# kubectl explain Deployment|svc --recursive #列出svc 的所有字段，比explain 好用！explain 只列出下一层的命令
~]# kubectl get namespace,pod,rc,svc #查看这几个状态，具体要看什么就加什么
~]# kubectl get all [-n default]
# kubectl get pods nginx-ds-4qplw -o yaml # 查看 pod 的yaml 信息
~]# kubectl create ns app # 创建 名称是ns 的命名空间
~]# kubectl delete namespace app
# kubectl scale deployment DP名称 --replicas=$数量 -n infra
kubectl run nginx-deployment --image=nginx:1.7.9 --replicas=2 创建一个叫nginx-depoyment 的deployment
~]# kubectl create deployment nginx-dp --image=nginx:v1.7.9 -n kube-public #kube-public 空间创建 nginx-dp
~]# kubectl get deploy -n kube-public #查看 kube-public名称空间的deployment, delployment调度pod 的无状态服务控制器。
~]# kubectl get deployment -o wide -n kube-public
~]# kubectl describe deployment|pods nginx-dp -n kube-public #查看pod 的详细信息，pod 有问题可以用这个命令来查。
~]# kubectl get pods -n kube-public
kubectl exec -it nginx-ds-ktcrn /bin/bash -n kube-public #进入在运行的容器，必须是运行的，要不进不去。
~]# kubectl delete pod nginx-dp-5dfc689474-gtfvv -n kube-public [--force --grace-period=0]
kubectl delete pod nginx-dp-5dfc689474-gtfvv -n kube-public 是重启pod 的一种方法，删除后会创建新的pod，加了 [--force --grace-period=0] 就是强制删除pod 资源了
~]# kubectl delete deploy nginx-dp -n kube-public # 删除名称为nginx-dp 的deployment 调度器。
~]# kubectl expose deployment nginx-dp --port=80 -n kube-public #暴露 nginx-dp 的80 端口，service 会生成一个随机端口，不推荐使用，一般都是在service 的yaml 文件写好。
~]# kubectl scale deployment nginx-dp --replicas=2 -n kube-public #扩容
# kubectl get deployment nginx-dp -n kube-public #查看dp
NAME READY UP-TO-DATE AVAILABLE AGE
nginx-dp 2/2 2 2 22m
~]# kubectl describe svc nginx-dp -n kube-public
[root@hdss7-22 ~]# kubectl logs -f dubbo-demo-service-55c7b699c-9jqjp -n app #如果哪个pod 起不来，可以看日志，很重要！
实时观察滚动升级状态
kubectl rollout status deployment/dubbo-demo-consumer -n app
deployment "dubbo-demo-consumer" successfully rolled out
部署flannel
集群规划
| 主机名 | 角色 | ip |
|---|---|---|
| HDSS7-21.host.com | flannel | 10.4.7.21 |
| HDSS7-22.host.com | flannel | 10.4.7.22 |
注意：这里部署文档以HDSS7-21.host.com主机为例，另外一台运算节点安装部署方法类似
下载软件，解压，做软连接
HDSS7-21.host.com上：
复制/opt/src
[root@hdss7-21 src]# ls -l|grep flannel
-rw-r--r-- 1 root root 417761204 Jan 17 18:46 flannel-v0.11.0-linux-amd64.tar.gz
[root@hdss7-21 src]# mkdir -p /opt/flannel-v0.11.0/cert
[root@hdss7-21 src]# tar xf flannel-v0.11.0-linux-amd64.tar.gz -C /opt/flannel-v0.11.0
[root@hdss7-21 src]# ln -s /opt/flannel-v0.11.0 /opt/flannel
[root@hdss7-21 src]# ls -l /opt|grep flannel
lrwxrwxrwx 1 root root 31 Jan 17 18:49 flannel -> flannel-v0.10.0-linux-amd64/
drwxr-xr-x 4 root root 50 Jan 17 18:47 flannel-v0.10.0-linux-amd64
最终目录结构
复制/opt
[root@hdss7-21 opt]# tree -L 2
.
|-- etcd -> etcd-v3.1.18-linux-amd64
|-- etcd-v3.1.18-linux-amd64
| |-- Documentation
| |-- README-etcdctl.md
| |-- README.md
| |-- READMEv2-etcdctl.md
| |-- certs
| |-- etcd
| |-- etcd-server-startup.sh
| `-- etcdctl
|-- flannel -> flannel-v0.10.0/
|-- flannel-v0.10.0
| |-- README.md
| |-- cert
| |-- flanneld
| `-- mk-docker-opts.sh
|-- kubernetes -> kubernetes-v1.13.2-linux-amd64/
|-- kubernetes-v1.13.2-linux-amd64
| |-- LICENSES
| |-- addons
| `-- server
`-- src
|-- etcd-v3.1.18-linux-amd64.tar.gz
|-- flannel-v0.10.0-linux-amd64.tar.gz
`-- kubernetes-server-linux-amd64.tar.gz复制/opt/flannel/subnet.env HDSS7-21.host.com上：创建配置
FLANNEL_NETWORK=172.7.0.0/16
FLANNEL_SUBNET=172.7.21.1/24
FLANNEL_MTU=1500
FLANNEL_IPMASQ=false
注意：flannel集群各主机的配置略有不同，部署其他节点时注意修改。
创建启动脚本
HDSS7-21.host.com上：
复制/opt/flannel/flanneld.sh
#!/bin/sh
./flanneld \
--public-ip=10.4.7.21 \
--etcd-endpoints=https://10.4.7.12:2379,https://10.4.7.21:2379,https://10.4.7.22:2379 \
--etcd-keyfile=./cert/client-key.pem \
--etcd-certfile=./cert/client.pem \
--etcd-cafile=./cert/ca.pem \
--iface=eth0 \
--subnet-file=./subnet.env \
--healthz-port=2401
注意：flannel集群各主机的启动脚本略有不同，部署其他节点时注意修改。
检查配置，权限，创建日志目录
HDSS7-21.host.com上：
复制/opt/flannel
[root@hdss7-21 flannel]# chmod +x /opt/flannel/flanneld.sh
[root@hdss7-21 flannel]# cp /opt/certs/ca* /opt/flannel/cert/
[root@hdss7-21 flannel]# cp /opt/certs/client* /opt/flannel/cert/
[root@hdss7-21 flannel]# mkdir -p /data/logs/flanneld
HDSS7-21.host.com上：操作etcd，增加host-gw
复制/opt/etcd
[root@hdss7-21 etcd]# ./etcdctl set /coreos.com/network/config '{"Network": "172.7.0.0/16", "Backend": {"Type": "host-gw"}}'
HDSS7-21.host.com上：创建supervisor配置
复制/etc/supervisord.d/flanneld.ini
[program:flanneld-7-21]
command=/opt/flannel/flanneld.sh ; the program (relative uses PATH, can take args)
numprocs=1 ; number of processes copies to start (def 1)
directory=/opt/flannel ; directory to cwd to before exec (def no cwd)
autostart=true ; start at supervisord start (default: true)
autorestart=true ; retstart at unexpected quit (default: true)
startsecs=30 ; number of secs prog must stay running (def. 1)
startretries=3 ; max # of serial start failures (default 3)
exitcodes=0,2 ; 'expected' exit codes for process (default 0,2)
stopsignal=QUIT ; signal used to kill process (default TERM)
stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
user=root ; setuid to this UNIX account to run the program
redirect_stderr=false ; redirect proc stderr to stdout (default false)
stdout_logfile=/data/logs/flanneld/flanneld.stdout.log ; stdout log path, NONE for none; default AUTO
stdout_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stdout_logfile_backups=4 ; # of stdout logfile backups (default 10)
stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stdout_events_enabled=false ; emit events on stdout writes (default false)
stderr_logfile=/data/logs/flanneld/flanneld.stderr.log ; stderr log path, NONE for none; default AUTO
stderr_logfile_maxbytes=64MB ; max # logfile bytes b4 rotation (default 50MB)
stderr_logfile_backups=4 ; # of stderr logfile backups (default 10)
stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
stderr_events_enabled=false ; emit events on stderr writes (default false)
启动服务并检查
HDSS7-21.host.com上：
复制
[root@hdss7-21 flanneld]# supervisorctl update
flanneld: added process group
[root@hdss7-21 flanneld]# supervisorctl status
etcd-server-7-21 RUNNING pid 9507, uptime 1 day, 20:35:42
flanneld STARTING
kube-apiserver RUNNING pid 9770, uptime 1 day, 19:01:43
kube-controller-manager RUNNING pid 37646, uptime 0:58:48
kube-kubelet RUNNING pid 32640, uptime 17:16:36
kube-proxy RUNNING pid 15097, uptime 17:55:36
kube-scheduler RUNNING pid 37803, uptime 0:55:47
安装部署启动检查所有集群规划主机上的flannel服务
略
再次验证集群
部署k8s资源配置清单的内网http服务
在运维主机HDSS7-200.host.com上，配置一个nginx虚拟主机，用以提供k8s统一的资源配置清单访问入口
/etc/nginx/conf.d/k8s-yaml.od.com.conf
server {
listen 80;
server_name k8s-yaml.od.com;
location / {
autoindex on;
default_type text/plain;
root /data/k8s-yaml;
}
}
配置内网DNS解析
HDSS7-11.host.com上
复制
/var/named/od.com.zone
k8s-yaml 60 IN A 10.4.7.200
以后所有的资源配置清单统一放置在运维主机的/data/k8s-yaml目录下即可
复制
[root@hdss7-200 ~]# nginx -s reloadiptables规则优化
# iptables -t nat -D POSTROUTING -s 172.7.21.0/24 ! -o docker0 -j MASQUERADE
# iptables -t nat -I POSTROUTING -s 172.7.21.0/24 ! -d 172.7.0.0/16 ! -o docker0 -j MASQUERADE
10.4.7.21主机上的，来源是172.7.21.0/24段的docker的ip，目标ip不是172.7.0.0/16段，网络发包不从docker0桥设备出站的，才进行SNAT转换
各运算节点保存iptables规则
# iptables-save > /etc/sysconfig/iptables
安装core dns
准备coredns-v1.6.1镜像
运维主机HDSS7-200.host.com上：
复制
[root@hdss7-200 ~]# docker pull coredns/coredns:1.6.1
1.3.1: Pulling from coredns/coredns
e0daa8927b68: Pull complete
3928e47de029: Pull complete
Digest: sha256:02382353821b12c21b062c59184e227e001079bb13ebd01f9d3270ba0fcbf1e4
Status: Downloaded newer image for coredns/coredns:1.6.1
[root@hdss7-200 ~]# docker tag eb516548c180 harbor.od.com/k8s/coredns:v1.6.1
[root@hdss7-200 ~]# docker push harbor.od.com/k8s/coredns:v1.6.1
docker push harbor.od.com/k8s/coredns:v1.6.1
The push refers to a repository [harbor.od.com/k8s/coredns]
c6a5fc8a3f01: Pushed
fb61a074724d: Pushed
v1.6.1: digest: sha256:e077b9680c32be06fc9652d57f64aa54770dd6554eb87e7a00b97cf8e9431fda size: 739
任意一台运算节点上：
复制
[root@hdss7-21 ~]# kubectl create secret docker-registry harbor --docker-server=harbor.od.com --docker-username=admin --docker-password=Harbor12345 --docker-email=yangg_zhao@163.com -n kube-system
准备资源配置清单
运维主机HDSS7-200.host.com上：
复制
[root@hdss7-200 ~]# mkdir -p /data/k8s-yaml/coredns && cd /data/k8s-yaml/coredns
vi /data/k8s-yaml/coredns/rbac.yaml
复制
apiVersion: v1
kind: ServiceAccount
metadata:
name: coredns
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: Reconcile
name: system:coredns
rules:
- apiGroups:
- ""
resources:
- endpoints
- services
- pods
- namespaces
verbs:
- list
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
annotations:
rbac.authorization.kubernetes.io/autoupdate: "true"
labels:
kubernetes.io/bootstrapping: rbac-defaults
addonmanager.kubernetes.io/mode: EnsureExists
name: system:coredns
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:coredns
subjects:
- kind: ServiceAccount
name: coredns
namespace: kube-systemvi /data/k8s-yaml/coredns/configmap.yaml
apiVersion: v1
kind: ConfigMap
metadata:
name: coredns
namespace: kube-system
data:
Corefile: |
.:53 {
errors
log
health
ready
kubernetes cluster.local 10.0.0.0/16
forward . /etc/resolv.conf #上级DNS地址
cache 30
loop
reload
loadbalance
}vi /data/k8s-yaml/coredns/deployment.yaml
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
replicas: 1
selector:
matchLabels:
k8s-app: coredns
template:
metadata:
labels:
k8s-app: coredns
spec:
serviceAccountName: coredns
containers:
- name: coredns
image: harbor.od.com/k8s/coredns:v1.3.1
args:
- -conf
- /etc/coredns/Corefile
volumeMounts:
- name: config-volume
mountPath: /etc/coredns
ports:
- containerPort: 53
name: dns
protocol: UDP
- containerPort: 53
name: dns-tcp
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: 8080
scheme: HTTP
initialDelaySeconds: 60
timeoutSeconds: 5
successThreshold: 1
failureThreshold: 5
dnsPolicy: Default
imagePullSecrets:
- name: harbor
volumes:
- name: config-volume
configMap:
name: coredns
items:
- key: Corefile
path: Corefile（1.16注意修改apiversion信息为apps/v1）
vi /data/k8s-yaml/coredns/svc.yaml
apiVersion: v1
kind: Service
metadata:
name: coredns
namespace: kube-system
labels:
k8s-app: coredns
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "CoreDNS"
spec:
selector:
k8s-app: coredns
clusterIP: 192.168.0.2
ports:
- name: dns
port: 53
protocol: UDP
- name: dns-tcp
port: 53
在任意运算节点上应用资源配置清单浏览器打开：http://k8s-yaml.od.com/coredns 检查资源配置清单文件是否正确创建依次执行创建
复制
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.od.com/coredns/rbac.yaml
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.od.com/coredns/configmap.yaml
configmap/coredns created
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.od.com/coredns/deployment.yaml
deployment.extensions/coredns created
[root@hdss7-21 ~]# kubectl apply -f http://k8s-yaml.od.com/coredns/svc.yaml
service/coredns created
检查
复制
[root@hdss7-21 ~]# kubectl get pods -n kube-system -o wide
NAME READY STATUS RESTARTS AGE
coredns-7ccccdf57c-5b9ch 1/1 Running 0 3m4s
[root@hdss7-21 coredns]# kubectl get svc -n kube-system
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
coredns ClusterIP 192.168.0.2 <none> 53/UDP,53/TCP 29s
[root@hdss7-21 ~]# dig -t A nginx-ds.default.svc.cluster.local. @192.168.0.2 +short
192.168.0.3
准备traefik镜像部署traefik（ingress）
在hdss7-200上执行：
下载镜像：
# docker pull traefik:v1.7.2-alpine
# docker tag add5fac61ae5 harbor.od.com/public/traefik:v1.7.2
# docker push harbor.od.com/public/traefik:v1.7.2
运维主机HDSS7-200.host.com上：准备资源配置清单
复制
[root@hdss7-200 ~]# mkdir -p /data/k8s-yaml/traefik && cd /data/k8s-yaml/traefik
复制rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: traefik-ingress-controller
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: traefik-ingress-controller
rules:
- apiGroups:
- ""
resources:
- services
- endpoints
- secrets
verbs:
- get
- list
- watch
- apiGroups:
- extensions
resources:
- ingresses
verbs:
- get
- list
- watch
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
metadata:
name: traefik-ingress-controller
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: traefik-ingress-controller
subjects:
- kind: ServiceAccount
name: traefik-ingress-controller
namespace: kube-systemvi /data/k8s-yaml/traefik/daemonset.yaml
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: traefik-ingress
namespace: kube-system
labels:
k8s-app: traefik-ingress
spec:
template:
metadata:
labels:
k8s-app: traefik-ingress
name: traefik-ingress
spec:
serviceAccountName: traefik-ingress-controller
terminationGracePeriodSeconds: 60
containers:
- image: harbor.od.com/public/traefik:v1.7.2
name: traefik-ingress
ports:
- name: controller
containerPort: 80
hostPort: 81
- name: admin-web
containerPort: 8080
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
args:
- --api
- --kubernetes
- --logLevel=INFO
- --insecureskipverify=true
- --kubernetes.endpoint=https://10.4.7.10:7443
- --accesslog
- --accesslog.filepath=/var/log/traefik_access.log
- --traefiklog
- --traefiklog.filepath=/var/log/traefik.log
- --metrics.prometheusvi /data/k8s-yaml/traefik/svc.yaml
kind: Service
apiVersion: v1
metadata:
name: traefik-ingress-service
namespace: kube-system
spec:
selector:
k8s-app: traefik-ingress
ports:
- protocol: TCP
port: 80
name: controller
- protocol: TCP
port: 8080
name: admin-webvi /data/k8s-yaml/traefik/ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: traefik-web-ui
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: traefik.od.com
http:
paths:
- path: /
backend:
serviceName: traefik-ingress-service
servicePort: 8080HDSS7-11.host.com上解析域名
复制/var/named/od.com.zone
traefik A 10.4.7.10
依次执行创建
浏览器打开：http://k8s-yaml.od.com/traefik 检查资源配置清单文件是否正确创建
在任意运算节点应用资源配置清单
复制
# kubectl create -f http://k8s-yaml.od.com/traefik/rbac.yaml
# kubectl create -f http://k8s-yaml.od.com/traefik/ds.yaml
# kubectl create -f http://k8s-yaml.od.com/traefik/svc.yaml
# kubectl create -f http://k8s-yaml.od.com/traefik/ingress.yaml
配置反代
HDSS7-11.host.com和HDSS7-12.host.com两台主机上的nginx均需要配置，这里可以考虑使用saltstack或者ansible进行统一配置管理
复制/etc/nginx/conf.d/od.com.conf
upstream default_backend_traefik {
server 10.4.7.21:81 max_fails=3 fail_timeout=10s;
server 10.4.7.22:81 max_fails=3 fail_timeout=10s;
}
server {
server_name *.od.com;
location / {
proxy_pass http://default_backend_traefik;
proxy_set_header Host $http_host;
proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
}
}
浏览器访问
部署dashboard
准备dashboard镜像
运维主机HDSS7-200.host.com上：
复制
# docker pull k8scn/kubernetes-dashboard-amd64:v1.8.3
# docker tag fcac9aa03fd6 harbor.od.com/public/dashboard:v1.8.3
# docker push harbor.od.com/public/dashboard:v1.8.3
准备资源配置清单
运维主机HDSS7-200.host.com上：
复制
[root@hdss7-200 ~]# mkdir -p /data/k8s-yaml/dashboard && cd /data/k8s-yaml/dashboard
vi /data/k8s-yaml/dashboard/rbac.yaml
复制
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard-admin
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard-admin
namespace: kube-systemvi dp.yamlapiVersion: apps/v1
kind: Deployment
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
priorityClassName: system-cluster-critical
containers:
- name: kubernetes-dashboard
image: harbor.od.com/public/dashboard:v1.8.3
resources:
limits:
cpu: 100m
memory: 300Mi
requests:
cpu: 50m
memory: 100Mi
ports:
- containerPort: 8443
protocol: TCP
args:
# PLATFORM-SPECIFIC ARGS HERE
- --auto-generate-certificates
volumeMounts:
- name: tmp-volume
mountPath: /tmp
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard-admin
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"vi svc.yamlapiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
k8s-app: kubernetes-dashboard
ports:
- port: 443
targetPort: 8443vi ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: dashboard.od.com
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443HDSS7-11.host.com上解析域名
复制/var/named/od.com.zone
dashboard 60 IN A 10.4.7.100
systemctl restart named
依次执行创建
浏览器打开：http://k8s-yaml.od.com/dashboard 检查资源配置清单文件是否正确创建
在任意运算节点应用资源配置清单
复制
# kubectl create -f http://k8s-yaml.od.com/dashboard/rbac.yaml
# kubectl create -f http://k8s-yaml.od.com/dashboard/dp.yaml
# kubectl create -f http://k8s-yaml.od.com/dashboard/svc.yaml
# kubectl create -f http://k8s-yaml.od.com/dashboard/ingress.yaml
浏览器访问
配置认证
下载新版dashboard
复制
# docker pull loveone/kubernetes-dashboard-amd64:v1.10.1
# docker tag f9aed6605b81 harbor.od.com/public/dashboard:v1.10.1
# docker push harbor.od.com/public/dashboard:v1.10.1修改dp.yaml重新应用，我直接用edit修改了，没有使用apply
# kubectl edit deploy kubernetes-dashboard -n kube-system等待滚动发布完成后，在刷新dashboard页面：
可以看到这里原来的skip跳过已经没有了，我们如果想登陆，必须输入token，那我们如何获取token呢：
# kubectl get secret -n kube-system
# kubectl describe secret kubernetes-dashboard-admin-token-pg77n -n kube-system复制获取token
这样我们就拿到了token，接下来我们试试能不能登录：
我们发现我们还是无法登录，原因是必须使用https登录，接下来我们需要申请证书：
接下来我们申请证书：
依然使用cfssl来申请证书：hdss7-200
# cd /opt/certs/
# vi dashboard-csr.json
复制代码
{
"CN": "*.od.com",
"hosts": [
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "od",
"OU": "ops"
}
]
}fssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server dashboard-csr.json |cfssl-json -bare dashboard然后拷贝到我们nginx的服务器上：7-11 7-12 都需要
# cd /etc/nginx/
# mkdir certs
# cd certs
# scp hdss7-200:/opt/cert/dash* ./# cd /etc/nginx/conf.d/
# vi dashboard.od.com.conf
server {
listen 80;
server_name dashboard.od.com;
rewrite ^(.*)$ https://${server_name}$1 permanent;
}
server {
listen 443 ssl;
server_name dashboard.od.com;
ssl_certificate "certs/dashboard.pem";
ssl_certificate_key "certs/dashboard-key.pem";
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 10m;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
location / {
proxy_pass http://default_backend_traefik;
proxy_set_header Host $http_host;
proxy_set_header x-forwarded-for $proxy_add_x_forwarded_for;
}
}
# nginx -t
# nginx -s reload可以登录了~
登录是登录了，但是我们要思考一个问题，我们使用rbac授权来访问dashboard,如何做到权限精细化呢？比如开发，只能看，不能摸，不同的项目组，看到的资源应该是不一样的，测试看到的应该是测试相关的资源。
小插曲：
k8s1.16支持的dashboard2.0.0beat版本
新版本 Dashboard 集成了一个metrics-scraper 的组件，可以通过 Kubernetes 的 Metrics API 收集一些基础资源的监控信息，并在 web 页面上展示，所以要想在页面上展示监控信息就需要提供 Metrics API，前提需要安装 Metrics Server。
安装Metrics Server 及和dashboard一起安装的metrics-scraper
原版镜像下载：
https://github.com/kubernetes/dashboard/releases/tag/v2.0.0-beta8
docker pull kubernetesui/dashboard:v2.0.0-beta8
docker pull kubernetesui/metrics-scraper:v1.0.2dashboard部署yaml文件记录：
rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
name: kubernetes-dashboard-admin
namespace: kube-system
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-certs
namespace: kube-system
type: Opaque
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-csrf
namespace: kube-system
type: Opaque
data:
csrf: ""
---
apiVersion: v1
kind: Secret
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-key-holder
namespace: kube-system
type: Opaque
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard-admin
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard-admin
namespace: kube-systemdp.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
matchLabels:
k8s-app: kubernetes-dashboard
template:
metadata:
labels:
k8s-app: kubernetes-dashboard
annotations:
scheduler.alpha.kubernetes.io/critical-pod: ''
spec:
priorityClassName: system-cluster-critical
containers:
- name: kubernetes-dashboard
image: harbor.kcwl.com/public/dashboard/v2.0.0-beta8
resources:
limits:
cpu: 100m
memory: 1000Mi
requests:
cpu: 50m
memory: 500Mi
ports:
- containerPort: 8443
protocol: TCP
args:
# PLATFORM-SPECIFIC ARGS HERE
- --auto-generate-certificates
volumeMounts:
- name: tmp-volume
mountPath: /tmp
livenessProbe:
httpGet:
scheme: HTTPS
path: /
port: 8443
initialDelaySeconds: 30
timeoutSeconds: 30
volumes:
- name: tmp-volume
emptyDir: {}
serviceAccountName: kubernetes-dashboard-admin
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"svc.yaml
apiVersion: v1
kind: Service
metadata:
name: kubernetes-dashboard
namespace: kube-system
labels:
k8s-app: kubernetes-dashboard
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
selector:
k8s-app: kubernetes-dashboard
ports:
- port: 443
targetPort: 8443ingress.yaml
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: kubernetes-dashboard
namespace: kube-system
annotations:
kubernetes.io/ingress.class: traefik
spec:
rules:
- host: dashboard.kcwl.com
http:
paths:
- backend:
serviceName: kubernetes-dashboard
servicePort: 443部署metrics-scraper获取监控指标插件yaml文件记录
apiVersion: v1
kind: ServiceAccount
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
---
kind: ConfigMap
apiVersion: v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard-settings
namespace: kube-system
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
rules:
# Allow Dashboard to get, update and delete Dashboard exclusive secrets.
- apiGroups: [""]
resources: ["secrets"]
resourceNames: ["kubernetes-dashboard-key-holder", "kubernetes-dashboard-certs", "kubernetes-dashboard-csrf"]
verbs: ["get", "update", "delete"]
# Allow Dashboard to get and update 'kubernetes-dashboard-settings' config map.
- apiGroups: [""]
resources: ["configmaps"]
resourceNames: ["kubernetes-dashboard-settings"]
verbs: ["get", "update"]
# Allow Dashboard to get metrics.
- apiGroups: [""]
resources: ["services"]
resourceNames: ["heapster", "dashboard-metrics-scraper"]
verbs: ["proxy"]
- apiGroups: [""]
resources: ["services/proxy"]
resourceNames: ["heapster", "http:heapster:", "https:heapster:", "dashboard-metrics-scraper", "http:dashboard-metrics-scraper"]
verbs: ["get"]
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
rules:
# Allow Metrics Scraper to get metrics from the Metrics server
- apiGroups: ["metrics.k8s.io"]
resources: ["pods", "nodes"]
verbs: ["get", "list", "watch"]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
labels:
k8s-app: kubernetes-dashboard
name: kubernetes-dashboard
namespace: kube-system
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: kubernetes-dashboard
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: kubernetes-dashboard
subjects:
- kind: ServiceAccount
name: kubernetes-dashboard
namespace: kube-system
---
kind: Service
apiVersion: v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kube-system
spec:
ports:
- port: 8000
targetPort: 8000
selector:
k8s-app: dashboard-metrics-scraper
---
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
k8s-app: dashboard-metrics-scraper
name: dashboard-metrics-scraper
namespace: kube-system
spec:
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
k8s-app: dashboard-metrics-scraper
template:
metadata:
labels:
k8s-app: dashboard-metrics-scraper
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'runtime/default'
spec:
containers:
- name: dashboard-metrics-scraper
image: harbor.kcwl.com/public/metrics-scraper:v1.0.2
ports:
- containerPort: 8000
protocol: TCP
livenessProbe:
httpGet:
scheme: HTTP
path: /
port: 8000
initialDelaySeconds: 30
timeoutSeconds: 30
volumeMounts:
- mountPath: /tmp
name: tmp-volume
securityContext:
allowPrivilegeEscalation: false
readOnlyRootFilesystem: true
runAsUser: 1001
runAsGroup: 2001
serviceAccountName: kubernetes-dashboard
nodeSelector:
"beta.kubernetes.io/os": linux
# Comment the following tolerations if Dashboard must not be deployed on master
#tolerations:
# - key: node-role.kubernetes.io/master
# effect: NoSchedule
volumes:
- name: tmp-volume
emptyDir: {}部署：metrics-server
github地址，获取配置文件方式
https://github.com/kubernetes/kubernetes/tree/release-1.16/cluster/addons/metrics-server集群部署好后，如果我们想知道集群中每个节点及节点上的pod资源使用情况，命令行下可以直接使用kubectl top node/pod来查看资源使用情况，默认此命令不能正常使用，需要我们部署对应api资源才可以使用此命令。从 Kubernetes 1.8 开始，资源使用指标（如容器 CPU 和内存使用率）通过 Metrics API 在 Kubernetes 中获取, metrics-server 替代了heapster。Metrics Server 实现了Resource Metrics API，Metrics Server 是集群范围资源使用数据的聚合器。 Metrics Server 从每个节点上的 Kubelet 公开的 Summary API 中采集指标信息。heapster从1.13版本开始被废弃，官方推荐使用Metrics Server+Prometheus方案进行集群监控。
修改配置文件（以下有改好的配置文件，检查下即可使用）
##镜像地址启动参数修改
[root@k8s-master-13 metrics]# vim metrics-server-deployment.yaml
###mertics-server 修改启动参数镜像地址
......
containers:
- name: metrics-server
image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server-amd64:v0.3.1
command:
- /metrics-server
- --metric-resolution=30s
- --kubelet-insecure-tls
- --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP
......
###metrics-server-nanny 修改镜像地址及启动参数
......
- name: metrics-server-nanny
image: registry.cn-hangzhou.aliyuncs.com/google_containers/addon-resizer:1.8.7
.....
command:
- /pod_nanny
- --config-dir=/etc/config
- --cpu=100m
- --extra-cpu=0.5m
- --memory=100Mi
- --extra-memory=50Mi
- --threshold=5
- --deployment=metrics-server-v0.3.6
- --container=metrics-server
- --poll-period=300000
- --estimator=exponential
......
## 在新的版本中，授权文内没有 node/stats 的权限，需要手动去添加
[root@k8s-master01 metrics]# vim resource-reader.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:metrics-server
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats ## 添加此参数
- namespacesmertics-server镜像参数解释：
- --kubelet-insecure-tls：不验证客户端证书
kube-apiserver配置文件修改
二进制部署安装，需要手动修改apiserver添加开启聚合服务的参数，当然如果你已经添加，那么请跳过这一步
## 编辑kube-apiserver.conf 添加如下参数，从下面参数中可以看出，需要生成新的证书，因此我们还需要为metrics生产证书
--requestheader-allowed-names="" \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--proxy-client-cert-file ./cert/metrics-proxy.pem \
--proxy-client-key-file ./cert/metrics-proxy-key.pem \参数说明：
- --requestheader-XXX、--proxy-client-XXX 是 kube-apiserver 的 aggregator layer 相关的配置参数，metrics-server & HPA 需要使用；
- --requestheader-client-ca-file：用于签名 --proxy-client-cert-file 和 --proxy-client-key-file 指定的证书(ca证书)，在启用了 metric aggregator 时使用；
注：如果 --requestheader-allowed-names 不为空，则--proxy-client-cert-file 证书的 CN 必须位于 allowed-names 中，默认为 aggregator；
如果 kube-apiserver 机器没有运行 kube-proxy，则还需要添加 --enable-aggregator-routing=true 参数
为metrics server生成证书
上面可以看到，kube-apiserver开启聚合层，也需要使用证书，为了便于区分，我们这里为mertics 单独生产证书
关于证书的创建也可参考之前部署其它组件时创建证书时候的步骤
vi metrics-proxy-csr.json
{
"CN": "metrics-proxy",
"hosts": [
"192.168.0.13",
"192.168.0.107",
"192.168.0.31"
],
"key": {
"algo": "rsa",
"size": 2048
},
"names": [
{
"C": "CN",
"ST": "beijing",
"L": "beijing",
"O": "kcwl",
"OU": "ops"
}
]
}生成kube-proxy证书与私钥
cfssl gencert -ca=/opt/certs/ca.pem -ca-key=/opt/certs/ca-key.pem -config=/opt/certs/ca-config.json -config=/opt/certs/ca-config.json -profile=client metrics-proxy-csr.json | cfssl-json -bare metrics-proxy拷贝证书到apiserver启动脚本的目录中
kubelet参数修改
添加--authentication-token-webhook=true参数根据清单文件创建mertics server服务
kubectl apply -f .
kubectl get pods -n kube-system |grep metrics-server
[root@k8s-master-13 certs]# kubectl top node
NAME CPU(cores) CPU% MEMORY(bytes) MEMORY%
192.168.0.107 66m 1% 2899Mi 18%
192.168.0.13 104m 5% 2135Mi 57%
192.168.0.31 64m 1% 2676Mi 16%
yaml文件记录：
auth-delegator.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: metrics-server:system:auth-delegator
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:auth-delegator
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-systemauth-reader.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: metrics-server-auth-reader
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: extension-apiserver-authentication-reader
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-systemmetrics-apiservice.yaml
apiVersion: apiregistration.k8s.io/v1
kind: APIService
metadata:
name: v1beta1.metrics.k8s.io
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
spec:
service:
name: metrics-server
namespace: kube-system
group: metrics.k8s.io
version: v1beta1
insecureSkipTLSVerify: true
groupPriorityMinimum: 100
versionPriority: 100metrics-server-deployment.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: metrics-server
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
---
apiVersion: v1
kind: ConfigMap
metadata:
name: metrics-server-config
namespace: kube-system
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: EnsureExists
data:
NannyConfiguration: |-
apiVersion: nannyconfig/v1alpha1
kind: NannyConfiguration
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: metrics-server-v0.3.6
namespace: kube-system
labels:
k8s-app: metrics-server
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
version: v0.3.6
spec:
selector:
matchLabels:
k8s-app: metrics-server
version: v0.3.6
template:
metadata:
name: metrics-server
labels:
k8s-app: metrics-server
version: v0.3.6
annotations:
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
spec:
priorityClassName: system-cluster-critical
serviceAccountName: metrics-server
nodeSelector:
kubernetes.io/os: linux
containers:
- name: metrics-server
image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server-amd64:v0.3.6
command:
- /metrics-server
- --metric-resolution=30s
- --kubelet-insecure-tls
- --kubelet-preferred-address-types=InternalIP,Hostname,InternalDNS,ExternalDNS,ExternalIP
ports:
- containerPort: 443
name: https
protocol: TCP
- name: metrics-server-nanny
image: registry.cn-hangzhou.aliyuncs.com/google_containers/addon-resizer:1.8.7
resources:
limits:
cpu: 100m
memory: 300Mi
requests:
cpu: 5m
memory: 50Mi
env:
- name: MY_POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: MY_POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
volumeMounts:
- name: metrics-server-config-volume
mountPath: /etc/config
command:
- /pod_nanny
- --config-dir=/etc/config
- --cpu=100m
- --extra-cpu=0.5m
- --memory=300Mi
- --extra-memory=100Mi
- --threshold=5
- --deployment=metrics-server-v0.3.6
- --container=metrics-server
- --poll-period=300000
- --estimator=exponential
# Specifies the smallest cluster (defined in number of nodes)
# resources will be scaled to.
#- --minClusterSize={{ metrics_server_min_cluster_size }}
volumes:
- name: metrics-server-config-volume
configMap:
name: metrics-server-config
tolerations:
- key: "CriticalAddonsOnly"
operator: "Exists"metrics-server-service.yaml
apiVersion: v1
kind: Service
metadata:
name: metrics-server
namespace: kube-system
labels:
addonmanager.kubernetes.io/mode: Reconcile
kubernetes.io/cluster-service: "true"
kubernetes.io/name: "Metrics-server"
spec:
selector:
k8s-app: metrics-server
ports:
- port: 443
protocol: TCP
targetPort: httpsresource-reader.yaml
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: system:metrics-server
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
rules:
- apiGroups:
- ""
resources:
- pods
- nodes
- nodes/stats
- namespaces
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
- "apps"
resources:
- deployments
verbs:
- get
- list
- update
- watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:metrics-server
labels:
kubernetes.io/cluster-service: "true"
addonmanager.kubernetes.io/mode: Reconcile
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: system:metrics-server
subjects:
- kind: ServiceAccount
name: metrics-server
namespace: kube-system踩坑记录：
1、apiserver需要安装kubelet，kubu-proxy，flannel，然后打污点不接受pod调度
kubectl taint nodes 192.168.0.13 node-role.kubernetes.io/master=:NoSchedule2、生成proxy证书需要用client的方式，否则401
3、apiserver需要加配置
4、kubectl get apiservices v1beta1.metrics.k8s.io -o yaml检查是否正确
5、kubelet需要加配置，并重启
1344
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
