K8S
kubeadm安装dashboard、Harbor私有仓库
master(4C/4G,cpu核心数要求大于2) 192.168.100.100 docker、kubeadm、kubelet、kubectl、flannel
node01(2C/4G) 192.168.100.110 docker、kubeadm、kubelet、kubectl、flannel
node02(2C/4G) 192.168.100.120 docker、kubeadm、kubelet、kubectl、flannel
Harbor节点(hub.lp.com) 192.168.100.130 docker、docker-compose、harbor-offline-v1.2.2
一、安装dashboard
所有节点安装dashboard
方法一
所有节点上传dashboard镜像 dashboard.tar 到 /opt 目录,master节点上传kubernetes-dashboard.yaml文件
所有节点上
cd /opt/
docker load < dashboard.tar
master节点上
cd /opt/
docker load < dashboard.tar
kubectl apply -f kubernetes-dashboard.yaml
方法二
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.0.0/aio/deploy/recommended.yaml
查看所有容器运行状态
kubectl get pods,svc -n kube-system -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/coredns-5c98db65d4-gv5p2 1/1 Running 0 58m 10.244.0.2 master <none> <none>
pod/coredns-5c98db65d4-mnwwb 1/1 Running 0 58m 10.244.0.3 master <none> <none>
pod/etcd-master 1/1 Running 0 57m 192.168.100.100 master <none> <none>
pod/kube-apiserver-master 1/1 Running 0 57m 192.168.100.100 master <none> <none>
pod/kube-controller-manager-master 1/1 Running 0 57m 192.168.100.100 master <none> <none>
pod/kube-flannel-ds-amd64-9d95m 1/1 Running 0 54m 192.168.100.110 node01 <none> <none>
pod/kube-flannel-ds-amd64-mst9v 1/1 Running 0 54m 192.168.100.100 master <none> <none>
pod/kube-flannel-ds-amd64-r2hwn 1/1 Running 0 54m 192.168.100.120 node02 <none> <none>
pod/kube-proxy-ghxsg 1/1 Running 0 58m 192.168.100.100 master <none> <none>
pod/kube-proxy-lx67s 1/1 Running 0 56m 192.168.100.110 node01 <none> <none>
pod/kube-proxy-xd4xp 1/1 Running 0 56m 192.168.100.120 node02 <none> <none>
pod/kube-scheduler-master 1/1 Running 0 57m 192.168.100.100 master <none> <none>
pod/kubernetes-dashboard-859b87d4f7-n2br6 1/1 Running 0 31m 10.244.2.3 node02 <none> <none>
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/kube-dns ClusterIP 10.96.0.10 <none> 53/UDP,53/TCP,9153/TCP 58m k8s-app=kube-dns
service/kubernetes-dashboard NodePort 10.96.16.10 <none> 443:30001/TCP 31m k8s-app=kubernetes-dashboard
使用火狐或者360浏览器访问
https://node02:30001/
https://192.168.100.120:30001/
创建service account并绑定默认cluster-admin管理员集群角色
kubectl create serviceaccount dashboard-admin -n kube-system
kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
获取令牌密钥
kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
Name: dashboard-admin-token-t2gfw
Namespace: kube-system
Labels: <none>
Annotations: kubernetes.io/service-account.name: dashboard-admin
kubernetes.io/service-account.uid: f4f03ee3-19a4-40bb-a0e0-e8db673aa424
Type: kubernetes.io/service-account-token
Data
====
ca.crt: 1025 bytes
namespace: 11 bytes
token: eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tdDJnZnciLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiZjRmMDNlZTMtMTlhNC00MGJiLWEwZTAtZThkYjY3M2FhNDI0Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.nuLMsyzuR1g0gg3PolrepXB7J5V4fXmCx0oo4zEhBfNoRBqv7j10GTgRAIjegE61hH699DE-M9CtRPd0N4sjFSgIgwIt5WjYIKWlJMBfxArNzusfUSVbMpyA1fS-kIBuA9nByddruZN3A_iD6zkrxlgCX83w-TJj4lYe6IPn7IRNV_Lrw8Wm9LFPCkp75ioCX0DcHsSFtAPYCBtt60-w3-VzT-X3f_bn3FsXgfY6HoDrxyxX9jAh211ytjYHYglo8A6DbqrCbLjkNR_7lVms4CeyDdp3PSlvJ1wXRaAPru9Bo2hPVo0yaVXus4vijCuTZDv1izBzOKhtsTdqMJNtTQ
eyJhbGciOiJSUzI1NiIsImtpZCI6IiJ9.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tdDJnZnciLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiZjRmMDNlZTMtMTlhNC00MGJiLWEwZTAtZThkYjY3M2FhNDI0Iiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.nuLMsyzuR1g0gg3PolrepXB7J5V4fXmCx0oo4zEhBfNoRBqv7j10GTgRAIjegE61hH699DE-M9CtRPd0N4sjFSgIgwIt5WjYIKWlJMBfxArNzusfUSVbMpyA1fS-kIBuA9nByddruZN3A_iD6zkrxlgCX83w-TJj4lYe6IPn7IRNV_Lrw8Wm9LFPCkp75ioCX0DcHsSFtAPYCBtt60-w3-VzT-X3f_bn3FsXgfY6HoDrxyxX9jAh211ytjYHYglo8A6DbqrCbLjkNR_7lVms4CeyDdp3PSlvJ1wXRaAPru9Bo2hPVo0yaVXus4vijCuTZDv1izBzOKhtsTdqMJNtTQ
//复制token令牌直接登录网站
二、安装Harbor私有仓库
修改主机名
hostnamectl set-hostname hub.lp.com
所有节点加上主机名映射
echo '192.168.100.130 hub.lp.com' >> /etc/hosts
安装 docker
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce docker-ce-cli containerd.io
mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["https://q7n9qid7.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"insecure-registries": ["https://hub.lp.com"]
}
EOF
EOF
systemctl start docker
systemctl enable docker
安装 Harbor
上传 harbor-offline-installer-v1.2.2.tgz 和 docker-compose 文件到 /opt 目录
cd /opt
cp docker-compose /usr/local/bin/
chmod +x /usr/local/bin/docker-compose
tar zxvf harbor-offline-installer-v1.2.2.tgz
cd harbor/
vim harbor.cfg
5 hostname = hub.lp.com
9 ui_url_protocol = https
24 ssl_cert = /data/cert/server.crt
25 ssl_cert_key = /data/cert/server.key
59 harbor_admin_password = Harbor12345
生成证书
mkdir -p /data/cert
cd /data/cert
#生成私钥
openssl genrsa -des3 -out server.key 2048
输入两遍密码:123456
#生成证书签名请求文件
openssl req -new -key server.key -out server.csr
输入私钥密码:123456
输入国家名:CN
输入省名:BJ
输入市名:BJ
输入组织名:LP
输入机构名:LP
输入域名:hub.lp.com
输入管理员邮箱:admin@lp.com
其它全部直接回车
#备份私钥
cp server.key server.key.org
#清除私钥密码
openssl rsa -in server.key.org -out server.key
输入私钥密码:123456
#签名证书
openssl x509 -req -days 1000 -in server.csr -signkey server.key -out server.crt
chmod +x /data/cert/*
cd /opt/harbor/
./install.sh
浏览器访问:https://hub.lp.com
用户名:admin
密码:Harbor12345
在所有节点上修改daemon.json 文件
cat > /etc/docker/daemon.json <<EOF
{
"registry-mirrors": ["https://q7n9qid7.mirror.aliyuncs.com"],
"exec-opts": ["native.cgroupdriver=systemd"],
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
},
"insecure-registries": ["https://hub.lp.com"]
}
EOF
systemctl daemon-reload
systemctl restart docker.service
systemctl enable docker.service
在一个node节点上登录harbor
docker login -u admin -p Harbor12345 https://hub.lp.com
上传镜像
docker tag nginx:latest hub.lp.com/library/nginx:v1
docker push hub.lp.com/library/nginx:v1
在master节点上删除之前创建的nginx资源
kubectl delete deployment nginx
kubectl run nginx-test1 --image=hub.lp.com/library/nginx:v1 --port=80 --replicas=2
kubectl expose deployment nginx-test1 --port=30000 --target-port=80
kubectl get svc,pods
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3h16m
service/nginx NodePort 10.96.76.72 <none> 80:32277/TCP 3h7m
service/nginx-deployment NodePort 10.96.31.178 <none> 30000:32584/TCP 36m
service/nginx-test1 ClusterIP 10.96.208.79 <none> 30000/TCP 22s
NAME READY STATUS RESTARTS AGE
pod/nginx-test1-7c5f7ccf5b-9ppfw 1/1 Running 0 53s
pod/nginx-test1-7c5f7ccf5b-sgvjl 1/1 Running 0 53s
yum install ipvsadm -y
ipvsadm -Ln
kubectl edit svc nginx-test1
25 type: NodePort #把调度策略改成NodePort
kubectl get svc
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 3h20m
nginx NodePort 10.96.76.72 <none> 80:32277/TCP 3h12m
nginx-deployment NodePort 10.96.31.178 <none> 30000:32584/TCP 40m
nginx-test1 NodePort 10.96.208.79 <none> 30000:31942/TCP 4m37s
浏览器访问
192.168.100.100:31942
192.168.100.110:31942
192.168.100.120:31942
内核参数优化方案
cat > /etc/sysctl.d/kubernetes.conf <<EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
net.ipv4.tcp_tw_recycle=0
vm.swappiness=0 #禁止使用 swap 空间,只有当系统内存不足(OOM)时才允许使用它
vm.overcommit_memory=1 #不检查物理内存是否够用
vm.panic_on_oom=0 #开启 OOM
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963 #指定最大文件句柄数
fs.nr_open=52706963 #仅4.4以上版本支持
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF