k8s的私人搭建

下载 centos7 docker pull centos:centos7

1 环境准备

修改源

#修改中文显示
vi /etc/locale.conf
LANG="zh_CN.UTF-8"
LC_ALL="zh_CN.UTF-8"
LC_CTYPE="zh_CN.UTF-8"

#更新
yum upgrade -y

#cat /etc/hosts
port 10000   192.168.0.104   centos-1.shared   master
port 10001   192.168.0.108   centos-2.shared   node01   
port 10002   192.168.0.109   centos-3. shared   node02  

2 host文件

cat /etc/hosts
192.168.1.104 centos-1 master
192.168.1.108 centos-2 node01
192.168.1.109 centos-3 node02

3 docker 环境配置

1 关闭系统默认防火墙和selinux

setenforce 0
sed -i -r "/^SELINUX=/c SELINUX=disabled" /etc/selinux/config
which systemctl && systemctl stop firewalld
which systemctl && systemctl disable firewalld
which systemctl && systemctl stop iptables || service iptables stop
which systemctl && systemctl disable iptables || chkconfig iptables off

2 卸载旧版本 docker

yum remove -y docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-selinux docker-engine-selinux docker-engine docker docker-ce docker-ee

3 每个环境里单独安装docker

# 1. 安装所需的包
# yum-utils提供了yum-config-manager 实用程序，并且devicemapper存储驱动需要device-mapper-persistent-data和lvm2
yum install -y yum-utils device-mapper-persistent-data lvm2
# 2.更新xfsprogs
yum update xfsprogs
#3.使用以下命令设置源
#官方源
yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
#阿里源（建议使用）
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 4.安装最新版本的Docker CE
yum install -y docker-ce
#或者安装指定版本
yum list docker-ce --showduplicates | sort -r 
yum install -y docker-ce-17.12.1.ce-1.el7.centos

4.启动Docker并设置开机运行

systemctl start docker
systemctl enable docker
systemctl status docker

5.常见问题

WARINING提示：
WARNING: bridge-nf-call-iptables is disabled 
WARNING: bridge-nf-call-ip6tables is disabled
#解决方案
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1  
net.bridge.bridge-nf-call-iptables = 1 
EOF
sysctl --system

4.其他准备工作

1 关闭防火墙

systemctl stop firewalld.service
systemctl stop iptables.service
systemctl disable firewalld.service
systemctl disable iptables.service
systemctl status firewalld
systemctl status iptables

2 禁用 selinux

#临时关闭：
setenforce 0
#永久关闭：
vim /etc/selinux/config
SELINUX=disabled

3 禁用swap 设备（影响性能， k8s集群初始化会报错）

#临时禁用
swapoff -a
#永久禁用
vim /etc/fstab
注释 /dev/mapper/VolGroup-lv_swap swap 所在行

4 启用ipvs内核模块创建内核模块载入相关的脚本文件/etc/sysconfig/modules/ipvs.modules，设定自动载入的内核模块。文件内容如下（用到再配置也可以）

#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir | grep -o "^[^.]*"); do
	/sbin/modinfo -F filename $i &> /dev/null
	if [ $? -eq 0 ]; then
		/sbin/modprobe $i
	fi
done

#赋权、运行并检查
chmod +x /etc/sysconfig/modules/ipvs.modules
/etc/sysconfig/modules/ipvs.modules
lsmod |grep ip_vs

5 docker文件配置(docker unit file: /usr/lib/systemd/system/docker.service)

vi /usr/lib/systemd/system/docker.service

ExecStart=/usr/bin/dockerd -H fd:// --containerd=/run/containerd/containerd.sock ExecStartPost=/usr/sbin/iptables -P FORWARD ACCEPT
ExecReload=/bin/kill -S HUP $MAINPID
TimeoutSe=0
RestartSec=2
Restart=always

#重启docker
systemctl daemon-reload
systemctl restart docker

#变量查看，确认是否载入成功
docker info
systemctl status docker

5 使用Kubeadm部署k8s集群

1 首先安装k8s相关软件包。你也可以在阿里云镜像仓库获取相关帮助。

配置如下所示：

vi /etc/yum.repos.d/k8s.repo

[kubernetes]
name=Kubernetes Repository
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
      https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg

2 yum仓库载入情况检查

#yum repolist
[root@centos-1 yum.repos.d]# yum list all |grep "^kube"
kubeadm.x86_64			1.16.3-0
kubectl.x86_64			1.16.3-0
kubelet.x86_64			1.16.3-0
kubernetes.x86_64		1.5.2-0.7.git269f9
kubernetes-client.x86_64	1.5.2-0.7.git269f9
kubernetes-cni.x86_64			0.7.5-0
kubernetes-master.x86_64	1.5.2-0.7.git269f9
kubernetes-node.x86_64		1.5.2-0.7.git269f9

3 使用yum安装kubeadm kubectl和kubelet

yum install -y kubeadm kubectl kubelet

如果阿里云仓库更新了，可以使用以下命令安装指定（v1.16）版本：
#查看相关版本，然后来选择
yum list kubelet --showduplicates | sort -r
yum list kubeadm --showduplicates | sort -r
yum list kubectl --showduplicates | sort -r
#安装，默认最新版
yum install -y kubelet kubeadm kubectl
#安装指定版本
yum install -y kubeadm-1.16.3-0 kubelet-1.16.3-0 kubectl-1.16.3-0

4 检查相关软件包是否完整

[root@centos-1 yum.repos.d]# rpm -ql kubelet 
/etc/kube rnetes/manifests
/etc/sysconfig/kubelet
/usr/bin/kubelet
/usr/lib/systemd/system/kubelet.service

5 配置kubelet。 swap处于启用状态时，不会报错（如果.上面配置了关闭swap，可省略）

vim /etc/sysconfig/kubelet
KUBELET_EXTRA_ARG="--fail-swap-on=false"

6 初始化集群（ Master节点）

#首先将集群初始化配置文件导出
kubeadm config print init-defaults --kubeconfig ClusterConfiguration > kubeadm.yml

#将imageRepository换成阿里云镜像，然后保存
vi kubeadm.yml
imageRepository: registry.aliyuncs.com/google_containers

#查看所需镜像列表
kubeadm config images list --config kubeadm.yml

#拉取镜像，等拉取完毕再执行后续操作
kubeadm config images pull --config kubeadm.yml

#方式一：命令行（--dry-run： 试运行，不会有改动
kubeadm init --kubernetes-version=v1.17.0 --pod-network-cidr="10.244.0.0/16" --dry-run
#方式二：yml配置文件，使用一config string

#集群初始化完毕后，创建用户（最好用普通账号创建） ：
mkdir -p $HOME/.kube
#切换至Root用户操作：
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config（主配置文件，至关重要，不能泄露）
sudo chown $(id -u):$(id -g) $HOME/.kube/config

#集成flannel网络插件，并观察
kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

#Pod情况查看：
kubectl get pods -n kube-system

#集群初始化完毕
kubectl get nodes

#!/bin/bash
yum -y install docker-ce

mkdir -p /etc/docker

echo '{"exec-opts": ["native.cgroupdriver=systemd"],"registry-mirrors": ["https://eyg9yi6d.mirror.aliyuncs.com"]}' > /etc/docker/daemon.json

systemctl  daemon-reload

yum install -y wget

echo '[k8s]
name=aliyun_k8s
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
#这里，如果要使用key认证，配置如下
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
#保存退出后，需要下载key然后导入
wget  https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
wget https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
rpm --import yum-key.gpg
rpm --import rpm-package-key.gpg' > /etc/yum.repos.d/k8s.repo

cd 

wget https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
rpm --import yum-key.gpg

wget https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
rpm --import rpm-package-key.gpg

yum -y install kubeadm kubelet kubectl

mkdir -p /etc/systemd/system/kubelet.service.d
echo 'Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"' > /etc/systemd/system/kubelet.service.d/10-kubeadm.conf

systemctl start kubelet

systemctl enable kubelet.service

kubeadm init --kubernetes-version=v1.17.0 --pod-network-cidr=10.244.0.0/16 --service-cidr=10.96.0.0/12

#!/bin/bash
docker tag registry.aliyuncs.com/google_containers/kube-proxy:v1.17.0 k8s.gcr.io/kube-proxy:v1.17.3             
docker tag registry.aliyuncs.com/google_containers/kube-apiserver:v1.17.0 k8s.gcr.io/kube-apiserver:v1.17.3       
docker tag registry.aliyuncs.com/google_containers/kube-controller-manager:v1.17.0 k8s.gcr.io/kube-controller-manager:v1.17.3  
docker tag registry.aliyuncs.com/google_containers/kube-scheduler:v1.17.0 k8s.gcr.io/kube-scheduler:v1.17.3   
docker tag registry.aliyuncs.com/google_containers/coredns:1.6.5 k8s.gcr.io/coredns:1.6.5            
docker tag registry.aliyuncs.com/google_containers/etcd:3.4.3-0 k8s.gcr.io/etcd:3.4.3-0             
docker tag registry.aliyuncs.com/google_containers/pause:3.1 k8s.gcr.io/pause:3.1

第一步，查看系统配置，至少双核，2g内存

#查看核心数，内存大小
grep 'physical id' /proc/cpuinfo | sort -u | wc -l
grep MemTotal /proc/meminfo
#关闭防火墙
sed -i '/SELINUX/s/enforcing/disabled/g' /etc/selinux/config
systemctl disable firewalld && systemctl stop firewalld
#配置转发规则
cat <<EOF > /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1  
net.bridge.bridge-nf-call-iptables = 1 
vm.swappiness=0
EOF

modprobe br_netfilter
sysctl --system

#启用ipvs内核模块 创建内核模块载入相关的脚本文件/etc/sysconfig/modules/ipvs.modules
vi /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
ipvs_mods_dir="/usr/lib/modules/$(uname -r)/kernel/net/netfilter/ipvs"
for i in $(ls $ipvs_mods_dir | grep -o "^[^.]*"); do
	/sbin/modinfo -F filename $i &> /dev/null
	if [ $? -eq 0 ]; then
		/sbin/modprobe $i
	fi
done

#赋权、运行并检查
chmod +x /etc/sysconfig/modules/ipvs.modules
/etc/sysconfig/modules/ipvs.modules
lsmod |grep ip_vs

#禁用swap
vi /etc/fstab
#/dev/mapper/VolGroup-lv_swap swap 

#安装chrony
yum install -y wget chrony
#配置master的chrony的配置文件(/etc/chrony.conf)
vi /etc/chrony.conf
#注释掉所有的server添加如下
server cn.pool.ntp.org iburst
#master开启chrony
systemctl enable chronyd.service
systemctl restart chronyd.service
systemctl status chronyd.service
#master校验
chronyc sources -v

第二步，配置软件包，安装docker，k8s

#查看firewalld/swap禁用情况
free -m
systemctl status firewalld

#docker源
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
#安装docker
yum install -y docker-ce && systemctl start docker && systemctl enable docker
#docker加速
cat <<EOF > /etc/docker/daemon.json
{
	"exec-opts": ["native.cgroupdriver=systemd"],
  "registry-mirrors": ["https://registry.docker-cn.com"]
}
EOF
systemctl restart docker && systemctl status docker

#k8s源
tee /etc/yum.repos.d/k8s.repo <<EOF
[kubernetes]
name=Kubernetes Repository
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
      https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
EOF

yum repolist
yum list all |grep "^kube"

yum install -y kubelet kubeadm kubectl 

rpm -ql kubelet
10./etc/kubernetes/manifests
11./etc/sysconfig/kubelet
12./usr/bin/kubelet
13./usr/lib/systemd/system/kubelet.service

#配置kubeadm
echo 'KUBELET_EXTRA_ARGS="--fail-swap-on=false"' >> /etc/sysconfig/kubelet

docker info| grep Cgroup
mkdir -p /etc/systemd/system/kubelet.service.d
vi  /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
#Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd"

#开启kubelet
systemctl enable kubelet && systemctl start kubelet && systemctl status kubelet

new

swapoff -a

systemctl stop firewalld
systemctl disable firewalld

wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

yum -y install docker-ce

mkdir -p /etc/docker && vi /etc/docker/daemon.json
{
 "exec-opts": ["native.cgroupdriver=systemd"],
 "registry-mirrors": ["https://registry.docker-cn.com"]
}

#启动docker-ce
systemctl restart docker
#设置开机启动
systemctl enable docker.service
#确保如下两个参数值为1，默认为1。
cat /proc/sys/net/bridge/bridge-nf-call-ip6tables
cat /proc/sys/net/bridge/bridge-nf-call-iptables

#####################安装k8s
tee /etc/yum.repos.d/k8s.repo <<EOF
[kubernetes]
name=Kubernetes Repository
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
      https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
EOF

vi /etc/yum.repos.d/k8s.repo
[k8s]
name=aliyun_k8s
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enable=1
#这里，如果要使用key认证，配置如下
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
#保存退出后，需要下载key然后导入
wget  https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
rpm --import yum-key.gpg rpm-package-key.gpg

docker info| grep Cgroup
mkdir /etc/systemd/system/kubelet.service.d && vi /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
#Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=cgroupfs"
Environment="KUBELET_CGROUP_ARGS=--cgroup-driver=systemd"

yum -y install kubeadm kubelet kubectl
systemctl start kubelet && systemctl status kubelet