k8s paas部署

服务器	角色
192.168.80.153	mster1、etcd1、docker、 flannel、 harbor
192.168.80.145	mster2、etcd2、docker、flannel
192.168.80.144	mster3、etcd3、docker、 flannel
192.168.80.154	nod1、docker、flannel、nginx、keepalived
192.168.80.151	nod2、docker、flannel、nginx、keepalived

1. 安装前准备

1.1 centos7 关闭SElinux

   sudo vim /etc/selinux/config
    ~]# sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
    将SELINUX=enforcing改为SELINUX=disabled
    关闭防火墙
    # systemctl stop firewalld.service
    设置后需要重启才能生效

1.2 linux修改文件打开最大句柄数

sudo vim /etc/security/limits.conf
添加
* soft nofile 65535
* hard nofile 65535
~]# sed -i -e '61a\* soft nofile 65535' -i -e'61a\* hard nofile 65535' /etc/security/limits.conf && cat /etc/security/limits.conf
修改以后保存，注销当前用户，重新登录，执行ulimit -a ,ok 

1.3 linux关闭swap

确认方式：
fdisk -l
1、先停止swap分区
/sbin/swapoff /dev/sdb2
2、删除自动挂载配置命令
vi /etc/fstab
这行删除
/dev/sdb2 swap swap defaults 0 0

sudo swapoff -a

1.4 centos7 升级内核

sudo rpm --import RPM-GPG-KEY-elrepo.org
sudo rpm -ivh kernel-lt-4.4.103-1.el7.elrepo.x86_64.rpm

cat /etc/default/grub && echo '##############################' && sed -i 's/GRUB_DEFAULT=saved/GRUB_DEFAULT=0/g' /etc/default/grub && cat /etc/default/grub
sudo vim /etc/default/grub
这行修改
GRUB_DEFAULT=0 //需要修改 
sudo grub2-mkconfig -o /boot/grub2/grub.cfg
sudo reboot
uname -r

2 部署etcd集群

2.1 先用yum安装
 #yum -y install etcd3

2.2 修改配置文件

# mv /etc/etcd/etcd.conf /etc/etcd/etcd.conf-bak
# vi /etc/etcd/etcd.conf
ETCD_NAME=etcd1
ETCD_DATA_DIR="/var/lib/etcd/etcd1.etcd"
ETCD_LISTEN_PEER_URLS="http://***192.168.56.11***:2380"
ETCD_LISTEN_CLIENT_URLS="http://***192.168.56.11***:2379,http://127.0.0.1:2379"
ETCD_INITIAL_ADVERTISE_PEER_URLS="http://***192.168.56.11***:2380"
ETCD_INITIAL_CLUSTER="etcd1=http://192.168.56.11:2380,etcd2=http://192.168.56.12:2380,etcd3=http://192.168.56.13:2380"
ETCD_INITIAL_CLUSTER_STATE="new"
ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"
ETCD_ADVERTISE_CLIENT_URLS="http://***192.168.56.11***:2379"

2.3 三台etcd的安装相同，注意修改配置文件中的ip。

2.4 启动etcd

分别启动 所有master节点的 etcd 服务
# systemctl daemon-reload
# systemctl enable etcd
# systemctl restart etcd
# systemctl status etc

查看 etcd 集群状态：
# etcdctl cluster-health
# 出现 cluster is healthy 表示成功

查看 etcd 集群成员：
# etcdctl member list

3 安装flannel

3.1 部署flannel

tar -zxf flannel-v0.9.1-linux-amd64.tar.gz
mv flanneld /usr/local/bin/

vi /etc/systemd/system/flanneld.service

[Unit]
Description=flanneld
Before=docker.service
After=network.target

[Service]
User=root
Type=notify
ExecStart=/usr/local/bin/flanneld \
--etcd-endpoints=http://etcd1:2379,etcd2:2379,http://etcd3:2379 \
--etcd-prefix=/flannel/network
ExecStop=/bin/pkill flanneld
Restart=on-failure

[Install]
WantedBy=multi-user.target

3.2 验证flanneld是否部署成功

# systemctl daemon-reload
# systemctl start flanneld
# systemctl enable flanneld
# systemctl status flanneld

4 安装docker

4.1 # yum install docker-ce

# mkdir /etc/docker

下面操作主要是创建harbor仓库，后期可以自动拖镜像，文件中的ip是部署harbor的ip，注意修改！
 # vi /etc/docker/daemon.json

{
        "log-driver": "journald",
        "data-root": "/apps/container_storage",
        "insecure-registries": [
        "hub.paas",
        "10.145.131.252",
        "hub.paas:80",
        "10.145.131.252:80"
        ]
}
overlay2
{
        "storage-driver": "overlay2",
        "storage-opts": "overlay2.override_kernel_check=true",
        "log-driver": "journald",
        "data-root": "/apps/container_storage",
        "insecure-registries": [
        "hub.paas",
        "10.145.131.252",
        "hub.paas:80",
        "10.145.131.252:80"
        ]
}

4.2 关于部署后docker的告警的处理方式

关于分区格式参数的告警
WARNING: overlay2: the backing xfs filesystem is formatted without d_type support, which leads to incorrect behavior.
         Reformat the filesystem with ftype=1 to enable d_type support.
         Running without d_type support will not be supported in future releases.
需要重新格式化
mkfs.xfs -n ftype=1 /dev/mapper/vg02-lv_data

4.3 关联docker和flannel

# vi /usr/lib/systemd/system/docker.service
（注释：只需要改动docker.service文件中两个部分，添加了一行和追加了一行）
[Service]
Type=notify
# the default is not to use systemd for cgroups because the delegate issues still
# exists and systemd currently does not support the cgroup feature set required
# for containers run by docker
EnvironmentFile=/run/flannel/subnet.env    #添加了此行
ExecStart=/usr/bin/dockerd --bip=${FLANNEL_SUBNET} --mtu=${FLANNEL_MTU}  # 后面的是追加了两个环境变量
...

4.4 启动服务

# systemctl daemon-reload
# systemctl start docker
# systemctl enable docker
# systemctl status docker

5、安装tengine

5.1.依赖包安装

# yum -y install gcc gcc-c++ openssl-devel pcre-devel zlib-devel bzip2
# cd /usr/local/src
# 下载安装 jemalloc
wget https://github.com/jemalloc/jemalloc/releases/download/4.4.0/jemalloc-4.4.0.tar.bz2
# tar jxvf jemalloc-4.4.0.tar.bz2
# cd jemalloc-4.4.0
# ./configure && make && make install
# echo '/usr/local/lib' > /etc/ld.so.conf.d/local.conf
# ldconfig
# 下载解压 OpenSSL
wget https://www.openssl.org/source/openssl-1.0.2j.tar.gz
tar zxvf openssl-1.0.2j.tar.gz
# 下载解压 pcre
wget ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/pcre-8.40.tar.gz
tar zxvf pcre-8.40.tar.gz
# 下载 zlib
wget https://ncu.dl.sourceforge.net/project/libpng/zlib/1.2.11/zlib-1.2.11.tar.gz
tar zxvf zlib-1.2.11.tar.gz
# 创建www用户和组，创建www虚拟主机使用的目录
# groupadd www
# useradd -g www www -s /sbin/nologin
# mkdir -p /data/www
# chmod +w /data/www
# chown -R www:www /data/www

5.2 编译安装tengine

# cd /usr/local/src
# wget http://tengine.taobao.org/download/tengine-2.2.0.tar.gz
# tar -zxvf tengine-2.2.0.tar.gz
# cd tengine-2.2.0
# ./configure --prefix=/usr/local/nginx \
--user=tengine --group=tengine \
--conf-path=/usr/local/nginx/conf/nginx.conf \
--error-log-path=/var/log/nginx/error.log \
--http-log-path=/var/log/nginx/access.log \
--pid-path=/var/run/nginx.pid \
--lock-path=/var/run/nginx.lock \
--with-http_ssl_module \
--with-http_flv_module \
--with-http_concat_module \
--with-http_realip_module \
--with-http_addition_module \
--with-http_gzip_static_module \
--with-http_random_index_module \
--with-http_stub_status_module \
--with-http_sub_module \
--with-http_dav_module \
--http-client-body-temp-path=/var/tmp/nginx/client/ \
--http-proxy-temp-path=/var/tmp/nginx/proxy/ \
--http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \
--http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \
--http-scgi-temp-path=/var/tmp/nginx/scgi \
--with-jemalloc --with-openssl=/usr/local/src/openssl-1.0.2j \
--with-zlib=/usr/local/src/zlib-1.2.11 \
--with-pcre=/usr/local/src/pcre-8.40
编译过程略......

# make && make install

5.3 创建/etc/init.d/nginx文件

vim /etc/init.d/nginx

#!/bin/bash
#
# chkconfig: - 85 15
# description: nginx is a World Wide Web server. It is used to serve
# Source function library.
. /etc/rc.d/init.d/functions
 
# Source networking configuration.
. /etc/sysconfig/network
 
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
 
nginx="/usr/tengine-2.2/sbin/nginx" #修改为自己的安装目录
prog=$(basename $nginx)
 
NGINX_CONF_FILE="/usr/tengine-2.2/conf/nginx.conf" #修改为自己的安装目录
 
#[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
 
lockfile=/var/lock/subsys/nginx
 
#make_dirs() {
#   # make required directories
#   user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`
#   options=`$nginx -V 2>&1 | grep 'configure arguments:'`
#   for opt in $options; do
#       if [ `echo $opt | grep '.*-temp-path'` ]; then
#           value=`echo $opt | cut -d "=" -f 2`
#           if [ ! -d "$value" ]; then
#               # echo "creating" $value
#               mkdir -p $value && chown -R $user $value
#           fi
#       fi
#   done
#}
 
start() {
    [ -x $