使用天翼云部署Openshift3.10集群

  本次部署采用了openshift的高级部署形式，使用ansible脚本完成，共需要4台云主机，三台高配置的云主机构建一个openshift单集群，一台普通配置云主机做openshift集群nfs存储节点。

云主机具体规划如下：

安装规划

ip	hostname	操作系统	内存	数据盘
192.168.1.100	master.xykz.com master	centos7.3	64G	1000G
192.168.1.55	node1.xykz.com node1	centos7.3	64G	1000G
192.168.1.208	node2.xykz.com node2	centos7.3	64G	1000G
192.168.1.74	nfs2.xykz.com  nfs2	centos7.3	64G	1000G

一、分别设置云主机名与修改/etc/hosts

hostnamectl set-hostname master.xykz.com

echo "192.168.1.100 master.xykz.com master" >> /etc/hosts
echo "192.168.1.55  node1.xykz.com node1" >> /etc/hosts
echo "192.168.1.208 node2.xykz.com node2" >> /etc/hosts
echo "192.168.1.74  nfs2.xykz.com  nfs2" >> /etc/hosts

--master主机设置完毕

hostnamectl set-hostname node1.xykz.com

echo "192.168.1.100 master.xykz.com master" >> /etc/hosts
echo "192.168.1.55  node1.xykz.com node1" >> /etc/hosts
echo "192.168.1.208 node2.xykz.com node2" >> /etc/hosts
echo "192.168.1.74  nfs2.xykz.com  nfs2" >> /etc/hosts

--node1主机设置完毕
hostnamectl set-hostname node2.xykz.com

echo "192.168.1.100 master.xykz.com master" >> /etc/hosts
echo "192.168.1.55  node1.xykz.com node1" >> /etc/hosts
echo "192.168.1.208 node2.xykz.com node2" >> /etc/hosts
echo "192.168.1.74  nfs2.xykz.com  nfs2" >> /etc/hosts

--node2主机设置完毕
hostnamectl set-hostname nfs2.xykz.com

echo "192.168.1.100 master.xykz.com master" >> /etc/hosts
echo "192.168.1.55  node1.xykz.com node1" >> /etc/hosts
echo "192.168.1.208 node2.xykz.com node2" >> /etc/hosts
echo "192.168.1.74  nfs2.xykz.com  nfs2" >> /etc/hosts

nfs服务器设置完毕

二、所有节点修改设置云主机ssh自动连接超时

sed -i 's/TMOUT=1800/#TMOUT=1800/g' /etc/profile
source /etc/profile

三、所有节点切换镜像到阿里云

mv /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.backup
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
yum makecache

四、所有节点修改selinux，需要开启，天翼云镜像默认关闭

sed -i 's/SELINUX=disabled/SELINUX=enforcing/g' /etc/selinux/config
reboot

sestatus

五、在所有节点安装依赖软件包

yum install wget git yum-utils net-tools bind-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct bash-completion.noarch bash-completion-extras.noarch java-1.8.0-openjdk-headless python-passlib NetworkManager -y

安装完毕后顺便对系统镜像进行升级

yum update -y
reboot
#升级后重启机器

六、在mater节点上生成rsa密钥，将生成的密钥拷贝至包括自己在内的节点

 ssh-keygen -t rsa
 #三次回车
 ssh-copy-id master.xykz.com
 ssh-copy-id node1.xykz.com
 ssh-copy-id node2.xykz.com
 ssh-copy-id nfs2.xykz.com

七、集群内除nfs无需安装docker，其余机器均需要安装指定版本的docker

yum install docker-1.13.1 -y

openshift3 依赖docker的版本为1.13.1

八、初始化安装docker的机器存储

vgremove DOCKER -y
fdisk /dev/xvde
echo "d"\n"w"\n >> fdisk.txt
vim fdisk.txt
d
w
fdisk /dev/xvde < fdisk.txt

#清理磁盘数据
wipefs --all /dev/xvde

九、为每台机器的docker配置专属存储

echo DEVS=/dev/xvde >> /etc/sysconfig/docker-storage-setup
echo VG=DOCKER >> /etc/sysconfig/docker-storage-setup
#echo SETUP_LVM_THIN_POOL=yes >> /etc/sysconfig/docker-storage-setup
#定义创建 DATA thin pool 的大小，默认为 VG 的 40%
#echo DATA_SIZE="100%FREE">> /etc/sysconfig/docker-storage-setup

#执行docker存储配置
docker-storage-setup

十、修改docker的镜像为国内加快下载速度

vim /etc/docker/daemon.json
{
  "registry-mirrors": [
    "https://registry.docker-cn.com"
  ]
}

#注意，一定要保证该文件符合 json 规范，否则 Docker 将不能启动。之后重新启动服务。

#启动docker
systemctl enable docker
systemctl start docker

十一、安装ansible扩展库

#3.9必須2.5版本
yum -y install epel-release

禁用扩展库防止后续安装出错

sed -i -e "s/^enabled=1/enabled=0/" /etc/yum.repos.d/epel.repo

十二、安装ansible软件

yum install ansible pyOpenSSL -y

rpm -Uvh  https://releases.ansible.com/ansible/rpm/release/epel-7-x86_64/ansible-2.6.4-1.el7.ans.noarch.rpm

十三、从github上下载openshift已经release版本

#release版本下载汇聚
https://github.com/openshift/openshift-ansible/releases
#3.10.60-1
wget https://github.com/openshift/openshift-ansible/archive/openshift-ansible-3.10.60-1.zip

下载完以后解压到/home下

十四、修改软件源

vim /home/openshift-ansible/roles/openshift_repos/templates/CentOS-OpenShift-Origin310.repo.j2

修改修改CentOS-OpenShift-Origin310.repo.j2中的源修改為國內的，比如阿里雲。
http://mirrors.aliyun.com/centos/7/paas/x86_64/openshift-origin310/

十五、配置nfs节点

#新硬盘分区
fdisk /dev/xvde
#格式化硬盘
mkfs.ext4 /dev/xvde1
#创建目录
mkdir -p /exports/
#挂载硬盘
mount /dev/xvde1 /exports/
#修改fstab文件，保障重启后自动挂载
echo "/dev/xvde1                                /exports                ext4    defaults        1 2" >> /etc/fstab

#默认情况下，SELinux不允许从pod写入远程NFS服务器。NFS卷正确安装，但是只读。
#要在每个节点上启用SELinux写入：
setsebool -P virt_use_nfs 1

十六、在master上配置ansible安装脚本

[root@master home]# vim /etc/ansible/hosts
[OSEv3:children]
masters
nodes
etcd
nfs

[OSEv3:vars]
ansible_ssh_user=root
openshift_deployment_type=origin
openshift_release="3.10"
openshift_pkg_version=-3.10.0
openshift_image_tag=v3.10.0
openshift_install_examples=true

openshift_master_default_subdomain=apps.192.168.1.100.nip.io
openshift_check_min_host_disk_gb=1.5
openshift_check_min_host_memory_gb=1.9
openshift_portal_net=172.30.0.0/16
debug_level=4
openshift_docker_insecure_registries=172.30.0.0/16

#禁止磁盤、內存和鏡像檢查
openshift_disable_check=disk_availability,docker_storage,memory_availability,docker_image_availability
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
openshift_clock_enabled=true
openshift_service_catalog_image_prefix=openshift/origin-
openshift_service_catalog_image_version=latest

openshift_enable_unsupported_configurations=True
openshift_hosted_registry_storage_kind=nfs
openshift_hosted_registry_storage_access_modes=['ReadWriteMany']
openshift_hosted_registry_storage_nfs_directory=/exports
openshift_hosted_registry_storage_nfs_options='*(rw,root_squash)'
openshift_hosted_registry_storage_volume_name=registry
openshift_hosted_registry_storage_volume_size=10Gi

openshift_metrics_storage_kind=nfs
openshift_metrics_storage_access_modes=['ReadWriteOnce']
openshift_metrics_storage_nfs_directory=/exports
openshift_metrics_storage_nfs_options='*(rw,root_squash)'
openshift_metrics_storage_volume_name=metrics
openshift_metrics_storage_volume_size=10Gi
openshift_metrics_storage_labels={'storage': 'metrics'}

openshift_logging_storage_kind=nfs
openshift_logging_storage_access_modes=['ReadWriteOnce']
openshift_logging_storage_nfs_directory=/exports
openshift_logging_storage_nfs_options='*(rw,root_squash)'
openshift_logging_storage_volume_name=logging
openshift_logging_storage_volume_size=10Gi
openshift_logging_storage_labels={'storage': 'logging'}

os_sdn_network_plugin_name=redhat/openshift-ovs-multitenant
openshift_ca_cert_expire_days=3650
openshift_node_cert_expire_days=3650
openshift_master_cert_expire_days=3650
etcd_ca_default_days=3650

# Enable cockpit
osm_use_cockpit=true
# Set cockpit plugins
osm_cockpit_plugins=['cockpit-kubernetes']

openshift_enable_service_catalog=false
template_service_broker_install=false
ansible_service_broker_install=false

[all:vars]
# bootstrap configs
#openshift_master_bootstrap_auto_approve=true
#openshift_master_bootstrap_auto_approver_node_selector={"node-role.kubernetes.io/master":"true"}
#osm_controller_args={"experimental-cluster-signing-duration": ["20m"]}
#osm_default_node_selector="node-role.kubernetes.io/compute=true"

[masters]
master

[etcd]
master

[nfs]
nfs2

[nodes]
master openshift_schedulable=True openshift_node_group_name="node-config-master-infra"
node1  openshift_schedulable=True openshift_node_group_name="node-config-infra" openshift_node_group_name="node-config-compute"
node2  openshift_schedulable=True openshift_node_group_name="node-config-infra" openshift_node_group_name="node-config-compute"

十七、通过ansible自动安装

#安装前检查与环境设置
ansible-playbook /home/openshift-ansible/playbooks/prerequisites.yml

#正式安装
ansible-playbook /home/openshift-ansible/playbooks/deploy_cluster.yml 

十八、安装完成后创建用户

#master节点下创建用户dev
htpasswd -bc /etc/origin/master/htpasswd dev dev

#查看所有节点状态

十九、在nfs节点上创建目录并赋予权限

for i in $(seq 1 20); do mkdir -p /exports/pv$i ; done;
chown -R nfsnobody. /exports/pv*
chmod -R 777 /exports/pv*

ls /exports/|xargs -i echo "/exports/{}" *(rw,root_squash) >> /etc/exports.d/openshift-ansible.exports

重启服务让nfs配置生效

systemctl restart rpcbind nfs-server

二十、在master节点上制作pv模板，配合nfs配置生成20个pv

cat >/tmp/pv.temp.yaml<<EOF
apiVersion: v1
kind: PersistentVolume
metadata:
  name: \$pvi
spec:
  capacity:
    storage: 5Gi
  accessModes:
  - ReadWriteOnce
  nfs:
    path: /exports/\$pvi
    server: nfs2
  persistentVolumeReclaimPolicy: Recycle
EOF

#添加pv
for i in $(seq 1 20); do export pvi=pv$i; cat /tmp/pv.temp.yaml|envsubst|oc create -f - ;done;