#无网环境下高可用openshift311 搭建记录
一、主机资源准备:
主机 | ip |
---|---|
master1 | 192.168.100.2 |
master2 | 192.168.100.3 |
master3 | 192.168.100.4 |
router1 | 192.168.100.5 |
router2 | 192.168.100.6 |
infra1 | 192.168.100.7 |
infra2 | 192.168.100.8 |
ansible | 192.168.100.100 |
dns | 192.168.100.100 |
harbor | 192.168.100.100 |
二、ansible机器操作:
1、搭建harbor仓库
假设搭建好的harbor仓库地址为harbor.test.com
2、镜像下载(无网络环境下。可也配置代理,或者从有网环境下载镜像)
#镜像列表(主节点)
echo ‘docker.io/cockpit/kubernetes:latest
docker.io/openshift/origin-haproxy-router:v3.11
docker.io/openshift/origin-haproxy-router:v3.11.0
docker.io/openshift/origin-service-catalog:v3.11
docker.io/openshift/origin-service-catalog:v3.11.0
docker.io/openshift/origin-node:v3.11
docker.io/openshift/origin-node:v3.11.0
docker.io/openshift/origin-deployer:v3.11
docker.io/openshift/origin-node:v3.11.0
docker.io/openshift/origin-control-plane:v3.11
docker.io/openshift/origin-control-plane:v3.11.0
docker.io/openshift/origin-pod:v3.11
docker.io/openshift/origin-web-console:3.11
quay.io/coreos/etcd’ >image.txt &&
while read line; do docker pull $line ; done<image.txt
3、镜像打上tag后上传harbor仓库
while read line;do docker tag $line harbor.test.com/$line;done < image.txt
4、yum源配置(主要为了配置openshift的源,下面是用nexus搭建的私有的yum源)
[root@openshift311-tower ~]# cat /etc/yum.repos.d/nexus.repo
[base]
name=CentOS-$releasever - Base
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/centos/$releasever/os/$basearch/
enabled=1
gpgcheck=0
[updates]
name=CentOS-$releasever - Updates
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/centos/$releasever/updates/$basearch/
enabled=1
gpgcheck=0
[extras]
name=CentOS-$releasever - Extras
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/centos/$releasever/extras/$basearch/
enabled=1
gpgcheck=0
[epel]
name=CentOS-$releasever - Epel
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/epel/$releasever/$basearch/
enabled=1
gpgcheck=0
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/docker-ce/linux/centos/7/$basearch/stable
enabled=1
gpgcheck=0
[kubernetes]
name=Kubernetes
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
[openshift]
name=Openshift
baseurl=http://nexus3-cicd.apps.test.openshift.com/repository/yum/centos/$releasever/paas/$basearch/openshift-origin311/
gpgcheck=0
5、ansible安装(建议2.6.8版本,便于输出日志)
ansible下载地址:ftp://ftp.pbone.net/mirror/ftp.redhat.com/pub/redhat/linux/enterprise/7Server/en/Ansible/SRPMS/ansible-2.6.8-1.el7ae.src.rpm
安装: yum install ansible-2.6.8-1.el7ae.src.rpm
6、配置ansible主机能够免密登录其他主机
创建ssh秘钥对,并将弓腰复制到其他需要免密登陆的主机上
ssh-keygen -t rsa
sh-copy-id -i ~/.ssh/id_rsa.pub root@master/router/infra
7、搭建dns服务器(使用dnsmasq)
dnsmasq配置文件目录:/etc/dnsmasq.conf
dnsmasq配置文件路径:可也看dnsmasq.conf中的conf-dir配置
yum install dnsmasq
systemctl start dnsmasq
systemctl enable dnsmasq
cd /etc/dnsmasq.d
touch host.conf
[root@NewTower dnsmasq.d]# cat hosts.conf
address=/master1.test.openshift.com/192.168.100.2
address=/master2.test.openshift.com/192.168.100.3
address=/master3.test.openshift.com/192.168.100.4
address=/router1.test.openshift.com/192.168.100.5
address=/router2.test.openshift.com/192.168.100.6
address=/node1.test.openshift.com/192.168.100.7
address=/node2.test.openshift.com/192.168.100.8
address=/master.test.openshift.com/192.168.100.100
address=/.apps.test.openshift.com/192.168.100.100
address=/harbor.test.com/192.168.100.100
systemctl restart dnsmasq
配置好conf后重启
8、配置好dns服务后,配置resolve.conf文件(解析test.openshift.com)
cat /etc/resolv.conf
search test.openshift.com
nameserver 192.168.100.100
9、配置ansible inventory(参考https://github.com/openshift/openshift-ansible/blob/release-3.11/inventory/hosts.example)
[root@openshift311-tower ~]# cat /etc/ansible/hosts
[OSEv3:children]
#这里放openshfit的角色,这里有三个角色,master,node,etcd
masters
nodes
etcd
[OSEv3:vars]
#这里是openshfit的安装参数
#指定ansible使用ssh的用户为root
ansible_ssh_user=root
#指定方式为origin
openshift_deployment_type=origin
#指定版本为3.11
openshift_release=3.11
#openshift_enable_service_catalog=false
#openshift_clock_enabled=true
#openshift_disable_check=disk_availability,docker_storage,memory_availability,docker_image_availability
# don't install monitor
# openshift_cluster_monitoring_operator_install=false
# install and don't install
openshift_metrics_install_metrics=true
openshift_enable_service_catalog=false
# skip checking docker images & memory
openshift_disable_check=memory_availability,disk_availability,docker_storage_driver,docker_storage,package_availability,package_update,docker_image_availability
# docker registry
oreg_url=harbor.test.com/openshift/origin-${component}:${version}
openshift_examples_modify_imagestreams=true
openshift_docker_additional_registries=harbor.test.com/
openshift_docker_insecure_registries=harbor.test.com/
openshift_docker_options="-l warn --ipv6=false --insecure-registry=0.0.0.0/0 --registry-mirror=http://harbor.test.com/ --log-opt max-size=1M --log-opt max-file=3"
# domain
openshift_master_cluster_hostname=master.lhc.openshift.com
openshift_master_cluster_public_hostname=master.lhc.openshift.com
openshift_master_default_subdomain=apps.lhc.openshift.com
[masters]
master1.lhc.openshift.com
master2.lhc.openshift.com
master3.lhc.openshift.com
[etcd]
master1.lhc.openshift.com
master2.lhc.openshift.com
master3.lhc.openshift.com
[nodes]
#node角色的机器名称包含
#all-in-one openshift_node_group_name='node-config-all-in-one'
#master openshift_node_group_name='node-config-master'
#router openshift_node_group_name='node-config-infra'
#node openshift_node_group_name='node-config-compute'
master1.lhc.openshift.com openshift_node_group_name='node-config-master'
master2.lhc.openshift.com openshift_node_group_name='node-config-master'
master3.lhc.openshift.com openshift_node_group_name='node-config-master'
router1.lhc.openshift.com openshift_node_group_name='node-config-infra'
router2.lhc.openshift.com openshift_node_group_name='node-config-infra'
node1.lhc.openshift.com openshift_node_group_name='node-config-compute'
node2.lhc.openshift.com openshift_node_group_name='node-config-compute'
#gluster[1:6].example.com openshift_node_group_name='node-config-compute-storage'
# openshiftclock_enabled=true
# ansible_service_broker_install=false
10、向其他ansible被管理节点推送resolve.conf文件
ansible all -m copy -a "src=/etc/resolve.conf dest=/etc/resolve.conf backup=yes"
11、向其他ansible被管理节点推送yum源
备份原有repo文件
ansible all -m file -a "path=/etc/yum.repos.d/backup state=directory"
ansible all -m shell -a "mv /etc/yum.repos.d/*.repo /etc/yum.repos.d/backup"
推送yum源文件
ansible all -m copy -a "src=/etc/yum.repos.d/nexus.repo dest=/etc/yum.repos.d"
ansible all -m shell -a "yum clean all &&yum makecache"
12、修改服务器selinux(操作后机器需要重启,否则部署过程中集群可能会死活起不来)
ansible all -m shell -a 'sed -i "s/^SELINUX=disabled/SELINUX=permissive/g" /etc/selinux/config'
ansible all -m shell -a "reboot"
##三、openshfit安装
1、从github上面下载openshift311的安装文件(ansible安装文件)
无网环境可以直接从网上下载
https://github.com/openshift/openshift-ansible/archive/release-3.11.zip
2、上传服务器解压
unzip openshift-ansible-release-3.11.zip
cd openshift-ansible-release-3.11
3、关闭openshift安装文件中的默认设置的yum源(安装过程中会通过ansible推送)
sed -i "s/^enabled=1/enabled=0/g" root/openshift-ansible-release-3.11/roles/openshift_repos/templates/CentOS-OpenShift-Origin311.repo.j2
4、关闭Enable the EPEL repo for installing Ansible(无网环境下需要关闭)
注释掉下面文件内容
vim /root/openshift-ansible-release-3.11/roles/openshift_repos/tasks/centos_repos.yml
#- name: Enable the EPEL repo for installing Ansible
# yum:
# name: "{{ openshift_epel_rpm_url }}"
# state: present
# notify: refresh cache
5、第四步操作后需要手动下载rpm文件(https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm)
epel-release-latest-7.noarch.rpm
ansible all -m copy -a "src=/root/epel-release-latest-7.noarch.rpm dest=/root"
ansible all -m shell -a "rpm -i /root/epel-release-latest-7.noarch.rpm"
6、运行ansible-playbook
#openshift安装环境检查
ansible-playbook playbooks/prerequisites.yml
#openshift部署
ansible-playbook playbooks/deploy_cluster.yml
##四、排错思路:
1、安装过程中有报错的话可以查看节点日志tail -f /var/log/message,集群没起来前主要看master节点。
2、执行部署后如果卡在control-plane这一步的话,可能是镜像的问题(镜像版本不对,具体信息看master节点日志)
3、ansible执行过程中,有文件推送,有可能覆盖掉yum源,或者修改了 /etc/resolv.conf文件内容,这时候可以为resolv.conf文件加锁(ansible all -m shell -a “chattr +i /etc/resolv.conf”)
4、集群部署好之后可能镜像版本各种不对,需要自己看日志,确定使用的那个版本,然后打上对应的tag号