kolla-ansible 部署 openstack

环境：cenotos7，ceph luminous，kolla-ansible-stable-rocky
建议：先熟悉 ansible，便于排错
ps: 这里掠过了 ansible 的节点的一些配置，可以参考最后的官方文档
network 节点 uat-gtw01-03 需要双网卡，第二块网卡 eth1 不能绑定 ip，需要交换机设置 trunck（esxi vm 需要连接到 vlanid 4095 的端口组）
1、编辑 multinode, 删除/注释掉不需要的组件。需要的组件有：mariadb, nova, neutron, keystone, horizon, memcache, rabbitmq, glance, cinder, haproxy, barbican, heat, placement, bifrost, Barbican, ceph-mon

[infrastructure]
uat-ctl01 ansible_host=192.168.1.142 cpu=32 memory=32768  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-ctl02 ansible_host=192.168.1.143 cpu=32 memory=32768  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-ctl03 ansible_host=192.168.1.147 cpu=32 memory=32768  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-gtw01 ansible_host=192.168.1.10  cpu=16 memory=32768  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-gtw02 ansible_host=192.168.1.11  cpu=16 memory=32768  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-gtw03 ansible_host=192.168.1.12  cpu=16 memory=32768  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-prx01 ansible_host=192.168.1.13  cpu=8  memory=16384  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-prx02 ansible_host=192.168.1.14  cpu=8  memory=16384  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-prx03 ansible_host=192.168.1.15  cpu=8  memory=16384  storage=500  api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-mon01 ansible_host=192.168.1.171 cpu=16 memory=16384  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-mon02 ansible_host=192.168.1.172 cpu=16 memory=16384  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
uat-mon03 ansible_host=192.168.1.173 cpu=16 memory=16384  storage=1000 api_interface=eth0 network_interface=eth0 storage_interface=eth0 cluster_interface=eth0 tunnel_interface=eth0 monitor_interface=eth0 radosgw_interface=eth0 neutron_external_interface=eth1
[virtualization]
cmp37 ansible_host=192.168.1.137 api_interface=bond0 network_interface=bond0 storage_interface=bond0 cluster_interface=bond0 tunnel_interface=bond0 monitor_interface=bond0 radosgw_interface=bond0 neutron_external_interface=bond0
cmp38 ansible_host=192.168.1.138 api_interface=bond0 network_interface=bond0 storage_interface=bond0 cluster_interface=bond0 tunnel_interface=bond0 monitor_interface=bond0 radosgw_interface=bond0 neutron_external_interface=bond0
cmp39 ansible_host=192.168.1.139 api_interface=bond0 network_interface=bond0 storage_interface=bond0 cluster_interface=bond0 tunnel_interface=bond0 monitor_interface=bond0 radosgw_interface=bond0 neutron_external_interface=bond0
# These initial groups are the only groups required to be modified. The
# additional groups are for more control of the environment.
[control]
# These hostname must be resolvable from your deployment host
uat-ctl01
uat-ctl02
uat-ctl03
# The network nodes are where your l3-agent and loadbalancers will run
# This can be the same as a host in the control group
[network]
uat-gtw01
uat-gtw02
uat-gtw03
[compute:children]
#inner-compute
#external-compute
virtualization
[monitoring]
uat-mon01
uat-mon02
uat-mon03
# When compute nodes and control nodes use different interfaces,
# you need to comment out "api_interface" and other interfaces from the globals.yml
# and specify like below:
#compute01 neutron_external_interface=eth0 api_interface=em1 storage_interface=em1 tunnel_interface=em1
[storage]
uat-ctl01
uat-ctl02
uat-ctl03
[deployment]
localhost       ansible_connection=local
[baremetal:children]
control
network
compute
storage
monitoring
proxy
[grafana:children]
monitoring
[prometheus:children]
monitoring
[kibana:children]
monitoring
[elasticsearch:children]
monitoring
[proxy]
uat-prx01
uat-prx02
uat-prx03
[haproxy:children]
proxy
[hyperv]
[hyperv:vars]
[mariadb:children]
control
[rabbitmq:children]
control
[outward-rabbitmq:children]
control
[keystone:children]
control
[glance:children]
control
[nova:children]
control
[neutron:children]
network
[openvswitch:children]
network
compute
#manila-share
[cinder:children]
storage
[memcached:children]
control
[horizon:children]
proxy
[swift:children]
storage
[barbican:children]
control
[heat:children]
control
[placement:children]
control
[bifrost:children]
deployment
# Glance
[glance-api:children]
glance
[glance-registry:children]
glance
# Nova
[nova-api:children]
nova
[nova-conductor:children]
nova
[nova-consoleauth:children]
nova
[nova-novncproxy:children]
nova
[nova-scheduler:children]
nova
[nova-spicehtml5proxy:children]
nova
[nova-compute-ironic:children]
nova
[nova-serialproxy:children]
nova
# Neutron
[neutron-server:children]
#control
neutron
[neutron-dhcp-agent:children]
neutron
[neutron-l3-agent:children]
neutron
[neutron-lbaas-agent:children]
neutron
[neutron-metadata-agent:children]
neutron
[neutron-bgp-dragent:children]
neutron
[neutron-infoblox-ipam-agent:children]
neutron
[ironic-neutron-agent:children]
neutron
# Ceph
#[ceph-mds:children]
#ceph
#
#[ceph-mgr:children]
#ceph
#
#[ceph-nfs:children]
#ceph
#
[ceph-mon:children]
#[ceph-rgw:children]
#ceph
#
#[ceph-osd:children]
#storage
# Cinder
[cinder-api:children]
cinder
[cinder-backup:children]
cinder
[cinder-scheduler:children]
cinder
[cinder-volume:children]
cinder
# Swift
[swift-proxy-server:children]
swift
[swift-account-server:children]
swift
[swift-container-server:children]
swift
[swift-object-server:children]
swift
# Barbican
[barbican-api:children]
barbican
[barbican-keystone-listener:children]
barbican
[barbican-worker:children]
barbican
# Heat
[heat-api:children]
heat
[heat-api-cfn:children]
heat
[heat-engine:children]
heat
# Placement
[placement-api:children]
placement
# Prometheus
[prometheus-node-exporter:children]
infrastructure
virtualization
[prometheus-mysqld-exporter:children]
mariadb
[prometheus-haproxy-exporter:children]
haproxy
[prometheus-memcached-exporter:children]
memcached
[prometheus-cadvisor:children]
infrastructure
virtualization
[prometheus-alertmanager:children]
monitoring
[all:vars]
ansible_user=demo
ansible_ssh_pass=123456
ansible_sudo_pass=123456
#ansible_become=yes

2、/etc/kolla/gloabal.yml 中增加 docker 源

docker_new_yum_url: "
http://mirror.yourdocker.com/docker-ce/linux/
{{ ansible_distribution | lower }}"
docker_new_yum_baseurl: "{{ docker_yum_url }}/{{ ansible_distribution_major_version | lower }}/$basearch/stable"
docker_new_yum_gpgkey: "{{ docker_yum_url }}/gpg"

3、每个节点配置好 docker 仓库

[root@cmp37 ~]# cat /etc/docker/daemon.json 
{
    "bip": "172.31.255.1/24",
    "graph": "/var/lib/docker",
    "hosts": [
        "unix:///var/run/docker.sock"
    ],
    "insecure-registries": [
        "192.168.1.11"
    ],
    "log-driver": "json-file",
    "log-level": "info",
    "registry-mirrors": [
        "
https://192.168.1.11"
;
    ],
    "storage-driver": "overlay2",
    "storage-opts": [
        "overlay2.override_kernel_check=true"
    ]
}

4、每个节点配置好 pip 源

[root@cmp37 ~]# cat /root/.pip/pip.conf 
[global]
timeout = 60
index-url = 
http://mirror.yourpip.com/pypi/simple

[install]
trusted-host = mirror.yourpip.com

5、multinode 中增加 proxy，不然执行 prechecks 会 出错，因为 proxy 不在 baremetal 中，会导致没有安装 docker

[baremetal:children]
control
network
compute
storage
monitoring
proxy

6、multinode 中增加 ceph-mon，但是为空，kolla-ansible-stable-rocky\ansible\roles\cinder\tasks\deploy.yml 需要用到 ceph-mon

[ceph-mon:children]

7、如果外部网络需要使用 vlan，那么需要增加配置 network_vlan_ranges = physnet1，可以等部署完成后，手动添加，然后重启 cmp、network 节点上的 neutron 和 openviswitch 。也可以增加配置文件 /etc/kolla/config/neutron/ml2_conf.ini，该文件最后会被 kolla-ansible 模块 merge_configs merge 到相应的 container 配置目录下的 ml2_conf.ini 文件中

[root@ansible002 ansible_scripts]# cat /etc/kolla/config/neutron/ml2_conf.ini 
[ml2_type_vlan]
network_vlan_ranges = physnet1
[ml2_type_flat]
flat_networks = physnet1

8、如果需要使用 “负载均衡”，需要修改配置文件 /etc/kolla/gloabal.yml

enable_neutron_lbaas: "yes"
enable_horizon_neutron_lbaas: "{{ enable_neutron_lbaas | bool }}"

增加 /etc/kolla/horizon/custom_local_settings 配置

[root@ansible002 kolla]# cat horizon/custom_local_settings
OPENSTACK_NEUTRON_NETWORK = {
    'enable_router': True,
    'enable_quotas': True,
    'enable_ipv6': False,
    'enable_distributed_router': False,
    'enable_ha_router': False,
    'enable_lb': True,
    'enable_firewall': True,
    'enable_vpn': False,
    'enable_fip_topology_check': True,
# Default dns servers you would like to use when a subnet is
    # created.  This is only a default, users can still choose a different
    # list of dns servers when creating a new subnet.
    # The entries below are examples only, and are not appropriate for
    # real deployments
    # 'default_dns_nameservers': ["8.8.8.8", "8.8.4.4", "208.67.222.222"],

    # The profile_support option is used to detect if an external router can be
    # configured via the dashboard. When using specific plugins the
    # profile_support can be turned on if needed.
    'profile_support': None,
    #'profile_support': 'cisco',

    # Set which provider network types are supported. Only the network types
    # in this list will be available to choose from when creating a network.
    # Network types include local, flat, vlan, gre, vxlan and geneve.
    'supported_provider_types': ['local','flat','vlan','vxlan'],

    # You can configure available segmentation ID range per network type
    # in your deployment.
    'segmentation_id_range': {
        'vxlan': [4097, 65536],
    },

    # You can define additional provider network types here.
    # 'extra_provider_types': {
    #     'awesome_type': {
    #         'display_name': 'Awesome New Type',
    #         'require_physical_network': False,
    #         'require_segmentation_id': True,
    #     }
    # },

    # Set which VNIC types are supported for port binding. Only the VNIC
    # types in this list will be available to choose from when creating a
     # port.
     # VNIC types include 'normal', 'macvtap' and 'direct'.
     # Set to empty list or None to disable VNIC type selection.
     'supported_vnic_types': ['*'],
 }

生成秘钥

cd kolla-ansible/tools/
./generate_passwords.py

9、基础设置

./kolla-ansible -i ../../multinode bootstrap-servers

10、检查

./kolla-ansible -i ../../multinode prechecks

11、部署

./kolla-ansible -i ../../multinode deploy

12、登陆 openstack dashboard，密码在 /etc/kolal/passwords.yml 中 keystone_admin_password
13、生成 openstack cli 命令所需要的文件 /etc/kolla/admin-openrc.sh

./kolla-ansible -i ../../multinode post-deploy

14、配置文件 /etc/kolla/gloabal.yml 完整信息

###############
# docker options
###############
docker_new_yum_url: "
http://mirror.yourdocker.com/docker-ce/linux/
{{ ansible_distribution | lower }}"
docker_new_yum_baseurl: "{{ docker_yum_url }}/{{ ansible_distribution_major_version | lower }}/$basearch/stable"
docker_new_yum_gpgkey: "{{ docker_yum_url }}/gpg"
kolla_base_distro: "centos"
# Valid options are [ binary, source ]
kolla_install_type: "binary"
# Valid option is Docker repository tag
openstack_release: "rocky"
kolla_internal_vip_address: "192.168.1.6"
neutron_plugin_agent: "openvswitch"
enable_ceph: "no"
enable_cinder: "yes"
enable_horizon_neutron_lbaas: "{{ enable_neutron_lbaas | bool }}"
enable_neutron_lbaas: "yes"
glance_backend_ceph: "yes"
glance_enable_rolling_upgrade: "no"
cinder_backend_ceph: "yes"
nova_backend_ceph: "yes"
enable_fluentd: "no"
ironic_dnsmasq_dhcp_range:
tempest_image_id:
tempest_flavor_ref_id:
tempest_public_network_id:
tempest_floating_network_name:

15、如果需要使用 dvr 模式，需要在 global.yml 中设置参数 enable_neutron_dvr，multinode 中需要 [inner-compute] 和 [external-compute]，可以没有 children。[virtualization] 需要设置 neutron_external_interface，neutron_external_interface 网口设置成 trunck，不绑定 ip

cat /etc/kolla/global.yml 
...
enable_neutron_dvr: "yes"
...

cat multinode
...
cmp37 ansible_host=192.168.1.137 api_interface=eno49 network_interface=eno49 storage_interface=eno49 cluster_interface=eno49 tunnel_interface=eno49 monitor_interface=eno49 radosgw_interface=eno49 neutron_external_interface=eno50
cmp38 ansible_host=192.168.1.138 api_interface=eno49 network_interface=eno49 storage_interface=eno49 cluster_interface=eno49 tunnel_interface=eno49 monitor_interface=eno49 radosgw_interface=eno49 neutron_external_interface=eno50
cmp39 ansible_host=192.168.1.139 api_interface=eno49 network_interface=eno49 storage_interface=eno49 cluster_interface=eno49 tunnel_interface=eno49 monitor_interface=eno49 radosgw_interface=eno49 neutron_external_interface=eno50
...
[inner-compute]
[external-compute]
...

参考文章
Quick Start