From 55bfbdbd93882fbb98df706f2a70ceabc08e2e9e Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 21 Nov 2018 14:29:01 +0000 Subject: [PATCH] Implement AIO changes to allow test usage by roles In this patch we change the 'scenario' for the AIO bootstrap to allow it to be overloaded with any additional service names which correspond to an appropriate conf.d AIO configuration file. This allows us to implement jobs using a scenario called something like 'aio_lxc_octavia' and it will implement the base AIO and add the octavia service. In order to make use of this for the experimental jobs, we change the job names so that the appropriate services are deployed. Related-bug: #1759784 Change-Id: If6b7e401021ea57a908a616ec8b86e7aa6798558 (cherry picked from commit ada265c8449334538a011300785634774787d54d) --- doc/source/user/aio/quickstart.rst | 31 +++++-- .../openstack_user_config.yml.aio.j2 | 12 +-- tests/bootstrap-aio.yml | 1 - tests/roles/bootstrap-host/defaults/main.yml | 19 +---- .../tasks/check-requirements.yml | 23 ++++- .../tasks/prepare_aio_config.yml | 26 ++++-- .../templates/user_variables.aio.yml.j2 | 2 +- tests/roles/bootstrap-host/vars/main.yml | 33 ++++++++ tests/vars/bootstrap-aio-vars.yml | 83 ------------------- zuul.d/jobs.yaml | 44 +++++----- zuul.d/project-templates.yaml | 12 +-- 11 files changed, 134 insertions(+), 152 deletions(-) create mode 100644 tests/roles/bootstrap-host/vars/main.yml delete mode 100644 tests/vars/bootstrap-aio-vars.yml diff --git a/doc/source/user/aio/quickstart.rst b/doc/source/user/aio/quickstart.rst index c902253fcc..f51cdf7ac7 100644 --- a/doc/source/user/aio/quickstart.rst +++ b/doc/source/user/aio/quickstart.rst @@ -204,14 +204,6 @@ executing: # scripts/bootstrap-aio.sh -If you wish to use a different scenario, for example, the Ceph scenario, -execute the following: - -.. code-block:: shell-session - - # export SCENARIO='ceph' - # scripts/bootstrap-aio.sh - To add OpenStack Services over and above the bootstrap-aio default services for the applicable scenario, copy the ``conf.d`` files with the ``.aio`` file extension into ``/etc/openstack_deploy`` and rename then to ``.yml`` files. @@ -224,6 +216,29 @@ following: # cp etc/openstack_deploy/conf.d/{aodh,gnocchi,ceilometer}.yml.aio /etc/openstack_deploy/conf.d/ # for f in $(ls -1 /etc/openstack_deploy/conf.d/*.aio); do mv -v ${f} ${f%.*}; done +It is possible to also do this (and change other defaults) during the bootstrap +script initial execution by changing the SCENARIO environment variable before +running the script. The key word 'aio' will ensure that a basic set of +OpenStack services (cinder, glance, horizon, neutron, nova) will be deployed. +The key words 'lxc' and 'nspawn' can be used to set the container back-end, +while the key word 'metal' will deploy all services without containers. In +order to implement any other services, add the name of the conf.d file name +without the `.yml.aio` extension into the SCENARIO environment variable. Each +key word should be delimited by an underscore. For example, the following will +implement an AIO with barbican, cinder, glance, horizon, neutron, and nova. It +will set the cinder storage back-end to ceph and will make use of LXC as the +container back-end. + +.. code-block:: shell-session + + # export SCENARIO='aio_lxc_barbican' + # scripts/bootstrap-aio.sh + +.. note:: + If the 'metal' and 'aio' key words are used together, horizon will not be + deployed because haproxy and horizon will conflict on the same listening + ports. + To add any global overrides, over and above the defaults for the applicable scenario, edit ``/etc/openstack_deploy/user_variables.yml``. In order to understand the various ways that you can override the default behaviour diff --git a/etc/openstack_deploy/openstack_user_config.yml.aio.j2 b/etc/openstack_deploy/openstack_user_config.yml.aio.j2 index 9a4bd8ac27..fea2b09050 100644 --- a/etc/openstack_deploy/openstack_user_config.yml.aio.j2 +++ b/etc/openstack_deploy/openstack_user_config.yml.aio.j2 @@ -1,9 +1,9 @@ --- cidr_networks: -{% if bootstrap_host_scenario == "translations" %} +{% if 'trove' in bootstrap_host_services %} dbaas: 172.29.232.0/22 {% endif %} -{% if bootstrap_host_scenario in ["octavia"] %} +{% if 'octavia' in bootstrap_host_services %} lbaas: 172.29.252.0/22 {% endif %} container: 172.29.236.0/22 @@ -11,11 +11,11 @@ cidr_networks: storage: 172.29.244.0/22 used_ips: -{% if bootstrap_host_scenario == "translations" %} +{% if 'trove' in bootstrap_host_services %} - "172.29.232.1,172.29.232.50" - "172.29.232.100" {% endif %} -{% if bootstrap_host_scenario in ["translations", "octavia"] %} +{% if 'octavia' in bootstrap_host_services %} - "172.29.252.1,172.29.252.50" - "172.29.252.100" {% endif %} @@ -55,7 +55,7 @@ global_overrides: net_name: "vxlan" group_binds: - neutron_linuxbridge_agent -{% if bootstrap_host_scenario == "translations" %} +{% if 'trove' in bootstrap_host_services %} - network: container_bridge: "br-dbaas" container_type: "veth" @@ -68,7 +68,7 @@ global_overrides: - neutron_linuxbridge_agent - rabbitmq {% endif %} -{% if bootstrap_host_scenario in ["octavia"] %} +{% if 'octavia' in bootstrap_host_services %} - network: container_bridge: "br-lbaas" container_type: "veth" diff --git a/tests/bootstrap-aio.yml b/tests/bootstrap-aio.yml index a6eb612bf2..0f53efef0c 100644 --- a/tests/bootstrap-aio.yml +++ b/tests/bootstrap-aio.yml @@ -22,7 +22,6 @@ - role: "bootstrap-host" vars_files: - "{{ playbook_dir }}/../playbooks/defaults/repo_packages/openstack_services.yml" - - vars/bootstrap-aio-vars.yml environment: "{{ deployment_environment_variables | default({}) }}" vars: ansible_python_interpreter: "/usr/bin/python" diff --git a/tests/roles/bootstrap-host/defaults/main.yml b/tests/roles/bootstrap-host/defaults/main.yml index 6f3ad4c9fb..7df3f1e0a8 100644 --- a/tests/roles/bootstrap-host/defaults/main.yml +++ b/tests/roles/bootstrap-host/defaults/main.yml @@ -40,23 +40,6 @@ bootstrap_host_target_config_paths: # The user variables template to use bootstrap_user_variables_template: user_variables.aio.yml.j2 -# Extra user variables files can be loaded into /etc/openstack_deploy by -# test scenarios. The dict uses scenario as the key to load a list of extra -# templates if necessary. -bootstrap_user_variables_extra_templates: - ceph: - - src: user_variables_ceph.yml.j2 - dest: user_variables_ceph.yml - congress: - - src: user_variables_congress.yml.j2 - dest: user_variables_congress.yml - translations: - - src: user_variables_translations.yml.j2 - dest: user_variables_translations.yml - barbican: - - src: user_variables_barbican.yml.j2 - dest: user_variables_barbican.yml - ## Loopback volumes # Sparse loopback disks are used for the containers if there is no secondary # disk available to partition for btrfs. They are also used for Ceph, Cinder, @@ -99,7 +82,7 @@ bootstrap_host_loopback_zfs: yes bootstrap_host_loopback_zfs_size: 1024 # # Boolean option to deploy the OSD loopback disks and cluster UUID for Ceph -bootstrap_host_ceph: "{{ (bootstrap_host_scenario == 'ceph') | bool }}" +bootstrap_host_ceph: "{{ 'ceph' in bootstrap_host_services }}" # Size of the Ceph OSD loopbacks bootstrap_host_loopback_ceph_size: 1024 # Ceph OSDs to create on the AIO host diff --git a/tests/roles/bootstrap-host/tasks/check-requirements.yml b/tests/roles/bootstrap-host/tasks/check-requirements.yml index 8bf577a09f..986a8b7cd8 100644 --- a/tests/roles/bootstrap-host/tasks/check-requirements.yml +++ b/tests/roles/bootstrap-host/tasks/check-requirements.yml @@ -20,11 +20,32 @@ (ansible_distribution == 'Ubuntu' and ansible_distribution_release == 'bionic') or (ansible_os_family == 'RedHat' and ansible_distribution_major_version == '7') or (ansible_os_family == 'Suse' and ansible_distribution_major_version == '42') - msg: "The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial), Ubuntu 18.04 LTS (Bionic), CentOS 7 (WIP) and openSUSE Leap 42.X (WIP)" + msg: >- + The only supported platforms for this release are Ubuntu 16.04 LTS (Xenial), + Ubuntu 18.04 LTS (Bionic), CentOS 7, and openSUSE Leap 42.X when: (check_operating_system | default(True))| bool tags: - check-operating-system +- name: Check that bootstrap_host_scenario is not set with mutually exclusive options + assert: + that: + - not item.conditional + msg: "{{ item.msg }}" + with_items: + - conditional: "{{ (bootstrap_host_scenario is search('distro')) and (bootstrap_host_scenario is search('source')) }}" + msg: "The scenario key words 'distro' and 'source' are mutually exclusive." + - conditional: "{{ (bootstrap_host_scenario is search('lxc')) and (bootstrap_host_scenario is search('nspawn')) }}" + msg: "The scenario key words 'lxc' and 'nspawn' are mutually exclusive." + - conditional: "{{ (bootstrap_host_scenario is search('lxc')) and (bootstrap_host_scenario is search('metal')) }}" + msg: "The scenario key words 'lxc' and 'metal' are mutually exclusive." + - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('nspawn')) }}" + msg: "The scenario key words 'metal' and 'nspawn' are mutually exclusive." + - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('haproxy')) }}" + msg: "The scenario key words 'metal' and 'haproxy' are mutually exclusive." + - conditional: "{{ (bootstrap_host_scenario is search('metal')) and (bootstrap_host_scenario is search('horizon')) }}" + msg: "The scenario key words 'metal' and 'horizon' are mutually exclusive." + - name: Identify the space available in / # NOTE(hwoarang): df does not work reliably on btrfs filesystems # https://btrfs.wiki.kernel.org/index.php/FAQ#How_much_free_space_do_I_have.3F diff --git a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml index cb4142677a..479fefacff 100644 --- a/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml +++ b/tests/roles/bootstrap-host/tasks/prepare_aio_config.yml @@ -23,11 +23,11 @@ - name: Deploy user conf.d configuration config_template: - src: "{{ item.path | default(bootstrap_host_aio_config_path ~ '/conf.d') }}/{{ item.name }}" - dest: "/etc/openstack_deploy/conf.d/{{ item.name | regex_replace('.aio$', '') }}" + src: "{{ bootstrap_host_aio_config_path }}/conf.d/{{ item }}.yml.aio" + dest: "/etc/openstack_deploy/conf.d/{{ item }}.yml" config_overrides: "{{ item.override | default({}) }}" config_type: "yaml" - with_items: "{{ confd_overrides[bootstrap_host_scenario] | default([]) }}" + with_items: "{{ bootstrap_host_services }}" tags: - deploy-confd @@ -175,21 +175,35 @@ dest: "/etc/openstack_deploy/{{ item.dest }}" config_overrides: "{{ item.config_overrides | default({}) }}" config_type: yaml - with_items: "{{ bootstrap_user_variables_extra_templates[bootstrap_host_scenario] | default([]) }}" + when: + - "item.condition | bool" + with_items: + - src: user_variables_ceph.yml.j2 + dest: user_variables_ceph.yml + condition: "{{ 'ceph' in bootstrap_host_services }}" + - src: user_variables_congress.yml.j2 + dest: user_variables_congress.yml + condition: "{{ 'congress' in bootstrap_host_services }}" + - src: user_variables_translations.yml.j2 + dest: user_variables_translations.yml + condition: "{{ bootstrap_host_scenario is search('translations') }}" + - src: user_variables_barbican.yml.j2 + dest: user_variables_barbican.yml + condition: "{{ 'barbican' in bootstrap_host_services }}" - name: Copy modified cinder-volume env.d file for ceph scenario copy: src: "{{ playbook_dir }}/../etc/openstack_deploy/env.d/cinder-volume.yml.container.example" dest: "/etc/openstack_deploy/env.d/cinder-volume.yml" when: - - "bootstrap_host_scenario == 'ceph'" + - "'ceph' in bootstrap_host_services" - name: Copy modified env.d file for metal scenario copy: src: "{{ playbook_dir }}/../etc/openstack_deploy/env.d/aio_metal.yml.example" dest: "/etc/openstack_deploy/env.d/aio_metal.yml" when: - - "bootstrap_host_scenario == 'aio_metal'" + - "bootstrap_host_scenario is search('metal')" - name: Create vars override folders if we need to test them file: diff --git a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 index 63bcc4ffdd..f4c8d6814a 100644 --- a/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 +++ b/tests/roles/bootstrap-host/templates/user_variables.aio.yml.j2 @@ -202,7 +202,7 @@ openstack_user_kernel_options: - key: 'kernel.printk' value: '4 1 7 4' -{% if bootstrap_host_scenario in ['octavia'] %} +{% if 'octavia' in bootstrap_host_services %} # Octavia specific stuff neutron_lbaas_octavia: True octavia_management_net_subnet_cidr: "{{ (bootstrap_host_container_tech == 'nspawn') | ternary('172.29.240.0/22', '172.29.252.0/22') }}" diff --git a/tests/roles/bootstrap-host/vars/main.yml b/tests/roles/bootstrap-host/vars/main.yml new file mode 100644 index 0000000000..8769e0f6d3 --- /dev/null +++ b/tests/roles/bootstrap-host/vars/main.yml @@ -0,0 +1,33 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +bootstrap_host_services: >- + {%- set scenario_list = (bootstrap_host_scenario.split('_') | reject('equalto', '')) | list %} + {%- set service_list = ['keystone'] %} + {%- set service_list_extra = scenario_list | difference(['aio', 'distro', 'lxc', 'nspawn', 'metal', 'source', 'translations']) %} + {%- if 'metal' not in scenario_list %} + {%- set _ = service_list.append('haproxy') %} + {%- endif %} + {%- if 'aio' in scenario_list or 'translations' in scenario_list %} + {%- set _ = service_list.extend(['cinder', 'glance', 'neutron', 'nova']) %} + {%- if 'metal' not in scenario_list %} + {%- set _ = service_list.append('horizon') %} + {%- endif %} + {%- endif %} + {%- if 'translations' in scenario_list %} + {%- set _ = service_list.extend(['designate', 'heat', 'magnum', 'sahara', 'swift', 'trove']) %} + {%- endif %} + {%- set _ = service_list.extend(service_list_extra) %} + {{- (service_list | unique) | sort }} diff --git a/tests/vars/bootstrap-aio-vars.yml b/tests/vars/bootstrap-aio-vars.yml deleted file mode 100644 index 87b9466cf8..0000000000 --- a/tests/vars/bootstrap-aio-vars.yml +++ /dev/null @@ -1,83 +0,0 @@ ---- -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -confd_overrides: - aio_lxc: - - name: cinder.yml.aio - - name: glance.yml.aio - - name: haproxy.yml.aio - - name: horizon.yml.aio - - name: keystone.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - aio_metal: - - name: cinder.yml.aio - - name: glance.yml.aio - - name: keystone.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - aio_nspawn: - - name: cinder.yml.aio - - name: glance.yml.aio - - name: haproxy.yml.aio - - name: horizon.yml.aio - - name: keystone.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - ceph: - - name: haproxy.yml.aio - - name: ceph.yml.aio - - name: cinder.yml.aio - - name: glance.yml.aio - - name: keystone.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - congress: - - name: congress.yml.aio - - name: haproxy.yml.aio - - name: glance.yml.aio - - name: keystone.yml.aio - - name: nova.yml.aio - - name: neutron.yml.aio - translations: - - name: cinder.yml.aio - - name: designate.yml.aio - - name: glance.yml.aio - - name: haproxy.yml.aio - - name: heat.yml.aio - - name: horizon.yml.aio - - name: keystone.yml.aio - - name: magnum.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - - name: sahara.yml.aio - - name: swift.yml.aio - - name: trove.yml.aio - octavia: - - name: glance.yml.aio - - name: haproxy.yml.aio - - name: keystone.yml.aio - - name: neutron.yml.aio - - name: nova.yml.aio - - name: octavia.yml.aio - tacker: - - name: haproxy.yml.aio - - name: heat.yml.aio - - name: keystone.yml.aio - - name: tacker.yml.aio - barbican: - - name: haproxy.yml.aio - - name: keystone.yml.aio - - name: barbican.yml.aio diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index d44da82a03..00857de681 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -90,28 +90,28 @@ install_method: distro - job: - name: openstack-ansible-deploy-ceph-ubuntu-bionic + name: openstack-ansible-deploy-aio_ceph-ubuntu-bionic parent: openstack-ansible-deploy-aio nodeset: ubuntu-bionic vars: action: deploy - scenario: ceph + scenario: aio_ceph - job: - name: openstack-ansible-deploy-ceph-ubuntu-xenial + name: openstack-ansible-deploy-aio_ceph-ubuntu-xenial parent: openstack-ansible-deploy-aio nodeset: ubuntu-xenial vars: action: deploy - scenario: ceph + scenario: aio_ceph - job: - name: openstack-ansible-deploy-distro_ceph-ubuntu-bionic + name: openstack-ansible-deploy-aio_distro_ceph-ubuntu-bionic parent: openstack-ansible-deploy-aio nodeset: ubuntu-bionic vars: action: deploy - scenario: ceph + scenario: aio_ceph install_method: distro - job: @@ -147,21 +147,21 @@ scenario: aio_nspawn - job: - name: openstack-ansible-upgrade-ceph-ubuntu-xenial + name: openstack-ansible-upgrade-aio_ceph-ubuntu-xenial parent: openstack-ansible-deploy-aio nodeset: ubuntu-xenial timeout: 10800 vars: action: upgrade - scenario: ceph + scenario: aio_ceph - job: - name: openstack-ansible-deploy-octavia-ubuntu-bionic + name: openstack-ansible-deploy-aio_octavia-ubuntu-bionic parent: openstack-ansible-deploy-aio nodeset: ubuntu-bionic vars: action: deploy - scenario: octavia + scenario: aio_octavia - job: name: openstack-ansible-deploy-translations-ubuntu-bionic @@ -172,12 +172,12 @@ scenario: translations - job: - name: openstack-ansible-deploy-congress-ubuntu-bionic + name: openstack-ansible-deploy-aio_congress-ubuntu-bionic parent: openstack-ansible-deploy-aio nodeset: ubuntu-bionic vars: action: deploy - scenario: congress + scenario: aio_congress - job: name: openstack-ansible-deploy-barbican-ubuntu-bionic @@ -201,20 +201,20 @@ install_method: distro - job: - name: openstack-ansible-deploy-ceph-centos-7 + name: openstack-ansible-deploy-aio_ceph-centos-7 parent: openstack-ansible-deploy-aio nodeset: centos-7 vars: action: deploy - scenario: ceph + scenario: aio_ceph - job: - name: openstack-ansible-deploy-distro_ceph-centos-7 + name: openstack-ansible-deploy-aio_distro_ceph-centos-7 parent: openstack-ansible-deploy-aio nodeset: centos-7 vars: action: deploy - scenario: ceph + scenario: aio_ceph install_method: distro - job: @@ -242,13 +242,13 @@ scenario: aio_nspawn - job: - name: openstack-ansible-upgrade-ceph-centos-7 + name: openstack-ansible-upgrade-aio_ceph-centos-7 parent: openstack-ansible-deploy-aio nodeset: centos-7 timeout: 10800 vars: action: upgrade - scenario: ceph + scenario: aio_ceph # opensuse - job: @@ -264,20 +264,20 @@ install_method: distro - job: - name: openstack-ansible-deploy-ceph-opensuse-423 + name: openstack-ansible-deploy-aio_ceph-opensuse-423 parent: openstack-ansible-deploy-aio nodeset: opensuse-423 vars: action: deploy - scenario: ceph + scenario: aio_ceph - job: - name: openstack-ansible-deploy-distro_ceph-opensuse-423 + name: openstack-ansible-deploy-aio_distro_ceph-opensuse-423 parent: openstack-ansible-deploy-aio nodeset: opensuse-423 vars: action: deploy - scenario: ceph + scenario: aio_ceph install_method: distro - job: diff --git a/zuul.d/project-templates.yaml b/zuul.d/project-templates.yaml index 1edbf27659..3c2bf9496c 100644 --- a/zuul.d/project-templates.yaml +++ b/zuul.d/project-templates.yaml @@ -29,8 +29,8 @@ jobs: - openstack-ansible-deploy-aio_nspawn-ubuntu-bionic - openstack-ansible-deploy-barbican-ubuntu-bionic - - openstack-ansible-deploy-congress-ubuntu-bionic - - openstack-ansible-deploy-octavia-ubuntu-bionic + - openstack-ansible-deploy-aio_congress-ubuntu-bionic + - openstack-ansible-deploy-aio_octavia-ubuntu-bionic - openstack-ansible-deploy_with_ansible_devel-aio-ubuntu-bionic - openstack-ansible-deploy_with_ansible_next-aio-ubuntu-bionic @@ -98,18 +98,18 @@ name: openstack-ansible-deploy-ceph-jobs check: jobs: - - openstack-ansible-deploy-ceph-ubuntu-bionic: + - openstack-ansible-deploy-aio_ceph-ubuntu-bionic: voting: false - - openstack-ansible-deploy-ceph-ubuntu-xenial + - openstack-ansible-deploy-aio_ceph-ubuntu-xenial gate: jobs: - - openstack-ansible-deploy-ceph-ubuntu-xenial + - openstack-ansible-deploy-aio_ceph-ubuntu-xenial - project-template: name: openstack-ansible-deploy-distro_ceph-jobs check: jobs: - - openstack-ansible-deploy-distro_ceph-opensuse-423: + - openstack-ansible-deploy-aio_distro_ceph-opensuse-423: voting: false gate: jobs: []