From e1cda1ca596f1002770f54e89446eadbf574774f Mon Sep 17 00:00:00 2001 From: Andrea Frittoli Date: Sat, 17 Feb 2018 22:21:26 +0000 Subject: [PATCH] Fix multinode mode for devstack Extend the devstack job so that it can support both single and multinode cases. Multinode mode require extra settings in devstack configuration, some of which as subnode specific, some controller specific. Also keep a simple devstack-multinode job defined for now so we can run a multinode job in devstack gate, until the full tempest multinode job is ready to match the old gate-tempest-dsvm-neutron-multinode-full-ubuntu-xenial-nv. Fixing multinode also requires sharing the CA configuration between controller and peers, overlay network configuration for communication between virtual machines and running discover_hosts for nova after the subnode has been setup. The extra orchestration required for multinode is encoded in a dedicated role to allow for jobs in other repos to re-use it. This backport is slightly modified from the original, since master does not use the test-matrix anymore, while on stable branches we still rely on it. Change-Id: I2dcbd9bdb401860820e655d97aa3c4775af2827f --- .zuul.yaml | 56 ++++++++++++++++--- playbooks/devstack.yaml | 2 +- playbooks/pre.yaml | 30 ++++++---- roles/orchestrate-devstack/README.rst | 24 ++++++++ roles/orchestrate-devstack/defaults/main.yaml | 1 + roles/orchestrate-devstack/tasks/main.yaml | 38 +++++++++++++ roles/sync-devstack-data/README.rst | 12 ++++ roles/sync-devstack-data/defaults/main.yaml | 1 + roles/sync-devstack-data/tasks/main.yaml | 48 ++++++++++++++++ 9 files changed, 192 insertions(+), 20 deletions(-) create mode 100644 roles/orchestrate-devstack/README.rst create mode 100644 roles/orchestrate-devstack/defaults/main.yaml create mode 100644 roles/orchestrate-devstack/tasks/main.yaml create mode 100644 roles/sync-devstack-data/README.rst create mode 100644 roles/sync-devstack-data/defaults/main.yaml create mode 100644 roles/sync-devstack-data/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index ec22b5373d..19be7b4fb1 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -16,19 +16,24 @@ - name: compute1 label: ubuntu-xenial groups: + # Node where tests are executed and test results collected - name: tempest nodes: - controller + # Nodes running the compute service - name: compute nodes: - controller - compute1 + # Nodes that are not the controller - name: subnode nodes: - compute1 + # Switch node for multinode networking setup - name: switch nodes: - controller + # Peer nodes for multinode networking setup - name: peers nodes: - compute1 @@ -45,7 +50,7 @@ all single Devstack jobs, single or multinode. Variables are defined in job.vars, which is what is then used by single node jobs and by multi node jobs for the controller, as well as in - job.group-vars.peers, which is what is used by multi node jobs for peer + job.group-vars.peers, which is what is used by multi node jobs for subnode nodes (everything but the controller). required-projects: - openstack-dev/devstack @@ -75,6 +80,7 @@ # from the location below for all the CI jobs. ETCD_DOWNLOAD_URL: http://tarballs.openstack.org/etcd/ devstack_services: + # Ignore any default set by devstack. Emit a "disable_all_services". base: false zuul_copy_output: '{{ devstack_conf_dir }}/local.conf': 'logs' @@ -118,7 +124,8 @@ stackenv: True auto: True group-vars: - peers: + subnode: + test_matrix_role: subnode devstack_localrc: DATABASE_PASSWORD: secretdatabase RABBIT_PASSWORD: secretrabbit @@ -155,7 +162,9 @@ name: devstack parent: devstack-base description: | - Single node devstack job for integration gate. + Base devstack job for integration gate. + + This base job can be used for single node and multinode devstack jobs. nodeset: openstack-single-node required-projects: - openstack/cinder @@ -178,20 +187,49 @@ NOVA_VNC_ENABLED: true VNCSERVER_LISTEN: 0.0.0.0 VNCSERVER_PROXYCLIENT_ADDRESS: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" + # Multinode specific settings + SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + HOST_IP: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + PUBLIC_BRIDGE_MTU: "{{ external_bridge_mtu }}" + devstack_localconf: + post-config: + $NEUTRON_CONF: + DEFAULT: + global_physnet_mtu: "{{ external_bridge_mtu }}" devstack_services: base: true + # We need this explicitly here to trigger the sync of the CA data + tls-proxy: true horizon: false tempest: false + # Test matrix emits ceilometer but ceilomenter is not installed in the + # integrated gate, so specifying the services has not effect. + # ceilometer-*: false + group-vars: + subnode: + devstack_services: + base: true + # We need this explicitly here to trigger the sync of the CA data + tls-proxy: true + devstack_localrc: + # Multinode specific settings + HOST_IP: "{{ hostvars[inventory_hostname]['nodepool']['private_ipv4'] }}" + SERVICE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + PUBLIC_BRIDGE_MTU: "{{ external_bridge_mtu }}" + # Subnode specific settings + DATABASE_TYPE: mysql + GLANCE_HOSTPORT: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}:9292" + Q_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + RABBIT_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" + DATABASE_HOST: "{{ hostvars['controller']['nodepool']['private_ipv4'] }}" - job: name: devstack-multinode - parent: devstack-base - description: Base devstack multinode job + parent: devstack nodeset: openstack-two-node - # NOTE(andreaf) The multinode job is useful to see the setup of different - # services on different nodes, however the subnode configuration is not - # ready yet. Until then this job should stay non-voting. - voting: false + description: | + Simple multinode test to verify multinode functionality on devstack side. + This is not meant to be used as a parent job. - job: name: devstack-tox-base diff --git a/playbooks/devstack.yaml b/playbooks/devstack.yaml index ede8382632..93d19f1c7a 100644 --- a/playbooks/devstack.yaml +++ b/playbooks/devstack.yaml @@ -1,3 +1,3 @@ - hosts: all roles: - - run-devstack + - orchestrate-devstack diff --git a/playbooks/pre.yaml b/playbooks/pre.yaml index 6681fb20a5..4689a6354f 100644 --- a/playbooks/pre.yaml +++ b/playbooks/pre.yaml @@ -1,15 +1,25 @@ -- hosts: controller - roles: - - role: test-matrix - test_matrix_role: primary - -- hosts: subnode - roles: - - role: test-matrix - test_matrix_role: subnode - - hosts: all + pre_tasks: + - name: Gather minimum local MTU + set_fact: + local_mtu: > + {% set mtus = [] -%} + {% for interface in ansible_interfaces -%} + {% set interface_variable = 'ansible_' + interface -%} + {% if interface_variable in hostvars[inventory_hostname] -%} + {% set _ = mtus.append(hostvars[inventory_hostname][interface_variable]['mtu']|int) -%} + {% endif -%} + {% endfor -%} + {{- mtus|min -}} + - name: Calculate external_bridge_mtu + # 50 bytes is overhead for vxlan (which is greater than GRE + # allowing us to use either overlay option with this MTU. + # TODO(andreaf) This should work, but it may have to be reconcilied with + # the MTU setting used by the multinode setup roles in multinode pre.yaml + set_fact: + external_bridge_mtu: "{{ local_mtu | int - 50 }}" roles: + - test-matrix - configure-swap - setup-stack-user - setup-tempest-user diff --git a/roles/orchestrate-devstack/README.rst b/roles/orchestrate-devstack/README.rst new file mode 100644 index 0000000000..7803ee4d74 --- /dev/null +++ b/roles/orchestrate-devstack/README.rst @@ -0,0 +1,24 @@ +Orchestrate a devstack + +Runs devstack in a multinode scenario, with one controller node +and a group of subnodes. + +The reason for this role is so that jobs in other repository may +run devstack in their plays with no need for re-implementing the +orchestration logic. + +The "run-devstack" role is available to run devstack with no +orchestration. + +This role sets up the controller and CA first, it then pushes CA +data to sub-nodes and run devstack there. The only requirement for +this role is for the controller inventory_hostname to be "controller" +and for all sub-nodes to be defined in a group called "subnode". + + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. diff --git a/roles/orchestrate-devstack/defaults/main.yaml b/roles/orchestrate-devstack/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/orchestrate-devstack/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/orchestrate-devstack/tasks/main.yaml b/roles/orchestrate-devstack/tasks/main.yaml new file mode 100644 index 0000000000..12db58c520 --- /dev/null +++ b/roles/orchestrate-devstack/tasks/main.yaml @@ -0,0 +1,38 @@ +- name: Run devstack on the controller + include_role: + name: run-devstack + when: inventory_hostname == 'controller' + +- name: Setup devstack on sub-nodes + block: + + - name: Sync CA data to subnodes (when any) + # Only do this if the tls-proxy service is defined and enabled + include_role: + name: sync-devstack-data + when: devstack_services['tls-proxy']|default(false) + + - name: Run devstack on the sub-nodes + include_role: + name: run-devstack + when: inventory_hostname in groups['subnode'] + + - name: Discover hosts + # Discovers compute nodes (subnodes) and maps them to cells. Only run + # on the controller node. + # NOTE(mriedem): We want to remove this if/when nova supports + # auto-registration of computes with cells, but that's not happening in + # Ocata. + # NOTE(andreaf) This is taken (NOTE included) from the discover_hosts + # function in devstack gate. Since this is now in devstack, which is + # branched, we know that the discover_hosts tool exists. + become: true + become_user: stack + shell: ./tools/discover_hosts.sh + args: + chdir: "{{ devstack_base_dir }}/devstack" + when: inventory_hostname == 'controller' + + when: + - '"controller" in hostvars' + - '"subnode" in groups' diff --git a/roles/sync-devstack-data/README.rst b/roles/sync-devstack-data/README.rst new file mode 100644 index 0000000000..500e8cccc4 --- /dev/null +++ b/roles/sync-devstack-data/README.rst @@ -0,0 +1,12 @@ +Sync devstack data for multinode configurations + +Sync any data files which include certificates to be used if TLS is enabled. +This role must be executed on the controller and it pushes data to all +subnodes. + +**Role Variables** + +.. zuul:rolevar:: devstack_base_dir + :default: /opt/stack + + The devstack base directory. diff --git a/roles/sync-devstack-data/defaults/main.yaml b/roles/sync-devstack-data/defaults/main.yaml new file mode 100644 index 0000000000..fea05c8146 --- /dev/null +++ b/roles/sync-devstack-data/defaults/main.yaml @@ -0,0 +1 @@ +devstack_base_dir: /opt/stack diff --git a/roles/sync-devstack-data/tasks/main.yaml b/roles/sync-devstack-data/tasks/main.yaml new file mode 100644 index 0000000000..46000159d4 --- /dev/null +++ b/roles/sync-devstack-data/tasks/main.yaml @@ -0,0 +1,48 @@ +- name: Ensure the data folder exists + become: true + file: + path: "{{ devstack_base_dir }}/data" + state: directory + owner: stack + group: stack + mode: 0755 + when: 'inventory_hostname in groups["subnode"]|default([])' + +- name: Ensure the CA folder exists + become: true + file: + path: "{{ devstack_base_dir }}/data/CA" + state: directory + owner: stack + group: stack + mode: 0755 + when: 'inventory_hostname in groups["subnode"]|default([])' + +- name: Pull the CA certificate and folder + become: true + synchronize: + src: "{{ item }}" + dest: "{{ zuul.executor.work_root }}/{{ item | basename }}" + mode: pull + with_items: + - "{{ devstack_base_dir }}/data/ca-bundle.pem" + - "{{ devstack_base_dir }}/data/CA" + when: inventory_hostname == 'controller' + +- name: Push the CA certificate + become: true + become_user: stack + synchronize: + src: "{{ zuul.executor.work_root }}/ca-bundle.pem" + dest: "{{ devstack_base_dir }}/data/ca-bundle.pem" + mode: push + when: 'inventory_hostname in groups["subnode"]|default([])' + +- name: Push the CA folder + become: true + become_user: stack + synchronize: + src: "{{ zuul.executor.work_root }}/CA/" + dest: "{{ devstack_base_dir }}/data/" + mode: push + when: 'inventory_hostname in groups["subnode"]|default([])'