From f8a150cc76802008806bd72c86fe2c28b3493e62 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Wed, 25 Oct 2017 12:56:04 +0100 Subject: [PATCH] Add the ability to run the role on all hosts We currently have spread out package/host management to multiple roles, sometimes repeating ourselves in the process (see pip_install and openstack_hosts overlap) That is against Ansible principles, and we should have one role that configures the minimum (to run openstack), applying it to all the nodes, maybe behaving slightly differently depending on some parameters. Here that parameter is if the host is a container or not. If the host is a container, all the physical host configuration (kernel and sysctl) is be skipped, the rest of the configuration (packages/repos) still applies. This needed a refactor to split the tasks into those two group while remaining efficient and avoid multiple back and forth of package installs/removal. For that last point, new defaults variables were introduced, allowing overrides per host/group. A node now member of a group x can now directly use this role to setup all its necessary repos and keys. Last, but not least, this override mechanism can now easily trigger pip_install role, which can from now on, be removed from every role. On top of that pip_install role can now remove its repo management, and focus on installing pip on hosts that don't have a proper version of pip installed. Change-Id: Ibf145e561c80a12055bd4d5dca3914c4d495a748 --- defaults/main.yml | 30 ++++- ...CentOS-SIG-Virtualization-RDO => 61E8806C} | 0 ...{RPM-GPG-KEY-CentOS-SIG-Cloud => 764429E6} | 0 ...ules_with_group_vars-8d169f564ffd450c.yaml | 25 ++++ tasks/configure_metal_hosts.yml | 86 ++++++++++++++ tasks/main.yml | 60 +++++----- tasks/openstack_host_install.yml | 16 --- tasks/openstack_host_install_apt.yml | 43 ------- tasks/openstack_host_install_dnf.yml | 1 - tasks/openstack_host_install_zypper.yml | 56 --------- tasks/openstack_hosts_configure_apt.yml | 56 +++++++++ tasks/openstack_hosts_configure_dnf.yml | 1 + ....yml => openstack_hosts_configure_yum.yml} | 108 ++++++++---------- tasks/openstack_hosts_configure_zypper.yml | 80 +++++++++++++ tasks/openstack_kernel_check.yml | 22 ---- tasks/openstack_kernel_modules.yml | 61 ---------- tasks/openstack_kernel_tuning.yml | 24 ---- tasks/openstack_sysstat.yml | 5 - templates/modprobe.conf.j2 | 15 +-- tests/openstack_hosts-overrides.yml | 3 +- vars/redhat-7.yml | 77 +++++++------ vars/suse-42.yml | 75 +++++++----- vars/ubuntu-16.04.yml | 59 ++++++---- 23 files changed, 480 insertions(+), 423 deletions(-) rename files/gpg/{RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO => 61E8806C} (100%) rename files/gpg/{RPM-GPG-KEY-CentOS-SIG-Cloud => 764429E6} (100%) create mode 100644 releasenotes/notes/specific_kernel_modules_with_group_vars-8d169f564ffd450c.yaml create mode 100644 tasks/configure_metal_hosts.yml delete mode 100644 tasks/openstack_host_install.yml delete mode 100644 tasks/openstack_host_install_apt.yml delete mode 120000 tasks/openstack_host_install_dnf.yml delete mode 100644 tasks/openstack_host_install_zypper.yml create mode 100644 tasks/openstack_hosts_configure_apt.yml create mode 120000 tasks/openstack_hosts_configure_dnf.yml rename tasks/{openstack_host_install_yum.yml => openstack_hosts_configure_yum.yml} (59%) create mode 100644 tasks/openstack_hosts_configure_zypper.yml delete mode 100644 tasks/openstack_kernel_check.yml delete mode 100644 tasks/openstack_kernel_modules.yml delete mode 100644 tasks/openstack_kernel_tuning.yml diff --git a/defaults/main.yml b/defaults/main.yml index cfddb620..bd0ba2aa 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Package cache +cache_timeout: 600 + # /etc/openstack-release settings openstack_distrib_id: "OSA" openstack_distrib_release: "{{ openstack_release | default('master') }}" @@ -21,6 +24,8 @@ openstack_distrib_description: "OpenStack-Ansible" openstack_distrib_file: yes openstack_distrib_file_path: "/etc/openstack-release" +is_container: "{{ ansible_virtualization_type == 'lxc' }}" + openstack_host_sysstat_enabled: true openstack_host_sysstat_interval: 1 openstack_host_sysstat_statistics_hour: 23 @@ -36,12 +41,28 @@ openstack_host_manage_hosts_file: true ## kernel modules for specific group hosts openstack_host_specific_kernel_modules: [] -# to include it in your play, an example is given below: +# If you want to include some specific modules per group +# of hosts, override this with a group/host var, like below: #openstack_host_specific_kernel_modules: -# - { name: "ebtables", pattern: "CONFIG_BRIDGE_NF_EBTABLES", group: "network_hosts" } +# - name: "ebtables" +# pattern: "CONFIG_BRIDGE_NF_EBTABLES" +## Where: ## :param name: name of the kernel module ## :param pattern: pattern to grep for in /boot/config-$kernel_version to check how module is configured inside kernel -## :param group: group of hosts where the module will be loaded +## Our default overrides will be combined with your overrides. + +# Overridable package list is composed of the old override +# named user_package_list and the standard defaults _package_list +openstack_hosts_package_list: "{{ _package_list + (user_package_list | default([])) }}" + +# Overridable package repo is composed of the old override +# named user_external_repo_lists and the standard defaults _package_repos +openstack_hosts_package_repos: "{{ _package_repos + (user_external_repos_list | default([])) }}" + +# Overridable package repo gpg is composed of the old override +# named user_external_repo_keys_list and the standard defaults _package_repos_keys +openstack_hosts_package_repos_keys: "{{ _package_repos_keys + (user_external_repo_keys_list | default([])) }}" +openstack_hosts_package_repos_priorities: "{{ _package_repos_priorities }}" # The following garbage collection values are set to better support lots of neutron networks/routers. # Used for setting the net.ipv4/6.neigh.default.gc_thresh* values. This assumes that facts were @@ -98,6 +119,9 @@ openstack_kernel_options: # above. openstack_user_kernel_options: [] +# Overridable set of packages to install on the host. +openstack_host_metal_distro_packages: "{{ _openstack_host_metal_distro_packages }}" + # Set the openstack domain name openstack_domain: openstack.local diff --git a/files/gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO b/files/gpg/61E8806C similarity index 100% rename from files/gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO rename to files/gpg/61E8806C diff --git a/files/gpg/RPM-GPG-KEY-CentOS-SIG-Cloud b/files/gpg/764429E6 similarity index 100% rename from files/gpg/RPM-GPG-KEY-CentOS-SIG-Cloud rename to files/gpg/764429E6 diff --git a/releasenotes/notes/specific_kernel_modules_with_group_vars-8d169f564ffd450c.yaml b/releasenotes/notes/specific_kernel_modules_with_group_vars-8d169f564ffd450c.yaml new file mode 100644 index 00000000..a9f3da87 --- /dev/null +++ b/releasenotes/notes/specific_kernel_modules_with_group_vars-8d169f564ffd450c.yaml @@ -0,0 +1,25 @@ +--- +upgrade: + - | + If you have overriden your + ``openstack_host_specific_kernel_modules``, please + remove its group matching, and move that override + directly to the appropriate group. + + Example, for an override like: + + .. code-block:: yaml + + - name: "ebtables" + pattern: "CONFIG_BRIDGE_NF_EBTABLES" + group: "network_hosts" + + You can create a file for the network_host group, + inside its group vars folder + ``/etc/openstack_deploy/group_vars/network_hosts``, + with the content: + + .. code-block:: yaml + + - name: "ebtables" + pattern: "CONFIG_BRIDGE_NF_EBTABLES" diff --git a/tasks/configure_metal_hosts.yml b/tasks/configure_metal_hosts.yml new file mode 100644 index 00000000..3fbbd6f9 --- /dev/null +++ b/tasks/configure_metal_hosts.yml @@ -0,0 +1,86 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Check Kernel Version + fail: + msg: > + Wrong kernel Version found + [ {{ ansible_kernel }} < {{ openstack_host_required_kernel }} ] + Resolve this issue before continuing. + when: ansible_kernel | version_compare(openstack_host_required_kernel, '<') + +- name: Disable cache for apt update for hosts + copy: + content: | + Acquire::http::No-Cache true; + dest: "/etc/apt/apt.conf.d/00apt-no-cache" + tags: + openstack_hosts-config + when: + - ansible_pkg_mgr == 'apt' + - > + global_environment_variables.http_proxy is defined or + global_environment_variables.HTTP_PROXY is defined or + global_environment_variables.https_proxy is defined or + global_environment_variables.HTTPS_PROXY is defined + +- name: Install distro packages for bare metal nodes + package: + name: "{{ openstack_host_metal_distro_packages }}" + state: "{{ openstack_hosts_package_state }}" + register: install_packages + until: install_packages | success + retries: 5 + delay: 2 + +- name: check how kernel modules are implemented (statically builtin, dynamic, not set) + slurp: + src: "/boot/config-{{ ansible_kernel }}" + register: modules + when: openstack_host_specific_kernel_modules | length > 0 + +- name: Fail fast if we can't load a module + fail: + msg: "{{ item.pattern }} is not set" + with_items: "{{ openstack_host_specific_kernel_modules }}" + when: + - (modules.content | b64decode).find(item.pattern + ' is not set') != -1 + +- name: "Load kernel module(s)" + modprobe: + name: "{{ item.name }}" + with_items: "{{ openstack_host_kernel_modules + openstack_host_specific_kernel_modules }}" + when: + - item.name != '' + - item.pattern is undefined or (item.pattern is defined and (modules.content | b64decode).find(item.pattern + '=m') != -1) + +- name: Write list of modules to load at boot + template: + src: modprobe.conf.j2 + dest: "{{ openstask_host_module_file }}" + +- name: Adding new system tuning + sysctl: + name: "{{ item.key }}" + value: "{{ item.value }}" + sysctl_set: "{{ item.set|default('yes') }}" + state: "{{ item.state|default('present') }}" + reload: no + with_items: "{{ openstack_kernel_options + openstack_user_kernel_options }}" + failed_when: false + +- name: Configure sysstat + include: openstack_sysstat.yml + when: openstack_host_sysstat_enabled | bool diff --git a/tasks/main.yml b/tasks/main.yml index 036e5f1f..4e92066f 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -24,19 +24,6 @@ tags: - always -- include: openstack_kernel_check.yml - tags: - - openstack_hosts-install - -- include: openstack_proxy_settings.yml - tags: - - openstack_hosts-install - - openstack_hosts-config - -- include: openstack_host_install.yml - tags: - - openstack_hosts-install - - name: Allow the usage of local facts file: path: /etc/ansible/facts.d/ @@ -44,30 +31,47 @@ tags: - openstack_hosts-install -- include: openstack_sysstat.yml +# Drop the release file everywhere +- include: openstack_release.yml tags: - openstack_hosts-install + +# Proxy configuration applies to all nodes +- name: Add global_environment_variables to environment file + blockinfile: + dest: "/etc/environment" + state: present + marker: "# {mark} Managed by OpenStack-Ansible" + insertbefore: EOF + block: "{{ lookup('template', 'environment.j2') }}" + tags: - openstack_hosts-config -- include: openstack_update_hosts_file.yml - static: no +# Configure host files should apply to all nodes +- name: Configure etc hosts files + include: openstack_update_hosts_file.yml when: openstack_host_manage_hosts_file | bool + tags: + - openstack_hosts-config + +# This allows to include this role to get all the distro +# specific configuration for all the nodes. +# It is also used for installing common packages to +# all nodes +- name: Apply package management distro specific configuration + include: "openstack_hosts_configure_{{ ansible_pkg_mgr | lower }}.yml" + +# Configure bare metal nodes: Kernel, sysctl, sysstat, hosts files, metal packages +- include: configure_metal_hosts.yml + when: + - not is_container tags: - openstack_hosts-install - - openstack_hosts-config - -- include: openstack_kernel_modules.yml - tags: - - openstack_hosts-config - -- include: openstack_kernel_tuning.yml - tags: - - openstack_hosts-config - include: openstack_authorized_keys.yml tags: - openstack_hosts-config -- include: openstack_release.yml - tags: - - openstack_hosts-install +# Now run the pip install role and your host should be ready! +#- include_role: pip_install +# when: host_need_pip | default(True) | bool diff --git a/tasks/openstack_host_install.yml b/tasks/openstack_host_install.yml deleted file mode 100644 index f1a3cbc9..00000000 --- a/tasks/openstack_host_install.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- include: "openstack_host_install_{{ ansible_pkg_mgr }}.yml" diff --git a/tasks/openstack_host_install_apt.yml b/tasks/openstack_host_install_apt.yml deleted file mode 100644 index a0fb535d..00000000 --- a/tasks/openstack_host_install_apt.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Remove conflicting distro packages - apt: - name: "{{ openstack_host_distro_packages_remove | default([]) }}" - state: absent - -- name: Disable cache for apt update if behind proxy - copy: - content: | - Acquire::http::No-Cache true; - dest: "/etc/apt/apt.conf.d/00apt-no-cache" - when: > - global_environment_variables.http_proxy is defined or - global_environment_variables.HTTP_PROXY is defined or - global_environment_variables.https_proxy is defined or - global_environment_variables.HTTPS_PROXY is defined - -- name: Install distro packages - apt: - pkg: "{{ openstack_host_distro_packages }}" - state: "{{ openstack_hosts_package_state }}" - update_cache: yes - cache_valid_time: "{{ cache_timeout }}" - register: install_packages - until: install_packages | success - retries: 5 - delay: 2 - tags: - - openstack-apt-packages diff --git a/tasks/openstack_host_install_dnf.yml b/tasks/openstack_host_install_dnf.yml deleted file mode 120000 index 66263050..00000000 --- a/tasks/openstack_host_install_dnf.yml +++ /dev/null @@ -1 +0,0 @@ -openstack_host_install_yum.yml \ No newline at end of file diff --git a/tasks/openstack_host_install_zypper.yml b/tasks/openstack_host_install_zypper.yml deleted file mode 100644 index 58110ee2..00000000 --- a/tasks/openstack_host_install_zypper.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -# Copyright 2017, SUSE LINUX GmbH. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE(hwoarang) snapper make take significant amount of CPU time -# when executing zypper over and over so it's best to disable the -# background comparison process. Snapper may need further tuning -# for example reducing the number of snapshots to keep, disable daily -# cleanup job etc but these may come later in the future if needed. -- name: Check if Snapper root configuration file exists - stat: - path: "/etc/snapper/configs/root" - register: snapper_root_config - tags: - - openstack_hosts-config - -- name: Disable background snapshot comparisons on Snapper - lineinfile: - path: "/etc/snapper/configs/root" - regexp: '^BACKGROUND_COMPARISON=.*' - line: 'BACKGROUND_COMPARISON="no"' - state: present - when: snapper_root_config.stat.exists - tags: - - openstack_hosts-config - -- name: Remove conflicting distro packages - zypper: - name: "{{ openstack_host_distro_packages_remove | default([]) }}" - state: absent - tags: - - openstack-zypper-packages - - openstack-packages - -- name: Install distro packages - zypper: - name: "{{ openstack_host_distro_packages }}" - state: "{{ openstack_hosts_package_state }}" - register: install_packages - until: install_packages|success - retries: 5 - delay: 2 - tags: - - openstack-zypper-packages - - openstack-packages diff --git a/tasks/openstack_hosts_configure_apt.yml b/tasks/openstack_hosts_configure_apt.yml new file mode 100644 index 00000000..5dc6ef5f --- /dev/null +++ b/tasks/openstack_hosts_configure_apt.yml @@ -0,0 +1,56 @@ +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# APT configuration tasks that apply on all nodes. + +- name: Remove the blacklisted packages + package: + name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: absent + +- name: Add/Remove repositories gpg keys manually + apt_key: + id: "{{ key.id | default(omit) }}" + data: "{{ key.data | default(omit) }}" # use lookup('file','armored_content.asc') + keyserver: "{{ key.keyserver | default(omit) }}" + url: "{{ key.url | default(omit) }}" + state: "{{ key.state | default('present') }}" + with_items: "{{ openstack_hosts_package_repos_keys }}" + loop_control: + loop_var: key + register: _add_apt_keys + until: _add_apt_keys | success + retries: 5 + delay: 2 + +- name: Add requirement packages (repositories gpg keys, toolkits...) + apt: + name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: "{{ openstack_hosts_package_state }}" + update_cache: yes + cache_valid_time: "{{ cache_timeout }}" + +- name: Add/Remove/Update standard and user defined repositories + apt_repository: + repo: "{{ repo.repo }}" + state: "{{ repo.state | default('present') }}" + filename: "{{ repo.filename | default(omit) }}" + update_cache: "{{ repo == package_repos[-1] }}" + with_items: "{{ openstack_hosts_package_repos }}" + loop_control: + loop_var: repo + register: _adding_apt_repo + until: _adding_apt_repo | success + retries: 5 + delay: 2 diff --git a/tasks/openstack_hosts_configure_dnf.yml b/tasks/openstack_hosts_configure_dnf.yml new file mode 120000 index 00000000..a8216fec --- /dev/null +++ b/tasks/openstack_hosts_configure_dnf.yml @@ -0,0 +1 @@ +openstack_hosts_configure_yum.yml \ No newline at end of file diff --git a/tasks/openstack_host_install_yum.yml b/tasks/openstack_hosts_configure_yum.yml similarity index 59% rename from tasks/openstack_host_install_yum.yml rename to tasks/openstack_hosts_configure_yum.yml index 3543a0df..2c10a190 100644 --- a/tasks/openstack_host_install_yum.yml +++ b/tasks/openstack_hosts_configure_yum.yml @@ -13,35 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install EPEL, and yum priorities plugin +- name: Disable requiretty for root sudo on centos + template: + dest: /etc/sudoers.d/openstack-ansible + owner: root + group: root + mode: "0440" + src: sudoers.j2 + +# yum configuration tasks that apply on all nodes. +- name: Remove the blacklisted packages package: - name: "{{ openstack_host_required_distro_packages }}" - state: "{{ openstack_hosts_package_state }}" - tags: - - openstack-yum-packages - - openstack-packages + name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: absent # Copy all factored-in GPG keys. # KeyID 764429E6 from https://raw.githubusercontent.com/rdo-infra/centos-release-openstack/ocata-rdo/RPM-GPG-KEY-CentOS-SIG-Cloud # KeyID 61E8806C from keyserver for rdo-qemu-ev -- name: Copy validated GPG keys +- name: If a keyfile is provided, copy the gpg keyfile to the key location copy: - src: "gpg/{{ item | basename }}" - dest: /etc/pki/rpm-gpg/ + src: "{{ item.keyfile }}" + dest: "{{ item.key }}" mode: '0644' - with_fileglob: - - "gpg/*" + with_items: "{{ openstack_hosts_package_repos_keys | selectattr('keyfile','defined') | list }}" - name: Ensure GPG keys have the correct SELinux contexts applied command: restorecon -Rv /etc/pki/rpm-gpg/ + # TODO(evrardjp): Be more idempotent + changed_when: false # Handle gpg keys manually - name: Install gpg keys rpm_key: - key: "{{ key.keyfile | default(key.key) }}" + key: "{{ key.key }}" validate_certs: "{{ key.validate_certs | default(omit) }}" state: "{{ key.state | default('present') }}" - with_items: "{{ openstack_host_rdo_repos_keys }}" + with_items: "{{ openstack_hosts_package_repos_keys }}" loop_control: loop_var: key register: _add_yum_keys @@ -49,28 +56,32 @@ retries: 5 delay: 2 +- name: Add requirement packages (repositories gpg keys packages, toolkits...) + package: + name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: "{{ openstack_hosts_package_state }}" + - name: Check for existing yum repositories shell: "yum-config-manager | grep 'repo:'" register: existing_yum_repos - tags: - - openstack-yum-packages - - openstack-packages - name: Add yum repositories if they do not exist yum_repository: - name: "{{ item.name }}" - description: "{{ item.description }}" - baseurl: "{{ item.baseurl }}" - file: "{{ item.file }}" - gpgcheck: "{{ item.gpgcheck }}" - enabled: "{{ item.enabled }}" - with_items: - - "{{ openstack_host_rdo_repos }}" + name: "{{ repo.name }}" + description: "{{ repo.description | default(omit) }}" + baseurl: "{{ repo.baseurl | default(omit) }}" + gpgkey: "{{ repo.gpgkey | default(omit) }}" + gpgcheck: "{{ repo.gpgcheck | default(omit) }}" + enabled: "{{ repo.enabled | default('yes') }}" + with_items: "{{ openstack_hosts_package_repos }}" + loop_control: + loop_var: repo when: - - item.name not in existing_yum_repos.stdout - tags: - - openstack-yum-packages - - openstack-packages + - repo.name not in existing_yum_repos.stdout + register: _adding_repo + until: _adding_repo | success + retries: 5 + delay: 2 - name: Update yum repositories if they already exist command: > @@ -79,47 +90,18 @@ {% for key in item.keys() if key != 'file' %} --setopt="{{ item.name }}.{{ key }}={{ item[key] }}" {% endfor %} - with_items: - - "{{ openstack_host_rdo_repos }}" + # TODO(evrardjp): Be more idempotent + changed_when: false + with_items: "{{ openstack_hosts_package_repos }}" when: - item.name in existing_yum_repos.stdout - tags: - - openstack-yum-packages - - openstack-packages -- name: Enable and set repo priorities +- name: Update repo priorities command: > yum-config-manager - {% for repo_priority in openstack_host_repo_priorities %} + {% for repo_priority in openstack_hosts_package_repos_priorities %} --enable {{ repo_priority['name'] }} \ --setopt="{{ repo_priority['name'] }}.priority={{ repo_priority['priority'] }}" {% endfor %} + # TODO(evrardjp): Be more idempotent changed_when: false - tags: - - openstack-yum-packages - - openstack-packages - -- name: Remove conflicting distro packages - package: - name: "{{ openstack_host_distro_packages_remove | default([]) }}" - state: absent - -- name: Install distro packages - package: - pkg: "{{ openstack_host_distro_packages }}" - state: "{{ openstack_hosts_package_state }}" - register: install_packages - until: install_packages | success - retries: 5 - delay: 2 - tags: - - openstack-yum-packages - - openstack-packages - -- name: Disable requiretty for root sudo on centos - template: - dest: /etc/sudoers.d/openstack-ansible - owner: root - group: root - mode: "0440" - src: sudoers.j2 diff --git a/tasks/openstack_hosts_configure_zypper.yml b/tasks/openstack_hosts_configure_zypper.yml new file mode 100644 index 00000000..0c19d540 --- /dev/null +++ b/tasks/openstack_hosts_configure_zypper.yml @@ -0,0 +1,80 @@ +--- +# Copyright 2017, SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(hwoarang) snapper make take significant amount of CPU time +# when executing zypper over and over so it's best to disable the +# background comparison process. Snapper may need further tuning +# for example reducing the number of snapshots to keep, disable daily +# cleanup job etc but these may come later in the future if needed. +- name: Check if Snapper root configuration file exists + stat: + path: "/etc/snapper/configs/root" + register: snapper_root_config + +- name: Disable background snapshot comparisons on Snapper + lineinfile: + path: "/etc/snapper/configs/root" + regexp: '^BACKGROUND_COMPARISON=.*' + line: 'BACKGROUND_COMPARISON="no"' + state: present + when: snapper_root_config.stat.exists + +- name: Remove the blacklisted packages + package: + name: "{{ openstack_hosts_package_list | selectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: absent + +- name: If a keyfile is provided, copy the gpg keyfile to the key location + copy: + src: "{{ item.keyfile }}" + dest: "{{ item.key }}" + with_items: "{{ openstack_hosts_package_repos_keys | selectattr('keyfile','defined') | list }}" + +- name: Add/Remove repositories gpg keys manually + rpm_key: + key: "{{ key.key }}" + state: "{{ key.state | default('present') }}" + validate_certs: "{{ key.validate_certs | default(omit) }}" + with_items: "{{ openstack_hosts_package_repos_keys }}" + loop_control: + loop_var: key + register: _add_rpm_keys + until: _add_rpm_keys | success + retries: 5 + delay: 2 + +- name: Add requirement packages (repositories gpg keys, toolkits...) + zypper: + name: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list }}" + state: "{{ openstack_hosts_package_state }}" + when: "{{ openstack_hosts_package_list | rejectattr('state','equalto','absent') | map(attribute='name') | list | length > 0}}" + +- name: Add/Remove/Update standard and user defined repositories + zypper_repository: + repo: "{{ repo.repo }}" + state: "{{ repo.state | default('present') }}" + name: "{{ repo.name | default(omit) }}" + enabled: "{{ repo.enabled | default(omit) }}" + disable_gpg_check: "{{ repo.disable_gpg_check | default(omit) }}" + description: "{{ repo.description | default(omit) }}" + autorefresh: "{{ repo.autorefresh | default(omit) }}" + auto_import_keys: "{{ repo.auto_import_keys | default(omit) }}" + with_items: "{{ openstack_hosts_package_repos }}" + loop_control: + loop_var: repo + register: _adding_repo + until: _adding_repo | success + retries: 5 + delay: 2 diff --git a/tasks/openstack_kernel_check.yml b/tasks/openstack_kernel_check.yml deleted file mode 100644 index a96c48bf..00000000 --- a/tasks/openstack_kernel_check.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Check Kernel Version - fail: - msg: > - Wrong kernel Version found - [ {{ ansible_kernel }} < {{ openstack_host_required_kernel }} ] - Resolve this issue before continuing. - when: ansible_kernel | version_compare(openstack_host_required_kernel, '<') diff --git a/tasks/openstack_kernel_modules.yml b/tasks/openstack_kernel_modules.yml deleted file mode 100644 index a9a5669f..00000000 --- a/tasks/openstack_kernel_modules.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: check how kernel modules are implemented (statically builtin, dynamic, not set) - slurp: - src: "/boot/config-{{ ansible_kernel }}" - register: modules - when: openstack_host_specific_kernel_modules | length > 0 - -- name: fail if a specific kernel module is not set - fail: - msg: "{{ item.pattern }} is not set" - with_items: "{{ openstack_host_specific_kernel_modules }}" - when: - - groups[item.group] is defined - - inventory_hostname in groups[item.group] - - (modules.content | b64decode).find(item.pattern + ' is not set') != -1 - -- name: fail if a specific pattern is not valid - fail: - msg: "{{ item.pattern }} is not a valid pattern" - with_items: "{{ openstack_host_specific_kernel_modules }}" - when: - - groups[item.group] is defined - - inventory_hostname in groups[item.group] - - (modules.content | b64decode).find(item.pattern + '=y') == -1 - - (modules.content | b64decode).find(item.pattern + '=m') == -1 - -- name: "Ensure kernel module(s)" - modprobe: - name: "{{ item }}" - with_items: "{{ openstack_host_kernel_modules }}" - when: - - openstack_host_kernel_modules | length > 0 - - item != '' - -- name: "Ensure dynamic specific kernel module(s) are loaded" - modprobe: - name: "{{ item.name }}" - with_items: "{{ openstack_host_specific_kernel_modules }}" - when: - - groups[item.group] is defined - - inventory_hostname in groups[item.group] - - (modules.content | b64decode).find(item.pattern + '=m') != -1 - -- name: Write list of modules to load at boot - template: - src: modprobe.conf.j2 - dest: "{{ openstask_host_module_file }}" diff --git a/tasks/openstack_kernel_tuning.yml b/tasks/openstack_kernel_tuning.yml deleted file mode 100644 index 142ca704..00000000 --- a/tasks/openstack_kernel_tuning.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Adding new system tuning - sysctl: - name: "{{ item.key }}" - value: "{{ item.value }}" - sysctl_set: "{{ item.set|default('yes') }}" - state: "{{ item.state|default('present') }}" - reload: no - with_items: "{{ openstack_kernel_options + openstack_user_kernel_options }}" - failed_when: false diff --git a/tasks/openstack_sysstat.yml b/tasks/openstack_sysstat.yml index f22407a4..ba19b36c 100644 --- a/tasks/openstack_sysstat.yml +++ b/tasks/openstack_sysstat.yml @@ -19,7 +19,6 @@ dest: "{{ openstack_host_sysstat_file }}" mode: "0644" when: - - openstack_host_sysstat_enabled | bool - ansible_pkg_mgr == 'apt' notify: Restart sysstat @@ -28,13 +27,10 @@ src: "{{ openstack_host_cron_template }}" dest: "/etc/cron.d/sysstat" mode: "{{ openstack_host_sysstat_cron_mode }}" - when: - - openstack_host_sysstat_enabled | bool - name: Restore SELinux contexts on sysstat cron file command: restorecon -v /etc/cron.d/sysstat when: - - openstack_host_sysstat_enabled | bool - ansible_pkg_mgr in ['yum', 'dnf'] - name: Start and enable the sysstat service @@ -43,5 +39,4 @@ state: started enabled: yes when: - - openstack_host_sysstat_enabled | bool - ansible_pkg_mgr in ['yum', 'dnf', 'zypper'] diff --git a/templates/modprobe.conf.j2 b/templates/modprobe.conf.j2 index e513500f..64b37135 100644 --- a/templates/modprobe.conf.j2 +++ b/templates/modprobe.conf.j2 @@ -1,16 +1,5 @@ # {{ ansible_managed }} # Modules from the openstack-ansible-openstack_hosts role -{% for module in openstack_host_kernel_modules %} -{{ module }} -{% endfor %} - -# Host-specific dynamic modules -{% - for module in openstack_host_specific_kernel_modules if ( - groups[module.group] is defined and - inventory_hostname in module.group and - item.pattern + '=m' in (modules.content | b64decode) - ) -%} -{{ module }} +{% for module in openstack_host_kernel_modules + openstack_host_specific_kernel_modules %} +{{ module.name }} {% endfor %} diff --git a/tests/openstack_hosts-overrides.yml b/tests/openstack_hosts-overrides.yml index e87bad2b..b3d3822f 100644 --- a/tests/openstack_hosts-overrides.yml +++ b/tests/openstack_hosts-overrides.yml @@ -1,3 +1,4 @@ --- openstack_host_specific_kernel_modules: - - { name: "ebtables", pattern: "CONFIG_BRIDGE_NF_EBTABLES", group: "hosts" } + - name: "ebtables" + pattern: "CONFIG_BRIDGE_NF_EBTABLES" diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index 94caf13d..8cfbdde3 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -23,34 +23,34 @@ openstack_host_sysstat_cron_mode: '0600' ## Kernel modules loaded on hosts openstack_host_kernel_modules: - - 8021q - - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('3.10.0-514', '>=') %}br_netfilter{% endif %}" - - dm_multipath - - dm_snapshot - - ebtables - - ip6table_filter - - ip6_tables - - ip_tables - - ipt_MASQUERADE - - ipt_REJECT - - iptable_filter - - iptable_mangle - - iptable_nat - - ip_vs - - iscsi_tcp - - nf_conntrack - - nf_conntrack_ipv4 - - nf_defrag_ipv4 - - nf_nat - - nf_nat_ipv4 + - name: 8021q + - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('3.10.0-514', '>=') %}br_netfilter{% endif %}" + - name: dm_multipath + - name: dm_snapshot + - name: ebtables + - name: ip6table_filter + - name: ip6_tables + - name: ip_tables + - name: ipt_MASQUERADE + - name: ipt_REJECT + - name: iptable_filter + - name: iptable_mangle + - name: iptable_nat + - name: ip_vs + - name: iscsi_tcp + - name: nf_conntrack + - name: nf_conntrack_ipv4 + - name: nf_defrag_ipv4 + - name: nf_nat + - name: nf_nat_ipv4 # TODO (odyssey4me): revise the minimum kernel version once this kernel version is commonplace # If we end up with more requirements like this, then we should change the approach. - - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}" - - vhost_net - - x_tables + - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}" + - name: vhost_net + - name: x_tables -## Base packages -openstack_host_distro_packages: +## Bare metal base packages +_openstack_host_metal_distro_packages: - bridge-utils - curl - device-mapper-event @@ -74,18 +74,23 @@ openstack_host_distro_packages: - time - wget -openstack_host_required_distro_packages: - - epel-release - - yum-plugin-priorities - - yum-utils +_package_repos_keys: + - name: openstack-pike + key: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud + keyfile: "gpg/764429E6" + - name: rdo-qemu-ev + key: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO + keyfile: "gpg/61E8806C" -openstack_host_rdo_repos_keys: - - repo: openstack-pike - keyfile: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud - - repo: rdo-qemu-ev - keyfile: /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Virtualization-RDO +_package_list: + - name: epel-release + state: present + - name: yum-plugin-priorities + state: present + - name: yum-utils + state: present -openstack_host_rdo_repos: +_package_repos: - file: rdo-qemu-ev name: rdo-qemu-ev description: "RDO CentOS-7 - QEMU EV" @@ -99,7 +104,7 @@ openstack_host_rdo_repos: gpgcheck: yes enabled: yes -openstack_host_repo_priorities: +_package_repos_priorities: - name: base priority: 50 - name: epel diff --git a/vars/suse-42.yml b/vars/suse-42.yml index fca44d5a..a397f95f 100644 --- a/vars/suse-42.yml +++ b/vars/suse-42.yml @@ -22,34 +22,34 @@ openstask_host_module_file: /etc/modules-load.d/openstack-ansible.conf ## Kernel modules loaded on hosts openstack_host_kernel_modules: - - 8021q - - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}" - - dm_multipath - - dm_snapshot - - ebtables - - ip6table_filter - - ip6_tables - - ip_tables - - ipt_MASQUERADE - - ipt_REJECT - - iptable_filter - - iptable_mangle - - iptable_nat - - ip_vs - - iscsi_tcp - - nf_conntrack - - nf_conntrack_ipv4 - - nf_defrag_ipv4 - - nf_nat - - nf_nat_ipv4 + - name: 8021q + - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}" + - name: dm_multipath + - name: dm_snapshot + - name: ebtables + - name: ip6table_filter + - name: ip6_tables + - name: ip_tables + - name: ipt_MASQUERADE + - name: ipt_REJECT + - name: iptable_filter + - name: iptable_mangle + - name: iptable_nat + - name: ip_vs + - name: iscsi_tcp + - name: nf_conntrack + - name: nf_conntrack_ipv4 + - name: nf_defrag_ipv4 + - name: nf_nat + - name: nf_nat_ipv4 # TODO (odyssey4me): revise the minimum kernel version once this kernel version is commonplace # If we end up with more requirements like this, then we should change the approach. - - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}" - - vhost_net - - x_tables + - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4.0-0', '<') %}scsi_dh{% endif %}" + - name: vhost_net + - name: x_tables ## Base packages -openstack_host_distro_packages: +_openstack_host_metal_distro_packages: - bridge-utils - patterns-openSUSE-devel_basis - curl @@ -71,6 +71,27 @@ openstack_host_distro_packages: - time - wget -## Packages to remove -openstack_host_distro_packages_remove: - - systemd-logger # conflicts with rsyslog +_package_repos_keys: [] +## example: +# - key: "http://url_to_gpg_key" +# validate_certs: +# state: +# - key: "/tmp/file1" +# keyfile: "gpg/file1" +# validate_certs: +# state: + +_package_list: + - name: systemd-logger + state: absent # conflicts with rsyslog + +_package_repos: [] +## example: +# - repo: +# state: +# name: +# enabled: +# disable_gpg_check: +# description: +# autorefresh: +# auto_import_keys: diff --git a/vars/ubuntu-16.04.yml b/vars/ubuntu-16.04.yml index 18056228..4eb42173 100644 --- a/vars/ubuntu-16.04.yml +++ b/vars/ubuntu-16.04.yml @@ -24,32 +24,32 @@ openstask_host_module_file: /etc/modules ## Kernel modules loaded on hosts openstack_host_kernel_modules: - - 8021q - - "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}" - - dm_multipath - - dm_snapshot - - ebtables - - ip6table_filter - - ip6_tables - - ip_tables - - ipt_MASQUERADE - - ipt_REJECT - - iptable_filter - - iptable_mangle - - iptable_nat - - ip_vs - - iscsi_tcp - - nbd - - nf_conntrack - - nf_conntrack_ipv4 - - nf_defrag_ipv4 - - nf_nat - - nf_nat_ipv4 - - vhost_net - - x_tables + - name: 8021q + - name: "{% if hostvars[inventory_hostname]['ansible_kernel'] | version_compare('4.4', '>=') %}br_netfilter{% endif %}" + - name: dm_multipath + - name: dm_snapshot + - name: ebtables + - name: ip6table_filter + - name: ip6_tables + - name: ip_tables + - name: ipt_MASQUERADE + - name: ipt_REJECT + - name: iptable_filter + - name: iptable_mangle + - name: iptable_nat + - name: ip_vs + - name: iscsi_tcp + - name: nbd + - name: nf_conntrack + - name: nf_conntrack_ipv4 + - name: nf_defrag_ipv4 + - name: nf_nat + - name: nf_nat_ipv4 + - name: vhost_net + - name: x_tables ## Base packages -openstack_host_distro_packages: +_openstack_host_metal_distro_packages: - apparmor-utils - apt-transport-https - bridge-utils @@ -75,3 +75,14 @@ openstack_host_distro_packages: - time - vlan - wget + +_package_repos_keys: [] +## example: +# - id: +# file: +# keyserver: +# url: +# state: + +_package_list: [] +_package_repos: []