From 0dfe021b41ef32d63c2ecfeb82ebb3697f18eeb6 Mon Sep 17 00:00:00 2001 From: Dmitry Nikishov Date: Mon, 17 Oct 2016 17:33:31 +0300 Subject: [PATCH] Graph for osd-upgrade procedure This commit introduces the task graphs that handle osd-upgrade command. Python part is in a separate commit. There are 3 graphs: 1. (orig) prepare environment: update repos 2. (seed) set noout & upgrade ceph on target nodes 3. (orig) unset noout and restart Change-Id: I0ce93831f715ad919acb4420f05b729afb9bb15b --- .../graphs/orig/upgrade-osd-post.yaml | 45 ++++++++++++ .../graphs/orig/upgrade-osd-pre.yaml | 73 +++++++++++++++++++ .../octane_tasks/graphs/seed/upgrade-osd.yaml | 62 ++++++++++++++++ .../parser/functions/ceph_equal_versions.rb | 21 ++++++ .../parser/functions/ceph_get_version.rb | 19 +++++ .../manifests/restart_ceph_osd.pp | 22 ++++++ .../octane_tasks/manifests/set_noout.pp | 23 ++++++ .../octane_tasks/manifests/unset_noout.pp | 16 ++++ .../manifests/upgrade_ceph_packages.pp | 26 +++++++ .../octane_tasks/modular/restart_ceph_osd.pp | 3 + .../puppet/octane_tasks/modular/set_noout.pp | 3 + .../octane_tasks/modular/unset_noout.pp | 3 + .../modular/upgrade_ceph_packages.pp | 3 + 13 files changed, 319 insertions(+) create mode 100644 deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-post.yaml create mode 100644 deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-pre.yaml create mode 100644 deployment/puppet/octane_tasks/graphs/seed/upgrade-osd.yaml create mode 100644 deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_equal_versions.rb create mode 100644 deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_get_version.rb create mode 100644 deployment/puppet/octane_tasks/manifests/restart_ceph_osd.pp create mode 100644 deployment/puppet/octane_tasks/manifests/set_noout.pp create mode 100644 deployment/puppet/octane_tasks/manifests/unset_noout.pp create mode 100644 deployment/puppet/octane_tasks/manifests/upgrade_ceph_packages.pp create mode 100644 deployment/puppet/octane_tasks/modular/restart_ceph_osd.pp create mode 100644 deployment/puppet/octane_tasks/modular/set_noout.pp create mode 100644 deployment/puppet/octane_tasks/modular/unset_noout.pp create mode 100644 deployment/puppet/octane_tasks/modular/upgrade_ceph_packages.pp diff --git a/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-post.yaml b/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-post.yaml new file mode 100644 index 00000000..15279478 --- /dev/null +++ b/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-post.yaml @@ -0,0 +1,45 @@ +# GROUPS +- id: ceph-osd + type: group + role: [ceph-osd] + fault_tolerance: 0 + +# TASKS +- id: restart_ceph_osd + type: puppet + version: 2.1.0 + groups: [ceph-osd] + requires: [upgrade_ceph_packages] + cross-depends: + - name: upgrade_ceph_packages + role: ceph-osd + required_for: [] + parameters: + puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/restart_ceph_osd.pp + puppet_modules: /etc/fuel/octane/puppet + timeout: 600 + +- id: unset_noout + type: puppet + version: 2.1.0 + groups: [ceph-osd] + requires: [restart_ceph_osd] + cross-depends: + - name: restart_ceph_osd + role: ceph-osd + required_for: [] + parameters: + puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/unset_noout.pp + puppet_modules: /etc/fuel/octane/puppet + timeout: 600 + +- id: remove_hiera_override + type: shell + version: 2.1.0 + groups: [ceph-osd] + requires: [unset_noout] + required_for: [] + parameters: + cmd: rm /etc/hiera/override/common.yaml || true + timeout: 60 + diff --git a/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-pre.yaml b/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-pre.yaml new file mode 100644 index 00000000..f9268346 --- /dev/null +++ b/deployment/puppet/octane_tasks/graphs/orig/upgrade-osd-pre.yaml @@ -0,0 +1,73 @@ +# GROUPS +- id: ceph-osd + type: group + role: [ceph-osd] + fault_tolerance: 0 + +# TASKS +- id: rsync_octane + type: sync + version: 2.1.0 + groups: [ceph-osd] + requires: [] + required_for: [] + parameters: + src: rsync://{MASTER_IP}:/octane_code/puppet + dst: /etc/fuel/octane/ + timeout: 180 + +- id: remove_hiera_section_repo_setup + type: shell + version: 2.1.0 + groups: [ceph-osd] + requires: [rsync_octane] + required_for: [] + parameters: + cmd: python /etc/fuel/octane/puppet/octane_tasks/files/delete_section.py /etc/astute.yaml repo_setup repos + timeout: 60 + +- id: override_repos_in_hiera + type: upload_file + version: 2.1.0 + groups: [ceph-osd] + requires: [] + required_for: [] + parameters: + path: /etc/hiera/override/common.yaml + data: + yaql_exp: > + ({"repo_setup" => {"repos" => $.repo_setup.upgrade_osd}}.toYaml()) + +- id: cleanup_existing_repos + type: shell + version: 2.1.0 + groups: [ceph-osd] + requires: [] + required_for: [] + parameters: + cmd: > + tar zcf /root/sources.list.d-backup-$(date +%F-%H%M).tar.gz /etc/apt/sources.list.d; + rm /etc/apt/sources.list.d/*.list || true + timeout: 60 + +- id: rsync_latest_puppet + type: sync + version: 2.1.0 + groups: [ceph-osd] + requires: [] + required_for: [] + parameters: + src: rsync://{MASTER_IP}:/puppet/modules/ + dst: /etc/fuel/octane/latest_modules + timeout: 180 + +- id: setup_new_repositories + type: puppet + version: 2.1.0 + groups: [ceph-osd] + requires: [cleanup_existing_repos, rsync_latest_puppet, override_repos_in_hiera, remove_hiera_section_repo_setup] + required_for: [] + parameters: + puppet_manifest: /etc/fuel/octane/latest_modules/osnailyfacter/modular/fuel_pkgs/setup_repositories.pp + puppet_modules: /etc/fuel/octane/latest_modules + timeout: 600 diff --git a/deployment/puppet/octane_tasks/graphs/seed/upgrade-osd.yaml b/deployment/puppet/octane_tasks/graphs/seed/upgrade-osd.yaml new file mode 100644 index 00000000..2d45daf5 --- /dev/null +++ b/deployment/puppet/octane_tasks/graphs/seed/upgrade-osd.yaml @@ -0,0 +1,62 @@ +# GROUPS +- id: primary-controller + type: group + role: [primary-controller] + fault_tolerance: 0 + +# TASKS +- id: rsync_octane + type: sync + version: 2.1.0 + groups: [primary-controller] + requires: [] + required_for: [] + parameters: + src: rsync://{MASTER_IP}:/octane_code/puppet + dst: /etc/fuel/octane/ + timeout: 180 + +- id: ceph_osd_hiera + type: upload_file + version: 2.1.0 + groups: [primary-controller] + requires: [] + required_for: [] + parameters: + path: /etc/hiera/override/common.yaml + data: + yaql_exp: > + ({"ceph_upgrade_release" => $.ceph_upgrade_release, + "ceph_upgrade_hostnames" => $.ceph_upgrade_hostnames}.toYaml()) + +- id: set_noout + type: puppet + version: 2.1.0 + groups: [primary-controller] + requires: [rsync_octane, ceph_osd_hiera] + required_for: [] + parameters: + puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/set_noout.pp + puppet_modules: /etc/fuel/octane/puppet + timeout: 600 + +- id: upgrade_ceph_packages + type: puppet + version: 2.1.0 + groups: [primary-controller] + requires: [set_noout] + required_for: [] + parameters: + puppet_manifest: /etc/fuel/octane/puppet/octane_tasks/modular/upgrade_ceph_packages.pp + puppet_modules: /etc/fuel/octane/puppet:/etc/puppet/modules + timeout: 600 + +- id: remove_hiera_override + type: shell + version: 2.1.0 + groups: [primary-controller] + requires: [upgrade_ceph_packages] + required_for: [] + parameters: + cmd: rm /etc/hiera/override/common.yaml || true + timeout: 60 diff --git a/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_equal_versions.rb b/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_equal_versions.rb new file mode 100644 index 00000000..a9de385d --- /dev/null +++ b/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_equal_versions.rb @@ -0,0 +1,21 @@ +Puppet::Parser::Functions.newfunction(:ceph_equal_versions, :type => :rvalue) do |args| + require 'json' + + versions_1 = args[0] + versions_2 = args[1] + + # Pre-check all versions for consistency. Each hash MUST contain same + # versions for all elements. + v1_equal = versions_1.values.all? {|val| val == versions_1.values[0]} + v2_equal = versions_2.values.all? {|val| val == versions_2.values[0]} + + # Either array contains some values, that are not equal. This means, that something + # went wrong and relevant component has only been partially upgraded. Fail with info message. + fail "Partial upgrade detected, aborting. Current version layout: #{versions_1}, #{versions_2}" unless v1_equal and v2_equal + + # Intersection of 2 arrays with any amount of equal elements will yield an + # array with only one element + ret = (versions_1.values & versions_2.values).length == 1 + + ret +end diff --git a/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_get_version.rb b/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_get_version.rb new file mode 100644 index 00000000..af640c1e --- /dev/null +++ b/deployment/puppet/octane_tasks/lib/puppet/parser/functions/ceph_get_version.rb @@ -0,0 +1,19 @@ +Puppet::Parser::Functions.newfunction(:ceph_get_version, :type => :rvalue) do |args| + require 'json' + + service_type = args[0] + id = '*' + versions = {} + + + version_string = Puppet::Util::Execution.execute("ceph tell #{service_type}.#{id} version -f json") + version_string.lines.each do |line| + line = line.strip + if line.length > 0 + entity, version = line.split(" ", 2) + entity = entity.tr(":", "") + versions[entity] = JSON.parse(version)['version'] + end + end + versions +end diff --git a/deployment/puppet/octane_tasks/manifests/restart_ceph_osd.pp b/deployment/puppet/octane_tasks/manifests/restart_ceph_osd.pp new file mode 100644 index 00000000..585f756e --- /dev/null +++ b/deployment/puppet/octane_tasks/manifests/restart_ceph_osd.pp @@ -0,0 +1,22 @@ +# == class: octane_tasks::restart_ceph_osd +# +# this class restarts ceph osd after package upgrade +# +class octane_tasks::restart_ceph_osd { + $ceph_mon_versions = ceph_get_version('mon') + $ceph_osd_versions = ceph_get_version('osd') + + Exec { + provider => shell, + } + + if ! ceph_equal_versions($ceph_mon_versions, $ceph_osd_versions) { + + exec { 'restart-ceph-osd': + command => 'restart ceph-osd-all', + } + + } else { + notice('the version of osd on current node matches mon version, nothing to upgrade.') + } +} diff --git a/deployment/puppet/octane_tasks/manifests/set_noout.pp b/deployment/puppet/octane_tasks/manifests/set_noout.pp new file mode 100644 index 00000000..8d0787f5 --- /dev/null +++ b/deployment/puppet/octane_tasks/manifests/set_noout.pp @@ -0,0 +1,23 @@ +# == class: octane_tasks::set_noout +# +# this class sets the noout flag for osd pre-upgrade +# +class octane_tasks::set_noout { + $ceph_mon_versions = ceph_get_version('mon') + $ceph_osd_versions = ceph_get_version('osd') + + Exec { + provider => shell, + } + + if ! ceph_equal_versions($ceph_mon_versions, $ceph_osd_versions) { + + exec { 'set-noout-flag': + command => 'ceph osd set noout', + unless => 'ceph -s | grep -q "noout flag.\+ set"', + } + + } else { + notice('the version of osd on current node matches mon version, nothing to upgrade.') + } +} diff --git a/deployment/puppet/octane_tasks/manifests/unset_noout.pp b/deployment/puppet/octane_tasks/manifests/unset_noout.pp new file mode 100644 index 00000000..99a09ed3 --- /dev/null +++ b/deployment/puppet/octane_tasks/manifests/unset_noout.pp @@ -0,0 +1,16 @@ +# == Class: octane_tasks::unset_noout +# +# This class unsets the noout flag for OSD pre-upgrade +# +class octane_tasks::unset_noout { + + Exec { + provider => shell, + } + + exec { 'unset-noout-flag': + command => 'ceph osd unset noout', + onlyif => 'ceph -s | grep -q "noout flag.\+ set"', + } + +} diff --git a/deployment/puppet/octane_tasks/manifests/upgrade_ceph_packages.pp b/deployment/puppet/octane_tasks/manifests/upgrade_ceph_packages.pp new file mode 100644 index 00000000..e39437e0 --- /dev/null +++ b/deployment/puppet/octane_tasks/manifests/upgrade_ceph_packages.pp @@ -0,0 +1,26 @@ +# == class: octane_tasks::upgrade_ceph_packages +# +# this class upgrades ceph packages on the current node +# +class octane_tasks::upgrade_ceph_packages { + $ceph_mon_versions = ceph_get_version('mon') + $ceph_osd_versions = ceph_get_version('osd') + + $ceph_release = hiera('ceph_upgrade_release') + $node_hostnames_string = join(hiera('ceph_upgrade_hostnames'), ' ') + + Exec { + provider => shell, + } + + if ! ceph_equal_versions($ceph_mon_versions, $ceph_osd_versions) { + + exec { 'upgrade-ceph-packages': + command => "ceph-deploy install --release ${ceph_release} ${node_hostnames_string}", + } + + } else { + notice('the version of osd on current node matches mon version, nothing to upgrade.') + } +} + diff --git a/deployment/puppet/octane_tasks/modular/restart_ceph_osd.pp b/deployment/puppet/octane_tasks/modular/restart_ceph_osd.pp new file mode 100644 index 00000000..d5df44fa --- /dev/null +++ b/deployment/puppet/octane_tasks/modular/restart_ceph_osd.pp @@ -0,0 +1,3 @@ +notice('MODULAR: octane_tasks::restart_ceph_osd.pp') + +include ::octane_tasks::restart_ceph_osd diff --git a/deployment/puppet/octane_tasks/modular/set_noout.pp b/deployment/puppet/octane_tasks/modular/set_noout.pp new file mode 100644 index 00000000..a2fd365a --- /dev/null +++ b/deployment/puppet/octane_tasks/modular/set_noout.pp @@ -0,0 +1,3 @@ +notice('MODULAR: octane_tasks::set_noout.pp') + +include ::octane_tasks::set_noout diff --git a/deployment/puppet/octane_tasks/modular/unset_noout.pp b/deployment/puppet/octane_tasks/modular/unset_noout.pp new file mode 100644 index 00000000..c861d601 --- /dev/null +++ b/deployment/puppet/octane_tasks/modular/unset_noout.pp @@ -0,0 +1,3 @@ +notice('MODULAR: octane_tasks::unset_noout.pp') + +include ::octane_tasks::unset_noout diff --git a/deployment/puppet/octane_tasks/modular/upgrade_ceph_packages.pp b/deployment/puppet/octane_tasks/modular/upgrade_ceph_packages.pp new file mode 100644 index 00000000..83a75d9e --- /dev/null +++ b/deployment/puppet/octane_tasks/modular/upgrade_ceph_packages.pp @@ -0,0 +1,3 @@ +notice('MODULAR: octane_tasks::upgrade_ceph_packages.pp') + +include ::octane_tasks::upgrade_ceph_packages