Use "safe" mirror list on upgrade

All "safe" repos for using in upgrade will be configured in UI and
available in Hiera. Then this list will be used to link this mirrors
to separate "sources.d" directory. This directory will be used to point
APT another source location. That allows us to get list of packages, which
will be upgraded with specified in "sources.d" mirrors. Then, available
packages will be upgraded.

To keep this process configurable we can specify hash of packages, which
should be upgraded throw hiera.

DocImpact

Closes-bug: #1616553
Closes-bug: #1621056

Change-Id: Idf4ff71c04f8a75c572bb24e560b3827f3d55bac

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/get_packages_for_upgrade.rb

@@ -0,0 +1,39 @@
+module Puppet::Parser::Functions
+  newfunction(:get_packages_for_upgrade, :arity => 1, :type => :rvalue, :doc => <<-EOS
+Return a hash of packages (for specified source.list) that can be upgraded. All
+packages have by default ensure => latest. This function is applicable only on Debian based OS.
+
+Example:
+  get_packages_for_upgrade('/etc/apt/source.list.d/')
+
+Output example for upgradable package list (in one line string):
+  Inst ntp [1:4.2.6.p5+dfsg-3ubuntu2.14.04.8] (2:4.2.6.p5+dfsg-3~u14.04+mos1 mos9.0:mos9.0-proposed [amd64])\n
+  Inst ntpdate [1:4.2.6.p5+dfsg-3ubuntu2.14.04.8] (2:4.2.6.p5+dfsg-3~u14.04+mos1 mos9.0:mos9.0-proposed [amd64])\n
+  Conf ntp (2:4.2.6.p5+dfsg-3~u14.04+mos1 mos9.0:mos9.0-proposed [amd64])\n
+  Conf ntpdate (2:4.2.6.p5+dfsg-3~u14.04+mos1 mos9.0:mos9.0-proposed [amd64])\n
+
+Function output example:
+  { 'ntp' => { 'ensure' => 'latest'}, 'ntpdate' => { 'ensure' => 'latest' } }
+EOS
+  ) do |args|
+    return {} if Facter.value(:osfamily) != 'Debian'
+    errmsg = "get_packages_for_upgrade($apt_source_list_path)"
+    source_list = args[0]
+    raise(Puppet::ParseError, "#{errmsg}: 1st argument should be a string") unless source_list.is_a?(String)
+    raise(Puppet::ParseError, "#{errmsg}: $apt_source_list_path should exists") unless file_exists(source_list)
+    packages_to_upgrade = Hash.new
+    package_list_common_out = %x(apt-get --just-print -o Dir::etc::sourcelist='-' -o Dir::Etc::sourceparts='#{source_list}' dist-upgrade -qq)
+    # take only lines which describe installation step
+    package_list = package_list_common_out.split("\n").select { |line| line =~ /^Inst\b/ }
+    package_list.each do |package|
+      package_name = package.split(' ')[1]
+      packages_to_upgrade[package_name] = { 'ensure' => 'latest' }
+    end
+    return packages_to_upgrade
+  end
+end
+
+# method is added to stub call for tests
+def file_exists(path)
+  File.exist?(path)
+end

deployment/puppet/osnailyfacter/manifests/upgrade/pkg_upgrade.pp

@@ -1,18 +1,11 @@
 class osnailyfacter::upgrade::pkg_upgrade {
-  # hardcode with retries and sleeps for resolving lock issue
-  # should be rewritten
-  exec { 'do_upgrade':
-    command     => "apt-get dist-upgrade -y --force-yes -o 'APT::Get::AllowUnauthenticated=1' -o Dpkg::Options::='--force-confdef' -o Dpkg::Options::='--force-confold'",
-    environment => [ 'DEBIAN_FRONTEND=noninteractive' ],
-    path        => ['/usr/bin', '/usr/local/sbin', '/usr/sbin', '/sbin', '/bin' ],
-    timeout     => 1700,
-    try_sleep   => 10,
-    tries       => 5,
-    logoutput   => true,
-  }
+  $packages_to_upgrade = hiera_hash('upgrade_packages', get_packages_for_upgrade('/etc/fuel/maintenance/apt/sources.list.d/'))
 
-  $corosync_roles = hiera('corosync_roles', ['primary-controller', 'controller'])
-  if roles_include($corosync_roles) {
+  create_resources('package', $packages_to_upgrade)
+
+  $corosync_roles   = hiera('corosync_roles', ['primary-controller', 'controller'])
+  $corosync_upgrade = has_key($packages_to_upgrade, 'corosync') or has_key($packages_to_upgrade, 'pacemaker')
+  if roles_include($corosync_roles) and $corosync_upgrade {
     $content_policy = "#!/bin/bash\n[[ \"\$1\" == \"pacemaker\" ]] && exit 101\n"
     $policyrc_file  = '/usr/sbin/policy-rc.d'
 
@@ -23,19 +16,19 @@ class osnailyfacter::upgrade::pkg_upgrade {
       mode    => '0755',
       owner   => 'root',
       group   => 'root',
-      before  => Exec['do_upgrade']
     })
 
     exec { 'remove_policy':
       command => "rm -rf ${policyrc_file}",
       path    => '/bin',
-      require => Exec['do_upgrade'],
     }
 
-    ensure_resource('service', 'pacemaker', {
-      ensure  => running,
-      require => Exec['remove_policy']
-    })
+    ensure_resource('service', 'pacemaker', { ensure  => running })
+
+    File['create-policy-rc.d'] ->
+      Package<| title == 'corosync' or title == 'pacemaker' |> ->
+        Exec['remove_policy'] ->
+          Service['pacemaker']
   }
 
   if roles_include(['controller', 'primary-controller']) {

deployment/puppet/osnailyfacter/manifests/upgrade/prepare_symlinks.pp

@@ -0,0 +1,18 @@
+# prepares symlinks for choosen repos, which should be
+# used for env update
+class osnailyfacter::upgrade::prepare_symlinks {
+  $mu_upgrade   = hiera_hash('mu_upgrade', {})
+  $update_repos = split($mu_upgrade['repos'], ',')
+
+  file { [ '/etc/fuel/', '/etc/fuel/maintenance/', '/etc/fuel/maintenance/apt/' ]:
+    ensure  => 'directory',
+  } ->
+
+  file { '/etc/fuel/maintenance/apt/sources.list.d/':
+    ensure  => 'directory',
+    recurse => true,
+    purge   => true,
+  } ->
+
+  ::osnailyfacter::upgrade::repo_symlink { $update_repos: }
+}

deployment/puppet/osnailyfacter/manifests/upgrade/repo_symlink.pp

@@ -0,0 +1,14 @@
+# == Type: osnailyfacter::repo_symlink
+#
+# Creates symlink to "update repo" in special directory
+#
+define osnailyfacter::upgrade::repo_symlink () {
+
+  $repo = strip($name)
+
+  file { "symlink_repo-${repo}":
+    ensure => 'link',
+    path   => "/etc/fuel/maintenance/apt/sources.list.d/${repo}.list",
+    target => "/etc/apt/sources.list.d/${repo}.list",
+  }
+}

deployment/puppet/osnailyfacter/modular/upgrade/prepare_symlinks.pp

@@ -0,0 +1 @@
+include ::osnailyfacter::upgrade::prepare_symlinks

deployment/puppet/osnailyfacter/modular/upgrade/tasks.yaml

@@ -1,3 +1,16 @@
+- id: prepare_symlinks
+  type: puppet
+  version: 2.1.0
+  groups: [primary-controller, controller, cinder, cinder-block-device, cinder-vmware, compute, ceph-osd, primary-mongo, mongo, ironic]
+  requires: [setup_repositories]
+  required_for: [pkg_upgrade]
+  condition:
+    yaql_exp: $.get('mu_upgrade', {}).get('enabled') and changedAny($.mu_upgrade.repos)
+  parameters:
+    puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/upgrade/prepare_symlinks.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 180
+
 - id: pkg_upgrade
   type: puppet
   version: 2.1.0
@@ -5,7 +18,7 @@
   requires: [setup_repositories]
   required_for: [fuel_pkgs]
   condition:
-    yaql_exp: $.get('mu_upgrade', {}).get('enabled')
+    yaql_exp: $.get('mu_upgrade', {}).get('enabled') and changedAny($.mu_upgrade)
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/upgrade/pkg_upgrade.pp
     puppet_modules: /etc/puppet/modules

deployment/puppet/osnailyfacter/spec/functions/get_packages_to_upgrade_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe 'get_packages_for_upgrade' do
+
+  let(:source_path) do
+    '/etc/fuel/maintenance/apt/sources.list.d/'
+  end
+
+  let(:output) do
+    { 'ntp' => {'ensure' => 'latest' } }
+  end
+
+  let(:cmd) do
+    "apt-get --just-print -o Dir::etc::sourcelist='-' -o Dir::Etc::sourceparts='#{source_path}' dist-upgrade -qq"
+  end
+
+  let(:cmd_output) do
+    'Inst ntp [1:4.2.6.p5+dfsg-3ubuntu2.14.04.8] (2:4.2.6.p5+dfsg-3~u14.04+mos1 mos9.0:mos9.0-proposed [amd64])\n'
+  end
+
+  it 'should get packages hash' do
+    scope.stubs(:file_exists).with(source_path).returns(true)
+    scope.stubs(:`).with(cmd).returns(cmd_output)
+    is_expected.to run.with_params(source_path).and_return(output)
+  end
+
+  it 'should get empty packages hash if osfamily is not Debian' do
+    Facter.expects(:value).with(:osfamily).returns('redhat')
+    is_expected.to run.with_params(source_path).and_return({})
+  end
+end

tests/noop/spec/hosts/upgrade/pkg_upgrade_spec.rb

@@ -0,0 +1,52 @@
+# ROLE: primary-mongo
+# ROLE: primary-controller
+# ROLE: mongo
+# ROLE: ironic
+# ROLE: controller
+# ROLE: compute
+# ROLE: compute-vmware
+# ROLE: cinder-vmware
+# ROLE: cinder-block-device
+# ROLE: cinder
+# ROLE: ceph-osd
+require 'spec_helper'
+require 'shared-examples'
+
+manifest = 'upgrade/pkg_upgrade.pp'
+
+describe manifest do
+
+  shared_examples 'catalog' do
+    # stub function output, for details see:
+    # https://github.com/rodjek/rspec-puppet#accessing-the-parser-scope-where-the-function-is-running
+    before do
+      Puppet::Parser::Functions.newfunction(:get_packages_for_upgrade, :arity => 1, :type => :rvalue) { |args|
+        return { 'pacemaker' => { 'ensure' => 'latest' } }
+      }
+    end
+
+    corosync_role = Noop.puppet_function 'roles_include', ['primary-controller', 'controller']
+    if corosync_role
+      it { is_expected.to contain_package('pacemaker').with(:ensure => 'latest') }
+      it { is_expected.to contain_file('create-policy-rc.d').with(
+             :path    => '/usr/sbin/policy-rc.d',
+             :content => "#!/bin/bash\n[[ \"\$1\" == \"pacemaker\" ]] && exit 101\n",
+             :mode    => '0755',
+             :owner   => 'root',
+             :group   => 'root',
+           ).that_comes_before('Package[pacemaker]') }
+
+      it { is_expected.to contain_exec('remove_policy').with(
+           :command => "rm -rf /usr/sbin/policy-rc.d",
+           :path    => '/bin',
+         ).that_requires('Package[pacemaker]') }
+
+      it { is_expected.to contain_service('pacemaker').with(
+             :ensure => 'running').that_requires('Exec[remove_policy]') }
+    else
+      it { is_expected.to_not contain_file('create-policy-rc.d') }
+      it { is_expected.to_not contain_service('pacemaker') }
+    end
+  end
+  test_ubuntu_and_centos manifest
+end

tests/noop/spec/hosts/upgrade/prepare_symlinks_spec.rb

@@ -0,0 +1,39 @@
+# ROLE: primary-mongo
+# ROLE: primary-controller
+# ROLE: mongo
+# ROLE: ironic
+# ROLE: controller
+# ROLE: compute
+# ROLE: compute-vmware
+# ROLE: cinder-vmware
+# ROLE: cinder-block-device
+# ROLE: cinder
+# ROLE: ceph-osd
+require 'spec_helper'
+require 'shared-examples'
+
+manifest = 'upgrade/prepare_symlinks.pp'
+
+describe manifest do
+
+  shared_examples 'catalog' do
+    # TODO degorenko : update noop fixtures with mu_upgrade hash
+    mu_upgrade = Noop.hiera('mu_upgrade', {})
+    context 'it should create all symlinks', :if => mu_upgrade['enabled'] do
+      [ '/etc/fuel/', '/etc/fuel/maintenance/', '/etc/fuel/maintenance/apt/' ].each do |dir|
+        it { is_expected.to contain_file(dir).with(:ensure => 'directory') }
+      end
+      it { is_expected.to contain_file('/etc/fuel/maintenance/apt/sources.list.d/').with(
+             :ensure => 'directory',
+             :recurse => true,
+             :purge   => true )}
+      it { is_expected.to contain_osnailyfacter__upgrade__repo_symlink('mos-updates') }
+      it { is_expected.to contain_file('symlink_repo-mos-updates').with(
+             :ensure => 'link',
+             :path   => "/etc/fuel/maintenance/apt/sources.list.d/mos-updates.list",
+             :target => "/etc/apt/sources.list.d/mos-updates.list",
+           ) }
+    end
+  end
+  test_ubuntu_and_centos manifest
+end