Replace upload_core_repos with Puppet-only task

Currently the code responsible for configuring repositories on the
managed nodes is kept directly it Nailgun. This commit introduces a new
task setup_repositories that replaces upload_core_repos.

Closes-Bug: 1508486
Closes-Bug: 1533682

Change-Id: I1b83e3bfaebecdb8455d5697e320f24fb4941536
(cherry picked from commit 4a6c031e79)

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/generate_apt_pins.rb

@@ -0,0 +1,51 @@
+require 'net/http'
+require 'uri'
+
+module Puppet::Parser::Functions
+  newfunction(
+      :generate_apt_pins,
+      :type  => :rvalue,
+      :arity => 1,
+      :doc   => <<-EOS
+Takes an array of repositories (in form used in astute.yaml) as argument.
+Returns a hash compatible with the apt::pin type provided by puppetlabs/apt
+module. It requires working connectivity to given repositories - the function
+parses repositories' Release files to obtain fields like Origin, Label, Suite
+and Codename. Repositories with no or empty priority are skipped.
+      EOS
+  ) do |args|
+    repositories = args[0]
+    raise Puppet::ParseError, "generate_apt_pins(): Requires array to work with" unless repositories.is_a? Array
+
+    result = {}
+    repositories.each do |repo|
+      next unless repo['priority']
+      uri = URI.parse "#{repo['uri']}/dists/#{repo['suite']}/Release"
+      response = Net::HTTP.get_response uri
+
+      unless response.kind_of? Net::HTTPSuccess
+        fail "GET HTTP request to: '#{uri.to_s}' have failed! (#{response.code} #{response.message})"
+      end
+
+      value_map = {
+          'originator' => /^Origin: (.*)/,
+          'label'      => /^Label: (.*)/,
+          'release'    => /^Suite: (.*)/,
+          'codename'   => /^Codename: (.*)/,
+      }
+
+      repo_data = {
+          'priority' => repo['priority'],
+      }
+
+      value_map.each do |key, regexp|
+        repo_data[key] = $1 if response.body =~ regexp
+      end
+
+      result.store repo['name'], repo_data
+    end
+
+    p result
+    result
+  end
+end

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/generate_apt_sources.rb

@@ -0,0 +1,28 @@
+module Puppet::Parser::Functions
+  newfunction(
+    :generate_apt_sources,
+    :type  => :rvalue,
+    :arity => 1,
+    :docs  => <<-EOS
+Transforms repo_setup['repos'] array provided by astute.yaml to a form easily
+usable with create_resources and the puppetlabs/apt's apt::source type.
+    EOS
+  ) do |args|
+    repositories = args[0]
+
+    raise(Puppet::ParseError, "generate_apt_sources(): Requires array to " +
+          "work with") unless repositories.is_a?(Array)
+
+    result = {}
+
+    repositories.each do |repo|
+      result.store repo['name'], {
+        'repos'    => repo['section'],
+        'release'  => repo['suite'],
+        'location' => repo['uri'],
+      }
+    end
+
+    return result
+  end
+end

deployment/puppet/osnailyfacter/modular/astute/tasks.yaml

@@ -125,18 +125,12 @@
     cwd: /
 
 #PRE DEPLOYMENT
-- id: upload_core_repos
-  type: upload_file
-  version: 2.0.0
-  role: '*'
-  requires: [pre_deployment_start]
-
 - id: rsync_core_puppet
   type: sync
   version: 2.0.0
   role: '*'
+  requires: [pre_deployment_start]
   required_for: [pre_deployment_end]
-  requires: [upload_core_repos]
   parameters:
     src: rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/
     dst: /etc/puppet/modules

deployment/puppet/osnailyfacter/modular/fuel_pkgs/fuel_pkgs.pp

@@ -1,10 +1,14 @@
 notice('MODULAR: fuel_pkgs.pp')
 
+$deep_merge_package_name = $::osfamily ? {
+  'RedHat' => 'rubygem-deep_merge',
+  'Debian' => 'ruby-deep-merge',
+}
+
 $fuel_packages = [
   'fuel-ha-utils',
   'fuel-misc',
+  $deep_merge_package_name
 ]
 
-package { $fuel_packages :
-  ensure => 'latest',
-}
+ensure_packages($fuel_packages)

deployment/puppet/osnailyfacter/modular/fuel_pkgs/setup_repositories.pp

@@ -0,0 +1,27 @@
+notice('MODULAR: setup_repositories.pp')
+
+$repo_setup = hiera('repo_setup', {})
+$repos = $repo_setup['repos']
+
+if $::osfamily == 'Debian' {
+  include ::apt
+
+  $repositories = generate_apt_sources($repos)
+  $pins         = generate_apt_pins($repos)
+
+  if ! empty($repositories) {
+    create_resources(apt::source, $repositories)
+  }
+
+  if ! empty($pins) {
+    create_resources(apt::pin, $pins)
+  }
+
+  apt::conf { 'AllowUnathenticated':
+    content       => 'APT::Get::AllowUnauthenticated 1;',
+    priority      => '02',
+    notify_update => false,
+  }
+
+  Apt::Source<||> ~> Exec<| title == 'apt_update' |>
+}

deployment/puppet/osnailyfacter/modular/fuel_pkgs/tasks.yaml

@@ -1,9 +1,20 @@
+- id: setup_repositories
+  type: puppet
+  version: 2.0.0
+  groups: [primary-controller, controller, compute, compute-vmware, cinder, cinder-vmware, primary-mongo, mongo, ceph-osd, virt]
+  requires: [hiera]
+  required_for: [fuel_pkgs]
+  parameters:
+    puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/fuel_pkgs/setup_repositories.pp
+    puppet_modules: /etc/puppet/modules
+    timeout: 600
+
 - id: fuel_pkgs
   type: puppet
   version: 2.0.0
   groups: [primary-controller, controller, cinder, cinder-vmware, compute, ceph-osd, primary-mongo, mongo, ironic]
-  requires: [deploy_start]
-  required_for: [hiera]
+  requires: [setup_repositories]
+  required_for: [globals]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/fuel_pkgs/fuel_pkgs.pp
     puppet_modules: /etc/puppet/modules

deployment/puppet/osnailyfacter/modular/hiera/hiera.pp

@@ -1,10 +1,5 @@
 notice('MODULAR: hiera.pp')
 
-$deep_merge_package_name = $::osfamily ? {
-  /RedHat/ => 'rubygem-deep_merge',
-  /Debian/ => 'ruby-deep-merge',
-}
-
 $data_dir            = '/etc/hiera'
 $override_dir        = 'plugins'
 $override_dir_path   = "${data_dir}/${override_dir}"
@@ -73,8 +68,3 @@ file { 'hiera_puppet_config' :
   target => $hiera_main_config,
 }
 
-# needed to support the 'deeper' merge_behavior setting for hiera
-package { 'rubygem-deep_merge':
-  ensure => present,
-  name   => $deep_merge_package_name,
-}

deployment/puppet/osnailyfacter/modular/hiera/tasks.yaml

@@ -4,7 +4,7 @@
   groups: [primary-controller, controller, cinder, cinder-block-device,
     cinder-vmware, compute, ceph-osd, primary-mongo, mongo, virt, ironic]
   requires: [deploy_start]
-  required_for: [deploy_end]
+  required_for: [setup_repositories]
   parameters:
     puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hiera/hiera.pp
     puppet_modules: /etc/puppet/modules

deployment/puppet/osnailyfacter/spec/functions/generate_apt_pins_spec.rb

@@ -0,0 +1,21 @@
+require 'spec_helper'
+
+describe 'generate_apt_pins' do
+  let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
+
+  let(:subject) {
+    Puppet::Parser::Functions.function(:generate_apt_pins)
+  }
+
+  it 'should exist' do
+    expect(subject).to eq 'function_generate_apt_pins'
+  end
+
+  it 'should expect 1 argument' do
+    expect { scope.function_generate_apt_pins([]) }.to raise_error(ArgumentError)
+  end
+
+  it 'should expect array as given argument' do
+    expect { scope.function_generate_apt_pins(['foobar']) }.to raise_error(Puppet::ParseError)
+  end
+end

deployment/puppet/osnailyfacter/spec/functions/generate_apt_sources_spec.rb

@@ -0,0 +1,71 @@
+require 'spec_helper'
+
+describe 'generate_apt_sources' do
+  let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
+
+  let(:subject) {
+    Puppet::Parser::Functions.function(:generate_apt_sources)
+  }
+
+  let(:input) {
+    [
+      {'name'     => 'ubuntu',
+       'section'  => 'main universe multiverse',
+       'uri'      => 'http://archive.ubuntu.com/ubuntu/',
+       'priority' => nil,
+       'suite'    => 'trusty',
+       'type'     => 'deb'},
+      {'name'     => 'ubuntu-updates',
+       'section'  => 'main universe multiverse',
+       'uri'      => 'http://archive.ubuntu.com/ubuntu/',
+       'priority' => nil,
+       'suite'    => 'trusty-updates',
+       'type'     => 'deb'},
+      {'name'     => 'ubuntu-security',
+       'section'  => 'main universe multiverse',
+       'uri'      => 'http://archive.ubuntu.com/ubuntu/',
+       'priority' => nil,
+       'suite'    => 'trusty-security',
+       'type'     => 'deb'},
+    ]
+  }
+
+  let (:output) {
+    {
+      'ubuntu' =>
+         {
+           'repos'    => 'main universe multiverse',
+           'release'  => 'trusty',
+           'location' => 'http://archive.ubuntu.com/ubuntu/'
+         },
+      'ubuntu-updates' =>
+         {
+           'repos'    => 'main universe multiverse',
+           'release'  => 'trusty-updates',
+           'location' => 'http://archive.ubuntu.com/ubuntu/'
+         },
+     'ubuntu-security' =>
+         {
+           'repos'    => 'main universe multiverse',
+           'release'  => 'trusty-security',
+           'location' => 'http://archive.ubuntu.com/ubuntu/'
+         },
+    }
+  }
+
+  it 'should exist' do
+    expect(subject).to eq 'function_generate_apt_sources'
+  end
+
+  it 'should expect 1 argument' do
+    expect { scope.function_generate_apt_sources([]) }.to raise_error(ArgumentError)
+  end
+
+  it 'should expect array as given argument' do
+    expect { scope.function_generate_apt_sources(['foobar']) }.to raise_error(Puppet::ParseError)
+  end
+
+  it 'should return apt::source compatible hash' do
+    expect(scope.function_generate_apt_sources([input])).to eq(output)
+  end
+end

tests/noop/spec/hosts/fuel_pkgs/fuel_pkgs_spec.rb

@@ -3,6 +3,29 @@ require 'shared-examples'
 manifest = 'fuel_pkgs/fuel_pkgs.pp'
 
 describe manifest do
+  shared_examples 'catalog' do
+    it 'should have ruby gem deep_merge installed' do
+      case facts[:operatingsystem]
+      when 'Ubuntu'
+        ruby_deep_merge = 'ruby-deep-merge'
+      when 'CentOS'
+        ruby_deep_merge = 'rubygem-deep_merge'
+      end
+
+      should contain_package(ruby_deep_merge).with(
+        'ensure' => 'present',
+      )
+    end
+
+    ['fuel-ha-utils', 'fuel-misc'].each do |pkg|
+      it "should install #{pkg} package" do
+        should contain_package(pkg).with(
+          'ensure' => 'present'
+        )
+      end
+    end
+  end
+
   test_ubuntu_and_centos manifest
 end
 

tests/noop/spec/hosts/fuel_pkgs/setup_repositories_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+require 'shared-examples'
+manifest = 'fuel_pkgs/setup_repositories.pp'
+
+describe manifest do
+  before(:each) do
+    Noop.puppet_function_load :generate_apt_pins
+    MockFunction.new(:generate_apt_pins) do |function|
+      allow(function).to receive(:call).and_return({})
+    end
+  end
+
+  test_ubuntu_and_centos manifest
+end
+

tests/noop/spec/hosts/hiera/hiera_spec.rb

@@ -39,19 +39,6 @@ describe manifest do
         'target' => '/etc/hiera.yaml'
       )
     end
-    it 'should have ruby deep_merge installed' do
-      case facts[:operatingsystem]
-      when 'Ubuntu'
-        package_name = 'ruby-deep-merge'
-      when 'CentOS'
-        package_name = 'rubygem-deep_merge'
-      end
-
-      should contain_package('rubygem-deep_merge').with(
-        'ensure' => 'present',
-        'name'   => package_name
-      )
-    end
   end
 
   test_ubuntu_and_centos manifest

tests/noop/spec/lib/facts.rb

@@ -36,6 +36,7 @@ class Noop
           :l3_default_route => '172.16.1.1',
           :concat_basedir => '/tmp/',
           :l23_os => 'ubuntu',
+          :lsbdistcodename => 'trusty',
       }.merge override_facts
     end