Replace upload_core_repos with Puppet-only task

Currently the code responsible for configuring repositories on the
managed nodes is kept directly it Nailgun. This commit introduces a new
task setup_repositories that replaces upload_core_repos.

Closes-Bug: 1508486
Change-Id: I1b83e3bfaebecdb8455d5697e320f24fb4941536
This commit is contained in:
Bartłomiej Piotrowski 2015-11-20 13:47:08 +01:00
parent 5f8a873627
commit 4a6c031e79
14 changed files with 259 additions and 36 deletions

View File

@ -0,0 +1,51 @@
require 'net/http'
require 'uri'
module Puppet::Parser::Functions
newfunction(
:generate_apt_pins,
:type => :rvalue,
:arity => 1,
:doc => <<-EOS
Takes an array of repositories (in form used in astute.yaml) as argument.
Returns a hash compatible with the apt::pin type provided by puppetlabs/apt
module. It requires working connectivity to given repositories - the function
parses repositories' Release files to obtain fields like Origin, Label, Suite
and Codename. Repositories with no or empty priority are skipped.
EOS
) do |args|
repositories = args[0]
raise Puppet::ParseError, "generate_apt_pins(): Requires array to work with" unless repositories.is_a? Array
result = {}
repositories.each do |repo|
next unless repo['priority']
uri = URI.parse "#{repo['uri']}/dists/#{repo['suite']}/Release"
response = Net::HTTP.get_response uri
unless response.kind_of? Net::HTTPSuccess
fail "GET HTTP request to: '#{uri.to_s}' have failed! (#{response.code} #{response.message})"
end
value_map = {
'originator' => /^Origin: (.*)/,
'label' => /^Label: (.*)/,
'release' => /^Suite: (.*)/,
'codename' => /^Codename: (.*)/,
}
repo_data = {
'priority' => repo['priority'],
}
value_map.each do |key, regexp|
repo_data[key] = $1 if response.body =~ regexp
end
result.store repo['name'], repo_data
end
p result
result
end
end

View File

@ -0,0 +1,28 @@
module Puppet::Parser::Functions
newfunction(
:generate_apt_sources,
:type => :rvalue,
:arity => 1,
:docs => <<-EOS
Transforms repo_setup['repos'] array provided by astute.yaml to a form easily
usable with create_resources and the puppetlabs/apt's apt::source type.
EOS
) do |args|
repositories = args[0]
raise(Puppet::ParseError, "generate_apt_sources(): Requires array to " +
"work with") unless repositories.is_a?(Array)
result = {}
repositories.each do |repo|
result.store repo['name'], {
'repos' => repo['section'],
'release' => repo['suite'],
'location' => repo['uri'],
}
end
return result
end
end

View File

@ -125,18 +125,12 @@
cwd: /
#PRE DEPLOYMENT
- id: upload_core_repos
type: upload_file
version: 2.0.0
role: '*'
requires: [pre_deployment_start]
- id: rsync_core_puppet
type: sync
version: 2.0.0
role: '*'
requires: [pre_deployment_start]
required_for: [pre_deployment_end]
requires: [upload_core_repos]
parameters:
src: rsync://{MASTER_IP}:/puppet/{OPENSTACK_VERSION}/modules/
dst: /etc/puppet/modules

View File

@ -1,10 +1,14 @@
notice('MODULAR: fuel_pkgs.pp')
$deep_merge_package_name = $::osfamily ? {
'RedHat' => 'rubygem-deep_merge',
'Debian' => 'ruby-deep-merge',
}
$fuel_packages = [
'fuel-ha-utils',
'fuel-misc',
$deep_merge_package_name
]
package { $fuel_packages :
ensure => 'latest',
}
ensure_packages($fuel_packages)

View File

@ -0,0 +1,27 @@
notice('MODULAR: setup_repositories.pp')
$repo_setup = hiera('repo_setup', {})
$repos = $repo_setup['repos']
if $::osfamily == 'Debian' {
include ::apt
$repositories = generate_apt_sources($repos)
$pins = generate_apt_pins($repos)
if ! empty($repositories) {
create_resources(apt::source, $repositories)
}
if ! empty($pins) {
create_resources(apt::pin, $pins)
}
apt::conf { 'AllowUnathenticated':
content => 'APT::Get::AllowUnauthenticated 1;',
priority => '02',
notify_update => false,
}
Apt::Source<||> ~> Exec<| title == 'apt_update' |>
}

View File

@ -1,9 +1,20 @@
- id: setup_repositories
type: puppet
version: 2.0.0
groups: [primary-controller, controller, compute, compute-vmware, cinder, cinder-vmware, primary-mongo, mongo, ceph-osd, virt]
requires: [hiera]
required_for: [fuel_pkgs]
parameters:
puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/fuel_pkgs/setup_repositories.pp
puppet_modules: /etc/puppet/modules
timeout: 600
- id: fuel_pkgs
type: puppet
version: 2.0.0
groups: [primary-controller, controller, cinder, cinder-vmware, compute, ceph-osd, primary-mongo, mongo, ironic]
requires: [deploy_start]
required_for: [hiera]
requires: [setup_repositories]
required_for: [globals]
parameters:
puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/fuel_pkgs/fuel_pkgs.pp
puppet_modules: /etc/puppet/modules

View File

@ -1,10 +1,5 @@
notice('MODULAR: hiera.pp')
$deep_merge_package_name = $::osfamily ? {
/RedHat/ => 'rubygem-deep_merge',
/Debian/ => 'ruby-deep-merge',
}
$data_dir = '/etc/hiera'
$override_dir = 'plugins'
$override_dir_path = "${data_dir}/${override_dir}"
@ -73,8 +68,3 @@ file { 'hiera_puppet_config' :
target => $hiera_main_config,
}
# needed to support the 'deeper' merge_behavior setting for hiera
package { 'rubygem-deep_merge':
ensure => present,
name => $deep_merge_package_name,
}

View File

@ -3,7 +3,7 @@
version: 2.0.0
groups: [primary-controller, controller, cinder, cinder-vmware, compute, ceph-osd, primary-mongo, mongo, virt, ironic]
requires: [deploy_start]
required_for: [deploy_end]
required_for: [setup_repositories]
parameters:
puppet_manifest: /etc/puppet/modules/osnailyfacter/modular/hiera/hiera.pp
puppet_modules: /etc/puppet/modules

View File

@ -0,0 +1,21 @@
require 'spec_helper'
describe 'generate_apt_pins' do
let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
let(:subject) {
Puppet::Parser::Functions.function(:generate_apt_pins)
}
it 'should exist' do
expect(subject).to eq 'function_generate_apt_pins'
end
it 'should expect 1 argument' do
expect { scope.function_generate_apt_pins([]) }.to raise_error(ArgumentError)
end
it 'should expect array as given argument' do
expect { scope.function_generate_apt_pins(['foobar']) }.to raise_error(Puppet::ParseError)
end
end

View File

@ -0,0 +1,71 @@
require 'spec_helper'
describe 'generate_apt_sources' do
let(:scope) { PuppetlabsSpec::PuppetInternals.scope }
let(:subject) {
Puppet::Parser::Functions.function(:generate_apt_sources)
}
let(:input) {
[
{'name' => 'ubuntu',
'section' => 'main universe multiverse',
'uri' => 'http://archive.ubuntu.com/ubuntu/',
'priority' => nil,
'suite' => 'trusty',
'type' => 'deb'},
{'name' => 'ubuntu-updates',
'section' => 'main universe multiverse',
'uri' => 'http://archive.ubuntu.com/ubuntu/',
'priority' => nil,
'suite' => 'trusty-updates',
'type' => 'deb'},
{'name' => 'ubuntu-security',
'section' => 'main universe multiverse',
'uri' => 'http://archive.ubuntu.com/ubuntu/',
'priority' => nil,
'suite' => 'trusty-security',
'type' => 'deb'},
]
}
let (:output) {
{
'ubuntu' =>
{
'repos' => 'main universe multiverse',
'release' => 'trusty',
'location' => 'http://archive.ubuntu.com/ubuntu/'
},
'ubuntu-updates' =>
{
'repos' => 'main universe multiverse',
'release' => 'trusty-updates',
'location' => 'http://archive.ubuntu.com/ubuntu/'
},
'ubuntu-security' =>
{
'repos' => 'main universe multiverse',
'release' => 'trusty-security',
'location' => 'http://archive.ubuntu.com/ubuntu/'
},
}
}
it 'should exist' do
expect(subject).to eq 'function_generate_apt_sources'
end
it 'should expect 1 argument' do
expect { scope.function_generate_apt_sources([]) }.to raise_error(ArgumentError)
end
it 'should expect array as given argument' do
expect { scope.function_generate_apt_sources(['foobar']) }.to raise_error(Puppet::ParseError)
end
it 'should return apt::source compatible hash' do
expect(scope.function_generate_apt_sources([input])).to eq(output)
end
end

View File

@ -3,6 +3,29 @@ require 'shared-examples'
manifest = 'fuel_pkgs/fuel_pkgs.pp'
describe manifest do
shared_examples 'catalog' do
it 'should have ruby gem deep_merge installed' do
case facts[:operatingsystem]
when 'Ubuntu'
ruby_deep_merge = 'ruby-deep-merge'
when 'CentOS'
ruby_deep_merge = 'rubygem-deep_merge'
end
should contain_package(ruby_deep_merge).with(
'ensure' => 'present',
)
end
['fuel-ha-utils', 'fuel-misc'].each do |pkg|
it "should install #{pkg} package" do
should contain_package(pkg).with(
'ensure' => 'present'
)
end
end
end
test_ubuntu_and_centos manifest
end

View File

@ -0,0 +1,15 @@
require 'spec_helper'
require 'shared-examples'
manifest = 'fuel_pkgs/setup_repositories.pp'
describe manifest do
before(:each) do
Noop.puppet_function_load :generate_apt_pins
MockFunction.new(:generate_apt_pins) do |function|
allow(function).to receive(:call).and_return({})
end
end
test_ubuntu_and_centos manifest
end

View File

@ -39,19 +39,6 @@ describe manifest do
'target' => '/etc/hiera.yaml'
)
end
it 'should have ruby deep_merge installed' do
case facts[:operatingsystem]
when 'Ubuntu'
package_name = 'ruby-deep-merge'
when 'CentOS'
package_name = 'rubygem-deep_merge'
end
should contain_package('rubygem-deep_merge').with(
'ensure' => 'present',
'name' => package_name
)
end
end
test_ubuntu_and_centos manifest

View File

@ -36,6 +36,7 @@ class Noop
:l3_default_route => '172.16.1.1',
:concat_basedir => '/tmp/',
:l23_os => 'ubuntu',
:lsbdistcodename => 'trusty',
}.merge override_facts
end