Update CI tasks and module fixtures

This change updates the fixtures files for the modules within
fuel-library to use the relative path to the upstream modules used by
fuel-library. With this change we are also adding an initial prep task
that is used by CI to pull down all the modules first and only one time
to reduce the amount of repeated calls to the upstream git repositories.

Included in this change is also an update to exclude all the upstream
modules from the spec and lint tasks as those are maintained in the
upstream modules and should not be done when testing fuel-library.

This change also adds documentation to the README.md on how to run the
tests from the root of fuel-library.

Change-Id: I9cde21b94a8367fd45cf046af48d7618076a385c
Related-Bug: #1535832

Gemfile

@@ -4,6 +4,7 @@ group :development, :test do
   gem 'puppetlabs_spec_helper',               :require => 'false'
   gem 'rspec', '~>3.3',                       :require => 'false'
   gem 'rspec-puppet', '~> 2.2.0',             :require => 'false'
+  gem 'librarian-puppet-simple',              :require => 'false'
   gem 'metadata-json-lint',                   :require => 'false'
   gem 'mime-types', '2.6.2',                  :require => 'false' # 3.0+ requires ruby 2.0
   gem 'puppet-lint-param-docs',               :require => 'false'

README.md

@@ -114,6 +114,74 @@ quality. The fuel-library leverages existing Puppet module rspec tests,
 [bats](https://github.com/sstephenson/bats) tests for bash scripts and noop
 tests for testing the module deployment tasks in fuel-library.
 
+### Module Unit Tests
+---------------------
+
+The modules contained within fuel-library require that the module dependencies
+have been downloaded prior to running their spec tests. Their fixtures.yml have
+been updated to use relative links to the modules contained within the
+deployment/puppet/ folder.  Because of this we have updated the rake tasks for
+the fuel-library root folder to include the ability to download the module
+dependencies as well as run all of the module unit tests with one command. You
+can run the following from the root of the fuel-library to run all module unit
+tests.
+
+```
+bundle install
+bundle exec rake spec
+```
+
+If you only wish to download the module dependencies, you can run the following
+in the root of the fuel-library.
+
+```
+bundle install
+bundle exec rake spec_prep
+```
+
+If you wish to clean up the dependencies, you can run the following in the root
+of the fuel-library.
+
+```
+bundle install
+bundle exec rake spec_clean
+```
+
+Once you have downloaded the dependencies, you can also just work within a
+particular module using the usual rake spec commands if you only want to run a
+single module's unit tests. The upstream module dependencies are not included
+in the unit tests run by this command. They are excluded by having their name
+in the utils/jenkins/modules.disable_rspec file.
+
+### Module Syntax Tests
+-----------------------
+
+From within the fuel-library root, you can run the following to perform the
+syntax checks for the files within fuel-library.
+
+```
+bundle install
+bundle exec rake syntax
+```
+
+This will run syntax checks against all puppet, python, shell and hiera files
+within fuel-libray.
+
+### Module Lint Checks
+
+From within the fuel-library root, you can run the following to run lint on
+all of our puppet files.
+
+```
+bundle install
+bundle exec rake lint
+```
+
+This will run puppet-lint against all of the modules within fuel-library but
+will skip checking the upstream module dependencies. The upstream module
+dependencies are skipped by adding their name to the
+util/jenkins/modules.disable_rake-lint file.
+
 ## Building docs
 ----------------
 

Rakefile

@@ -35,8 +35,14 @@ PuppetSyntax.exclude_paths << "**/vendor/**/*"
 
 
 # Main task list
-task :spec => ["spec:gemfile"]
+task :default => ["common:help"]
+desc "Pull down module dependencies, run tests and cleanup"
+task :spec => ["spec:prep", "spec:gemfile", "spec:clean"]
+task :spec_prep => ["spec:prep"]
+task :spec_clean => ["spec:clean"]
+task :spec_standalone => ["spec:gemfile"]
 # TODO(aschultz): Use puppet-lint for the lint tasks
+desc "Run lint tasks"
 task :lint => ["lint:manual"]
 task :syntax => ["syntax:manifests", "syntax:hiera", "syntax:files", "syntax:templates"]
 
@@ -66,11 +72,30 @@ namespace :common do
       $module_directories << mod
     end
   end
+
+  desc "Display the list of available rake tasks"
+  task :help do
+        system("rake -T")
+  end
 end
 
 
 # our spec task to loop through the modules and run the tests
 namespace :spec do
+
+  desc 'Run prep to install gems and pull down module dependencies'
+  task :prep do |t|
+    library_dir = Dir.pwd
+    ENV['GEM_HOME']="#{library_dir}/.bundled_gems"
+    system("gem install bundler --no-rdoc --no-ri --verbose")
+    system("./deployment/update_modules.sh")
+  end
+
+  desc 'Remove module dependencies'
+  task :clean do |t|
+    system("./deployment/remove_modules.sh")
+  end
+
   desc 'Run spec tasks via module bundler with Gemfile'
   task :gemfile do |t|
     Rake::Task["common:modulelist"].invoke('./utils/jenkins/modules.disable_rspec')
@@ -78,7 +103,6 @@ namespace :spec do
     status = true
 
     ENV['GEM_HOME']="#{library_dir}/.bundled_gems"
-    system("gem install bundler --no-rdoc --no-ri --verbose")
 
     $module_directories.each do |mod|
       next unless File.exists?("#{mod}/Gemfile")

deployment/puppet/ceph/.fixtures.yml

@@ -1,7 +1,4 @@
 fixtures:
-  repositories:
-    firewall:
-      repo: "git://github.com/puppetlabs/puppetlabs-firewall"
-      ref: "1.0.2"
   symlinks:
-    "ceph": "#{source_dir}"
+    ceph: "#{source_dir}"
+    firewall: "#{source_dir}/../firewall"

deployment/puppet/ceph/.gitignore

@@ -0,0 +1 @@
+spec/fixtures

deployment/puppet/cluster/.fixtures.yml

@@ -1,9 +1,8 @@
 fixtures:
-  repositories:
-    'stdlib': 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
   symlinks:
-    'cluster': "#{source_dir}"
-    'corosync': "#{source_dir}/../corosync"
-    'openstack': "#{source_dir}/../openstack"
-    'pacemaker': "#{source_dir}/../pacemaker"
-    'pacemaker_wrappers': "#{source_dir}/../pacemaker_wrappers"
+    cluster: "#{source_dir}"
+    corosync: "#{source_dir}/../corosync"
+    openstack: "#{source_dir}/../openstack"
+    stdlib: "#{source_dir}/../stdlib"
+    pacemaker: "#{source_dir}/../pacemaker"
+    pacemaker_wrappers: "#{source_dir}/../pacemaker_wrappers"

deployment/puppet/cobbler/.fixtures.yml

@@ -1,7 +1,7 @@
 fixtures:
-  repositories:
-    'stdlib': 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
-    'openssl': 'https://review.fuel-infra.org/puppet-modules/puppet-openssl.git'
-    'firewall': 'https://review.fuel-infra.org/puppet-modules/puppetlabs-firewall.git'
   symlinks:
-    'cobbler': "#{source_dir}"
+    cobbler: "#{source_dir}"
+    openssl: "#{source_dir}/../openssl"
+    inifile: "#{source_dir}/../inifile"
+    firewall: "#{source_dir}/../firewall"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/cobbler/.gitignore

@@ -0,0 +1 @@
+spec/fixtures

deployment/puppet/corosync/.fixtures.yml

@@ -1,5 +1,4 @@
 fixtures:
-  repositories:
-    stdlib: "git://github.com/puppetlabs/puppetlabs-stdlib.git"
   symlinks:
     corosync: "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/docker/.fixtures.yml

@@ -1,5 +1,4 @@
 fixtures:
-  repositories:
-    'stdlib': 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
   symlinks:
-    'docker': "#{source_dir}"
+    docker: "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/galera/.fixtures.yml

@@ -1,9 +1,8 @@
 fixtures:
-  forge_modules:
-    firewall: "puppetlabs/firewall"
-    stdlib: "puppetlabs/stdlib"
   symlinks:
     galera: "#{source_dir}"
     pacemaker: "#{source_dir}/../pacemaker"
     tweaks: "#{source_dir}/../tweaks"
     openstack: "#{source_dir}/../openstack"
+    stdlib: "#{source_dir}/../stdlib"
+    firewall: "#{source_dir}/../firewall"

deployment/puppet/haproxy/.fixtures.yml

@@ -1,10 +1,5 @@
 fixtures:
-  repositories:
-    concat:
-      repo: "git://github.com/puppetlabs/puppetlabs-concat.git"
-      ref: '1.0.0'
-    stdlib:
-      repo: "git://github.com/puppetlabs/puppetlabs-stdlib.git"
-      ref: '2.4.0'
   symlinks:
     haproxy: "#{source_dir}"
+    concat: "#{source_dir}/../concat"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/haproxy/spec/defines/listen_spec.rb

@@ -139,7 +139,6 @@ describe 'haproxy::listen' do
     let(:params) do
       {
         :name      => 'apache',
-        :ipaddress => '',
         :bind      => {'10.0.0.1:333' => ['ssl', 'crt', 'public.puppetlabs.com'], '192.168.122.1:8082' => []},
       }
     end

deployment/puppet/haproxy/spec/spec_helper.rb

@@ -1 +1,7 @@
 require 'puppetlabs_spec_helper/module_spec_helper'
+
+RSpec.configure do |c|
+  c.alias_it_should_behave_like_to :it_configures, 'configures'
+  c.alias_it_should_behave_like_to :it_raises, 'raises'
+  c.default_facts = { :concat_basedir => '/var/lib/puppet/concat' }
+end

deployment/puppet/l23network/.fixtures.yml

@@ -1,7 +1,6 @@
 fixtures:
-  repositories:
-    'stdlib': 'git://github.com/puppetlabs/puppetlabs-stdlib.git'
-    'filemapper': 'git://github.com/adrienthebo/puppet-filemapper.git'
-    'sysctl': 'git://github.com/duritong/puppet-sysctl.git'
   symlinks:
-    'l23network': "#{source_dir}"
+    l23network: "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"
+    sysctl: "#{source_dir}/../sysctl"
+    filemapper: "#{source_dir}/../filemapper"

deployment/puppet/mysql/.fixtures.yml

@@ -1,3 +1,3 @@
 fixtures:
   symlinks:
-    "mysql": "#{source_dir}"
+    mysql: "#{source_dir}"

deployment/puppet/nailgun/.fixtures.yml

@@ -1,10 +1,8 @@
 fixtures:
   symlinks:
-    "nailgun": "#{source_dir}"
-    "mcollective": "#{source_dir}/../mcollective"
-    "rabbitmq": "#{source_dir}/../rabbitmq"
-  repositories:
-    "stdlib": "git://github.com/puppetlabs/puppetlabs-stdlib.git"
-    "apt": "git://github.com/puppetlabs/puppetlabs-apt.git"
-    "staging": "git://github.com/nanliu/puppet-staging.git"
-    "erlang": "https://github.com/garethr/garethr-erlang.git"
+    nailgun: "#{source_dir}"
+    mcollective: "#{source_dir}/../mcollective"
+    rabbitmq: "#{source_dir}/../rabbitmq"
+    stdlib: "#{source_dir}/../stdlib"
+    apt: "#{source_dir}/../apt"
+    staging: "#{source_dir}/../staging"

deployment/puppet/openstack/.fixtures.yml

@@ -1,57 +1,33 @@
 fixtures:
-  repositories:
-    'apache':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppetlabs-apache.git'
-      branch: '1.6.0'
-    'apt': 'https://github.com/puppetlabs/puppetlabs-apt.git'
-    'concat': 'https://github.com/ripienaar/puppet-concat.git'
-    'stdlib': 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
-    'sysctl': 'https://github.com/duritong/puppet-sysctl.git'
-    'inifile':  'https://github.com/puppetlabs/puppetlabs-inifile.git'
-    'firewall':  'https://github.com/puppetlabs/puppetlabs-firewall.git'
-    'openstacklib':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-openstacklib.git'
-      branch: '7.0.0-mos-rc4'
-    'horizon':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-horizon.git'
-      branch: '7.0.0-mos-rc3'
-    'keystone':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-keystone.git'
-      branch: '7.0.0-mos-rc3'
-    'neutron':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-neutron.git'
-      branch: '7.0.0-mos-rc4'
-    'nova':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-nova.git'
-      branch: '7.0.0-mos-rc2'
-    'heat':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-heat.git'
-      branch: '7.0.0-mos-rc2'
-    'glance':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-glance.git'
-      branch: '7.0.0-mos-rc2'
-    'ceilometer':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-ceilometer.git'
-      branch: '7.0.0-mos-rc2'
-    'cinder':
-      repo: 'https://review.fuel-infra.org/puppet-modules/openstack-cinder.git'
-      branch: '7.0.0-mos-rc2'
-    'sahara':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-sahara.git'
-      branch: '7.0.0-mos-rc2'
-    'murano':
-      repo: 'https://review.fuel-infra.org/puppet-modules/puppet-murano.git'
-      branch: '7.0.0-mos-rc1'
-    'xinetd':
-      repo: 'https://review.fuel-infra.org/p/puppet-modules/puppetlabs-xinetd.git'
-      branch: '1.5.0'
   symlinks:
-    'openstack': "#{source_dir}"
-    'osnailyfacter': "#{source_dir}/../osnailyfacter"
-    'l23network': "#{source_dir}/../l23network"
-    'rsyslog': "#{source_dir}/../rsyslog"
-    'mysql': "#{source_dir}/../mysql"
-    'tweaks': "#{source_dir}/../tweaks"
-    'corosync': "#{source_dir}/../corosync"
-    'pacemaker': "#{source_dir}/../pacemaker"
-    'haproxy': "#{source_dir}/../haproxy"
+    openstack: "#{source_dir}"
+    apache: "#{source_dir}/../apache"
+    apt: "#{source_dir}/../apt"
+    concat: "#{source_dir}/../concat"
+    stdlib: "#{source_dir}/../stdlib"
+    sysctl: "#{source_dir}/../sysctl"
+    inifile: "#{source_dir}/../inifile"
+    firewall: "#{source_dir}/../firewall"
+    openstacklib: "#{source_dir}/../openstacklib"
+    horizon: "#{source_dir}/../horizon"
+    keystone: "#{source_dir}/../keystone"
+    neutron: "#{source_dir}/../neutron"
+    nova: "#{source_dir}/../nova"
+    heat: "#{source_dir}/../heat"
+    glance: "#{source_dir}/../glance"
+    ceilometer: "#{source_dir}/../ceilometer"
+    cinder: "#{source_dir}/../cinder"
+    sahara: "#{source_dir}/../sahara"
+    murano: "#{source_dir}/../murano"
+    xinetd: "#{source_dir}/../xinetd"
+    osnailyfacter: "#{source_dir}/../osnailyfacter"
+    l23network: "#{source_dir}/../l23network"
+    rsyslog: "#{source_dir}/../rsyslog"
+    mysql: "#{source_dir}/../mysql"
+    tweaks: "#{source_dir}/../tweaks"
+    corosync: "#{source_dir}/../corosync"
+    pacemaker: "#{source_dir}/../pacemaker"
+    haproxy: "#{source_dir}/../haproxy"
+    ceilometer_ha: "#{source_dir}/../ceilometer_ha"
+    pacemaker_wrappers: "#{source_dir}/../pacemaker_wrappers"
+    mellanox_openstack: "#{source_dir}/../mellanox_openstack"

deployment/puppet/osnailyfacter/.fixtures.yml

@@ -1,12 +1,7 @@
 fixtures:
-  repositories:
-    'stdlib': 'git://github.com/puppetlabs/puppetlabs-stdlib.git'
-    'apache':
-      repo: 'https://review.fuel-infra.org/p/puppet-modules/puppetlabs-apache.git'
-      branch: '1.6.0'
-    'concat':
-      repo: 'https://review.fuel-infra.org/p/puppet-modules/puppetlabs-concat.git'
-      branch: '1.2.3'
   symlinks:
-    'osnailyfacter': "#{source_dir}"
-    'l23network': "#{source_dir}/../l23network"
+    osnailyfacter: "#{source_dir}"
+    l23network: "#{source_dir}/../l23network"
+    stdlib: "#{source_dir}/../stdlib"
+    apache: "#{source_dir}/../apache"
+    concat: "#{source_dir}/../concat"

deployment/puppet/pacemaker/.fixtures.yml

@@ -1,5 +1,4 @@
 fixtures:
-  repositories:
-    stdlib: "git://github.com/puppetlabs/puppetlabs-stdlib.git"
   symlinks:
     pacemaker: "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/rabbitmq/.fixtures.yml

@@ -1,9 +1,6 @@
 fixtures:
-  repositories:
-    "stdlib": "git://github.com/puppetlabs/puppetlabs-stdlib.git"
-    "apt": "git://github.com/puppetlabs/puppetlabs-apt.git"
-    "staging": "git://github.com/nanliu/puppet-staging.git"
-    erlang:
-      repo: "https://github.com/garethr/garethr-erlang.git"
   symlinks:
-    "rabbitmq": "#{source_dir}"
+    rabbitmq: "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"
+    apt: "#{source_dir}/../apt"
+    staging: "#{source_dir}/../staging"

deployment/puppet/sysfs/.fixtures.yml

@@ -1,6 +1,5 @@
 fixtures:
-  repositories:
-    'stdlib': 'https://github.com/puppetlabs/puppetlabs-stdlib.git'
   symlinks:
-    'sysfs': "#{source_dir}"
-    'tweaks': "#{source_dir}/../tweaks"
+    sysfs: "#{source_dir}"
+    tweaks: "#{source_dir}/../tweaks"
+    stdlib: "#{source_dir}/../stdlib"

deployment/puppet/tweaks/.fixtures.yml

@@ -1,3 +1,4 @@
 fixtures:
   symlinks:
     'tweaks': "#{source_dir}"
+    stdlib: "#{source_dir}/../stdlib"

utils/jenkins/fuel_unit_tests.sh

@@ -48,6 +48,14 @@ test -f $WORKSPACE/utils/jenkins/modules.disable_rspec || exit 1
 
 export GEM_HOME=$WORKSPACE/.bundled_gems
 
+function get_module_deps {
+  current_dir=`pwd`
+  cd $WORKSPACE/deployment
+  bundle update
+  ./update_modules.sh
+  cd $current_dir
+}
+
 # Function that runs rake spec using bundle
 function rake_spec {
   MODULE=`basename $(pwd)`
@@ -84,6 +92,10 @@ else
   modules=$(git diff --name-only HEAD~ | grep -o 'deployment/puppet/[^/]*/' | sort -u)
 fi
 
+echo "Pulling module dependencies"
+
+get_module_deps
+
 echo "Checking modules: $modules"
 
 for mod in $modules; do

utils/jenkins/modules.disable_rake-lint

@@ -1,6 +1,37 @@
 # List of modules with disabled 'rake lint' check.
 # Such modules will be checked with 'puppet-lint' command.
 apache
+apt
+ceilometer
+cinder
+concat
+datacat
+filemapper
+firewall
+glance
+heat
+horizon
+inifile
+ironic
+keystone
+mcollective
+memcached
+mongodb
+monit
+murano
+neutron
+nova
 ntp
+openssl
+openstacklib
 postgresql
+rsync
+sahara
+ssh
 staging
+stdlib
+swift
+sysctl
+tftp
+vcsrepo
+xinetd

utils/jenkins/modules.disable_rspec

@@ -1,21 +1,28 @@
-apache
 anacron
+apache
+apt
 ceilometer
 cinder
 common
-pacemaker
+concat
+datacat
 docker
 erlang
+filemapper
 firewall
 glance
 heat
 heat_ha
 horizon
 inifile
+ironic
+keystone
+mcollective
 mellanox_openstack
 memcached
 mongodb
 monit
+murano
 mysql
 neutron
 nova
@@ -23,19 +30,25 @@ ntp
 openssl
 openstacklib
 operatingsystem
+pacemaker
 pacemaker_wrappers
 package
 postgresql
 puppet
-puppetmaster
 puppet-network
+puppetmaster
 qpid
 rabbitmq
 rpmcache
+rsync
 rsyslog
+sahara
 ssh
+staging
 stdlib
+swift
 sysctl
+tftp
 vcsrepo
 vmware
 xinetd