Fix error in the noops tests framework

* Fix broken remove_yaml function
* Improvement to rspec debugging
* Add more debug and spec for generate_vip function
* Remove unneeded globals cache options from rspec launcher

Change-Id: Ie4388e3fd226dd55a920bdfdbee6a05430ed1805
Closes-Bug: 1478643

.gitignore

@@ -29,3 +29,5 @@ Gemfile.lock
 /rdoc
 
 task_graph.png
+
+*.log

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/generate_vips.rb

@@ -3,31 +3,43 @@ require 'puppetx/l23_hash_tools'
 
 module Puppet::Parser::Functions
   newfunction(:generate_vips) do |args|
+    debug 'Call: generate_vips'
+
     network_metadata = function_hiera_hash ['network_metadata']
     raise Puppet::ParseError, 'Missing or incorrect network_metadata in Hiera!' unless network_metadata.is_a? Hash
+
     this_node_role = function_hiera ['role']
     raise Puppet::ParseError, "Could not get this node's role from Hiera!" if this_node_role.empty?
 
+    network_scheme   = function_hiera_hash ['network_scheme']
+    raise Puppet::ParseError, 'Missing or incorrect network_scheme in Hiera!' unless network_scheme.is_a? Hash
+
     default_node_roles = %w(controller primary-controller)
 
-    # prepare network configuration if it was not prepared
     unless L23network::Scheme.has_config?
-      network_scheme   = function_hiera_hash ['network_scheme']
-      raise Puppet::ParseError, 'Missing or incorrect network_scheme in Hiera!' unless network_scheme.is_a? Hash
+      debug 'Running "prepare_network_config"'
       function_prepare_network_config [ network_scheme ]
     end
 
     vips = network_metadata.fetch 'vips', {}
 
+    debug "VIPS structure: #{vips.inspect}"
+
     vips.each do |name, parameters|
+
+      debug "Processing VIP: '#{name}' with parameters: #{parameters.inspect}"
       network_role = parameters['network_role']
 
-      # skip vip without network role defined
-      next unless network_role
+      unless network_role
+        debug "Skipping vip: '#{name}' because it's 'network_role' parameter is not defined!"
+        next
+      end
       node_roles = parameters.fetch 'node_roles', default_node_roles
 
-      # skip vip if vip is not enables on thie node
-      next unless node_roles.include? this_node_role
+      unless node_roles.include? this_node_role
+        debug "Skipping vip: '#{name}' because it's 'node_roles' parameter doesn't include this node's role: #{this_node_role}!"
+        next
+      end
 
       # create a hash of vip parameters
       vip = {}
@@ -62,7 +74,7 @@ module Puppet::Parser::Functions
       # TODO: get_network_role_property should support gateway and metric
       # gateway = function_get_network_role_property [network_role, 'gateway']
       # gateway_metric = function_get_network_role_property [network_role, 'gateway_metric']
-      
+
       gateway = network_scheme.fetch('endpoints', {}).fetch(vip['nic'], {}).fetch('gateway', nil) unless vip['gateway']
 
       if gateway

deployment/puppet/osnailyfacter/lib/puppet/parser/functions/remove_file.rb

@@ -3,7 +3,14 @@ Puppet::Parser::Functions::newfunction(
     :doc => 'Remove a file during the catalog compilation phase'
 ) do |argv|
   file = argv.first
-  break true unless File.file? file
-  File.unlink file
-  debug("File: '#{file}' was removed!")
+  if File.file? file
+    begin
+      File.unlink file
+      debug "File: '#{file}' was removed!"
+      true
+    rescue => e
+      debug "File: '#{file}' could not remove! (#{e.message})"
+      false
+    end
+  end
 end

tests/noop/Gemfile

@@ -1,12 +1,14 @@
 source 'https://rubygems.org'
 
 group :development, :test do
-  gem 'puppetlabs_spec_helper', :require => false
+  gem 'puppetlabs_spec_helper'
   gem 'puppet-lint', '~> 0.3.2'
   gem 'rspec-puppet', '~> 2.0.0'
+  gem 'rspec-puppet-utils', '~> 2.0.0'
   gem 'openstack', :require => false
-  gem 'netaddr', :require => false
-  gem 'pry'
+  gem 'netaddr'
+  gem 'deep_merge'
+  gem 'pry', :require => false
 end
 
 if puppetversion = ENV['PUPPET_GEM_VERSION']

tests/noop/spec/hosts/cluster-haproxy/cluster-haproxy_spec.rb

@@ -8,19 +8,9 @@ describe manifest do
       Noop.hiera('network_scheme', {}).fetch('endpoints', {})
     end
 
-    let(:scope) do
-      scope = PuppetlabsSpec::PuppetInternals.scope
-      Puppet::Parser::Functions.autoloader.loadall unless scope.respond_to? :function_direct_networks
-      scope
-    end
-
-    let(:other_networks) do
-      scope.function_direct_networks [endpoints]
-    end
-
     it "should delcare cluster::haproxy with correct other_networks" do
       expect(subject).to contain_class('cluster::haproxy').with(
-        'other_networks' => other_networks,
+        'other_networks' => Noop.puppet_function('direct_networks', endpoints),
       )
     end
 

tests/noop/spec/hosts/cluster-vrouter/cluster-vrouter_spec.rb

@@ -9,19 +9,9 @@ describe manifest do
       Noop.hiera('network_scheme', {}).fetch('endpoints', {})
     end
 
-    let(:scope) do
-      scope = PuppetlabsSpec::PuppetInternals.scope
-      Puppet::Parser::Functions.autoloader.loadall unless scope.respond_to? :function_direct_networks
-      scope
-    end
-
-    let(:other_networks) do
-      scope.function_direct_networks [endpoints]
-    end
-
     it "should delcare cluster::vrouter_ocf with correct other_networks" do
       expect(subject).to contain_class('cluster::vrouter_ocf').with(
-        'other_networks' => other_networks,
+        'other_networks' => Noop.puppet_function('direct_networks', endpoints),
       )
     end
 

tests/noop/spec/hosts/database/database_spec.rb

@@ -8,14 +8,8 @@ describe manifest do
       Noop.hiera('network_scheme', {}).fetch('endpoints', {})
     end
 
-    let(:scope) do
-      scope = PuppetlabsSpec::PuppetInternals.scope
-      Puppet::Parser::Functions.autoloader.loadall unless scope.respond_to? :function_direct_networks
-      scope
-    end
-
     let(:other_networks) do
-      scope.function_direct_networks [endpoints, 'br-mgmt', 'netmask']
+      Noop.puppet_function 'direct_networks', endpoints, 'br-mgmt', 'netmask'
     end
 
     let(:access_networks) do

tests/noop/spec/hosts/glance/keystone_spec.rb

@@ -15,13 +15,14 @@ describe manifest do
       public_address  = public_vip
       public_protocol = 'http'
     end
-    auth_name           = Noop.hiera_structure('glance_hash/auth_name', 'glance')
-    password            = Noop.hiera_structure('glance_hash/user_password')
-    configure_endpoint  = Noop.hiera_structure('glance_hash/configure_endpoint', true)
-    configure_user      = Noop.hiera_structure('glance_hash/configure_user', true)
-    configure_user_role = Noop.hiera_structure('glance_hash/configure_user_role', true)
-    region              = Noop.hiera_structure('glance_hash/region', 'RegionOne')
-    service_name        = Noop.hiera_structure('glance_hash/service_name', 'glance')
+
+    auth_name           = Noop.hiera_structure('glance/auth_name', 'glance')
+    password            = Noop.hiera_structure('glance/user_password')
+    configure_endpoint  = Noop.hiera_structure('glance/configure_endpoint', true)
+    configure_user      = Noop.hiera_structure('glance/configure_user', true)
+    configure_user_role = Noop.hiera_structure('glance/configure_user_role', true)
+    region              = Noop.hiera_structure('glance/region', 'RegionOne')
+    service_name        = Noop.hiera_structure('glance/service_name', 'glance')
     public_url          = "#{public_protocol}://#{public_address}:9292"
     admin_url           = "http://#{admin_address}:9292"
 
@@ -31,7 +32,7 @@ describe manifest do
         'password'            => password,
         'configure_endpoint'  => configure_endpoint,
         'configure_user'      => configure_user,
-        'configure_user_role' => configure_user,
+        'configure_user_role' => configure_user_role,
         'service_name'        => service_name,
         'public_url'          => public_url,
         'admin_url'           => admin_url,

tests/noop/spec/hosts/globals/globals_spec.rb

@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'shared-examples'
+
 manifest = 'globals/globals.pp'
 
 describe manifest do
@@ -7,13 +8,11 @@ describe manifest do
   shared_examples 'catalog' do
     it { should contain_file '/etc/hiera/globals.yaml' }
     it 'should save the globals yaml file' do
-      catalog = subject
-      catalog = subject.call if subject.is_a? Proc
-      globals_file_resource = catalog.resource 'file', '/etc/hiera/globals.yaml'
+      globals_yaml_content = Noop.resource_parameter_value subject, 'file', '/etc/hiera/globals.yaml', 'content'
       globals_yaml_path = Noop.globals_yaml_path
-      raise 'No globals file resouerce!' unless globals_file_resource
-      File.open(globals_yaml_path, 'w') { |file| file.write globals_file_resource[:content] }
-      puts "Globals yaml saved to: '#{globals_yaml_path}'"
+      raise 'Could not get globals file content!' unless globals_yaml_content
+      File.open(globals_yaml_path, 'w') { |file| file.write globals_yaml_content }
+      puts "Globals yaml saved to: '#{globals_yaml_path}'" if ENV['SPEC_PUPPET_DEBUG']
     end
   end
 

tests/noop/spec/hosts/keystone/keystone_spec.rb

@@ -11,19 +11,12 @@ describe manifest do
     memcache_roles       = Noop.hiera 'memcache_roles'
     memcache_server_port = Noop.hiera 'memcache_server_port', '11211'
 
-    let(:scope) do
-      scope = PuppetlabsSpec::PuppetInternals.scope
-      Puppet::Parser::Functions.autoloader.loadall unless scope.respond_to? :get_nodes_hash_by_roles
-      Puppet::Parser::Functions.autoloader.loadall unless scope.respond_to? :get_node_to_ipaddr_map_by_network_role
-      scope
-    end
-
     let(:memcache_nodes) do
-      scope.function_get_nodes_hash_by_roles [network_metadata, memcache_roles]
+      Noop.puppet_function 'get_nodes_hash_by_roles', network_metadata, memcache_roles
     end
 
     let(:memcache_address_map) do
-      scope.function_get_node_to_ipaddr_map_by_network_role [memcache_nodes, 'mgmt/memcache']
+      Noop.puppet_function 'get_node_to_ipaddr_map_by_network_role', memcache_nodes, 'mgmt/memcache'
     end
 
     let (:memcache_servers) do

tests/noop/spec/hosts/murano/murano_spec.rb

@@ -54,7 +54,7 @@ describe manifest do
       end
 
       if use_neutron
-        external_network = PuppetlabsSpec::PuppetInternals.scope.function_get_ext_net_name([predefined_networks])
+        external_network = Noop.puppet_function 'get_ext_net_name', predefined_networks
       else
         external_network = nil
       end

tests/noop/spec/hosts/openstack-network/keystone_spec.rb

@@ -13,7 +13,7 @@ describe manifest do
       public_address  = public_vip
       public_protocol = 'http'
     end
-    admin_address       = Noop.hiera_structure('management_vip')
+    admin_address       = Noop.hiera('management_vip')
     admin_protocol      = 'http'
     region              = Noop.hiera_structure('quantum_settings/region', 'RegionOne')
     password            = Noop.hiera_structure('quantum_settings/keystone/admin_password')

tests/noop/spec/hosts/openstack-network/openstack-network-controller_spec.rb

@@ -9,7 +9,7 @@ describe manifest do
 
     use_neutron = Noop.hiera 'use_neutron'
     ceilometer_enabled = Noop.hiera_structure 'ceilometer/enabled'
-    service_endpoint   = Noop.hiera_structure 'service_endpoint'
+    service_endpoint   = Noop.hiera 'service_endpoint'
 
     # Network
     if use_neutron

tests/noop/spec/hosts/roles/mongo_spec.rb

@@ -3,19 +3,19 @@ require 'shared-examples'
 manifest = 'roles/mongo.pp'
 
 describe manifest do
-  before (:each) do
-    Puppet::Parser::Functions::newfunction(:file, :arity => -2, :type => :rvalue) do |vals|
-      return 'key' if vals.first == '/var/lib/astute/mongodb/mongodb.key'
-      raise Puppet::ParseError, "Could not find any files from #{vals.join(", ")}"
+
+  before(:each) do
+    Noop.puppet_function_load :file
+    MockFunction.new(:file) do |function|
+      allow(function).to receive(:call).with(['/var/lib/astute/mongodb/mongodb.key']).and_return('key')
     end
   end
 
   shared_examples 'catalog' do
-
     debug = Noop.hiera 'debug'
     use_syslog = Noop.hiera 'use_syslog'
     ceilometer_hash = Noop.hiera_structure 'ceilometer'
-    nodes_hash = Noop.hiera_structure 'nodes'
+    nodes_hash = Noop.hiera 'nodes'
 
     it 'should configure MongoDB only with replica set' do
       should contain_class('mongodb::server').with('replset' => 'ceilometer')

tests/noop/spec/hosts/virtual_ips/virtual_ips_spec.rb

@@ -3,5 +3,18 @@ require 'shared-examples'
 manifest = 'virtual_ips/virtual_ips.pp'
 
 describe manifest do
+  shared_examples 'catalog' do
+    # TODO: test vip parameters too
+
+    Noop.hiera_structure('network_metadata/vips', {}).each do |name, params|
+      next unless params['network_role']
+      next unless params['node_roles']
+
+      it "should have '#{name}' VIP" do
+        expect(subject).to contain_cluster__virtual_ip(name)
+      end
+    end
+  end
+
   test_ubuntu_and_centos manifest
 end

tests/noop/spec/shared-examples.rb

@@ -1,6 +1,6 @@
 shared_examples 'compile' do
   it do
-    should compile
+    expect(subject).to compile
   end
 end
 
@@ -120,6 +120,7 @@ end
 ###############################################################################
 
 def test_ubuntu_and_centos(manifest_file, force_manifest = false)
+
   # check if task is present in the task list
   unless force_manifest or Noop.manifest_present? manifest_file
     Noop.debug "Manifest '#{manifest_file}' is not enabled on the node '#{Noop.hostname}'. Skipping tests."
@@ -128,9 +129,16 @@ def test_ubuntu_and_centos(manifest_file, force_manifest = false)
 
   # set manifest file
   before(:all) do
+    GC.disable
     Noop.manifest = manifest_file
   end
 
+  after(:each) do
+    GC.enable
+    GC.start
+    GC.disable
+  end
+
   let(:os_name) do
     os = facts[:operatingsystem]
     os = os.downcase if os
@@ -152,6 +160,9 @@ def test_ubuntu_and_centos(manifest_file, force_manifest = false)
 
   if Noop.test_ubuntu?
     context 'on Ubuntu platforms' do
+      before(:all) do
+        Noop.setup_overrides
+      end
       let(:facts) { Noop.ubuntu_facts }
       it_behaves_like 'OS'
     end
@@ -159,6 +170,9 @@ def test_ubuntu_and_centos(manifest_file, force_manifest = false)
 
   if Noop.test_centos?
     context 'on CentOS platforms' do
+      before(:all) do
+        Noop.setup_overrides
+      end
       let(:facts) { Noop.centos_facts }
       it_behaves_like 'OS'
     end

tests/noop/spec/spec_helper.rb

@@ -2,6 +2,7 @@ require 'rubygems'
 require 'puppet'
 require 'hiera_puppet'
 require 'rspec-puppet'
+require 'rspec-puppet-utils'
 require 'puppetlabs_spec_helper/module_spec_helper'
 require 'yaml'
 require 'fileutils'
@@ -113,31 +114,55 @@ module Noop
     false
   end
 
+  ## Hiera ##
+
   def self.hiera_config
+    if ENV['SPEC_PUPPET_DEBUG']
+      logger = 'console'
+    else
+      logger = 'noop'
+    end
     {
-        :backends=> [
+        :backends => [
             'yaml',
         ],
-        :yaml=>{
+        :yaml => {
             :datadir => hiera_data_path,
         },
-        :hierarchy=> [
+        :hierarchy => [
             hiera_data_globals,
             hiera_data_astute,
         ],
-        :logger => 'noop',
+        :logger => logger,
+        :merge_behavior => :deeper,
     }
   end
 
   def self.hiera_object
-    Hiera.new(:config => hiera_config)
+    # @hiera = {} unless @hiera
+    # if @hiera[astute_yaml_name]
+    #   return @hiera[astute_yaml_name]
+    # end
+    # @hiera[astute_yaml_name] = Hiera.new(:config => hiera_config)
+    # return @hiera[astute_yaml_name]
+    return @hiera_object if @hiera_object
+    @hiera_object = Hiera.new(:config => hiera_config)
   end
 
-  def self.hiera(key, default = nil)
-    hiera_object.lookup key, default, {}
+  def self.hiera(key, default = nil, resolution_type = :priority)
+    # def lookup(key, default, scope, order_override=nil, resolution_type=:priority)
+    hiera_object.lookup key, default, {}, nil, resolution_type
   end
 
-  def self.hiera_structure(key, default=nil)
+  def self.hiera_hash(key, default = nil)
+    hiera key, default, :hash
+  end
+
+  def self.hiera_array(key, default = nil)
+    hiera key, default, :array
+  end
+
+  def self.hiera_structure(key, default = nil, separator = '/', resolution_type = :priority)
     path_lookup = lambda do |data, path, default_value|
       break default_value unless data
       break data unless path.is_a? Array and path.any?
@@ -154,20 +179,47 @@ module Noop
       path_lookup.call data[key], path, default_value
     end
 
-    path = key.split '/'
+    path = key.split separator
     key = path.shift
-    data = hiera key
+    data = hiera key, nil, resolution_type
     path_lookup.call data, path, default
   end
 
+  ## Overrides ##
+
   def self.hiera_puppet_override
     class << HieraPuppet
       def hiera
         Noop.hiera_object
       end
     end
+
+    class << Hiera::Config
+      def load(source)
+        @config = Noop.hiera_config
+      end
+      def yaml_load_file(source)
+        @config = Noop.hiera_config
+      end
+      def []=(key, value)
+        @config[key] = value
+      end
+      attr_accessor :config
+    end
   end
 
+  def self.puppet_debug_override
+    Puppet::Util::Log.level = :debug
+    Puppet::Util::Log.newdestination(:console)
+  end
+
+  def self.setup_overrides
+    hiera_puppet_override
+    puppet_debug_override if ENV['SPEC_PUPPET_DEBUG']
+  end
+
+  ## Facts ##
+
   def self.ubuntu_facts
     {
         :fqdn                   => fqdn,
@@ -208,6 +260,8 @@ module Noop
     }
   end
 
+  ## Manifest processing ##
+
   def self.modular_manifests_node_dir
     '/etc/puppet/modules/osnailyfacter/modular'
   end
@@ -230,6 +284,8 @@ module Noop
     @manifest
   end
 
+  ## Test selections ##
+
   def self.test_ubuntu?
     return true unless ENV['SPEC_TEST_UBUNTU'] or ENV['SPEC_TEST_CENTOS']
     true if ENV['SPEC_TEST_UBUNTU']
@@ -290,13 +346,18 @@ module Noop
     end
   end
 
+  ## Catalog helpers ##
+  # TODO: move to Utils
+
   def self.show_catalog(subject)
     catalog = subject
     catalog = subject.call if subject.is_a? Proc
+    puts '===== catalog show start ====='
     catalog.resources.each do |resource|
       puts '=' * 70
       puts resource.to_manifest
     end
+    puts '===== catalog show end ====='
   end
 
   def self.resource_test_template(binding)
@@ -315,6 +376,7 @@ module Noop
   end
 
   def self.catalog_to_spec(subject)
+    puts '===== spec generate start ====='
     catalog = subject
     catalog = subject.call if subject.is_a? Proc
     catalog.resources.each do |resource|
@@ -322,10 +384,35 @@ module Noop
       next if resource.type == 'Class' and %w(Settings main).include? resource.title.to_s
       puts resource_test_template binding
     end
+    puts '===== spec generate end ====='
+  end
+
+  # extract a parameter value from a resource in the catalog
+  def self.resource_parameter_value(subject, resource_type, resource_name, parameter)
+    catalog = subject
+    catalog = subject.call if subject.is_a? Proc
+    resource = catalog.resource resource_type, resource_name
+    fail "No resource type: '#{resource_type}' name: '#{resource_name}' in the catalog!" unless resource
+    resource[parameter.to_sym]
+  end
+
+  # load a puppet function if it's not alreay loaded
+  def self.puppet_function_load(name)
+    name = name.to_sym unless name.is_a? Symbol
+    Puppet::Parser::Functions.autoloader.load name
+  end
+
+  # call a puppet function and return it's value
+  def self.puppet_function(name, *args)
+    name = name.to_sym unless name.is_a? Symbol
+    puppet_scope = PuppetlabsSpec::PuppetInternals.scope
+    puppet_function_load name
+    fail "Could not load Puppet function '#{name}'!" unless puppet_scope.respond_to? "function_#{name}".to_sym
+    puppet_scope.send "function_#{name}".to_sym, args
   end
 
   def self.debug(msg)
-    puts msg if ENV['SPEC_RSPEC_DEBUG']
+    puts msg if ENV['SPEC_PUPPET_DEBUG']
   end
 
   def self.current_spec(example)
@@ -338,6 +425,8 @@ module Noop
     example_group.call example.metadata
   end
 
+  ## Misc utils ##
+
   module Utils
     def self.filter_nodes(hash, name, value)
       hash.select do |it|
@@ -381,28 +470,9 @@ RSpec.configure do |c|
     Facter::Util::Loader.any_instance.stubs(:load_all)
     Facter.clear
     Facter.clear_messages
-
-    # Puppet logs creation
-    if Noop.puppet_logs_dir
-      Puppet::Util::Log.newdestination(Noop.puppet_log_file)
-      Puppet::Util::Log.level = :debug
-    end
-  end
-
-  c.after :each do
-    # Puppet logs cleanup
-    if Noop.puppet_logs_dir
-      Puppet::Util::Log.close_all
-      # Remove puppet log if there are no compilation errors
-      unless example.exception
-        File.unlink Noop.puppet_log_file if File.file? Noop.puppet_log_file
-      end
-    end
   end
 
   c.mock_with :rspec
 
 end
 
-Noop.hiera_puppet_override
-

utils/jenkins/fuel_noop_tests.rb

@@ -42,9 +42,6 @@ module NoopTests
       opts.on('-i', '--individually', 'Run each spec individually') do
         @options[:run_individually] = true
       end
-      opts.on('-g', '--purge_globals', 'Purge globals yaml files') do
-        @options[:purge_globals] = true
-      end
       opts.on('-a', '--astute_yaml_dir DIR', 'Path to astute_yaml folder') do |dir|
         @options[:astute_yaml_dir] = dir
         ENV['SPEC_YAML_DIR'] = dir
@@ -55,8 +52,20 @@ module NoopTests
       opts.on('-S', '--list_specs', 'List all noop spec files') do
         @options[:list_specs] = true
       end
+      opts.on('-g', '--skip_globals', "Don't run 'globals' task") do
+        @options[:skip_globals] = true
+      end
+      opts.on('-A', '--failed_log FILE', 'Log failed specs and yamls to this file') do |file|
+        @options[:failed_log] = file
+      end
       opts.separator 'Filter options:'
       opts.on('-s', '--specs SPEC1,SPEC2', Array, 'Run only these specs. Example: "hosts/hosts_spec.rb"') do |specs|
+        specs = specs.map do |spec|
+          if spec.end_with? '.pp'
+            spec.gsub! '.pp', '_spec.rb'
+          end
+          spec
+        end
         @options[:filter_specs] = specs
       end
       opts.on('-y', '--yamls YAML1,YAML2', Array, 'Run only these yamls. Example: "novanet-primary-controller.yaml"') do |yamls|
@@ -97,9 +106,6 @@ module NoopTests
       opts.on('-R', '--test_centos', 'Run tests for CentOS facts') do
         ENV['SPEC_TEST_CENTOS'] = 'YES'
       end
-      opts.on('-r', '--rspec_debug', 'Show debug messages in rspec tests') do
-        ENV['SPEC_RSPEC_DEBUG'] = 'YES'
-      end
       opts.on('-p', '--puppet_debug', 'Show Puppet debug messages') do
         ENV['SPEC_PUPPET_DEBUG'] = 'YES'
       end
@@ -113,6 +119,7 @@ module NoopTests
       opts.on('-u', '--update-librarian-puppet', 'Run librarian-puppet update in the deployment directory prior to testing') do
         @options[:update_librarian_puppet] = true
       end
+
     end
     optparse.parse!
     @options
@@ -290,7 +297,16 @@ module NoopTests
   # run the globals task for the given yaml file
   # @param [String] astute_yaml YAML file
   def self.globals(astute_yaml)
+    return if options[:skip_globals]
     globals_file = File.join astute_yaml_directory, GLOBALS_PREFIX + astute_yaml
+    if File.file? globals_file
+      begin
+        File.unlink globals_file
+        debug "Globals file  was removed: '#{globals_file}'"
+      rescue => e
+        debug "Could not remove globals file: '#{globals_file}'! (#{e.message})"
+      end
+    end
     rspec spec_path(GLOBALS_SPEC)
   end
 
@@ -431,6 +447,21 @@ module NoopTests
     end
   end
 
+  def self.save_failed_log(results)
+    return unless options[:failed_log]
+    File.open(options[:failed_log], 'w') do |file|
+      results.each do |astute_yaml, yaml_result|
+        if yaml_result[:report].is_a? Hash
+          yaml_result.fetch(:report, {}).each do |spec, spec_result|
+            file.puts "#{astute_yaml} #{spec}" unless spec_result[:success]
+          end
+        else
+          file.puts astute_yaml unless yaml_result[:success]
+        end
+      end
+    end
+  end
+
   # output the test results
   # @param [Hash]
   def self.show_results(results)
@@ -446,18 +477,6 @@ module NoopTests
     end
   end
 
-  # remove all globals yaml files
-  def self.purge_globals
-    Dir.new(astute_yaml_directory).each do |file|
-      path = File.join astute_yaml_directory, file
-      next unless File.file? path
-      next unless file.start_with? GLOBALS_PREFIX
-      next unless file.end_with? '.yaml'
-      debug "Remove: '#{path}'"
-      File.unlink path
-    end
-  end
-
   # the main function
   def self.main
     if options[:console]
@@ -488,15 +507,12 @@ module NoopTests
       exit 0
     end
 
-    if options[:purge_globals]
-      purge_globals
-      exit 0
+    if options[:update_librarian_puppet]
+      prepare_library
     end
 
-    # run librarian-puppet update to prepare the library
-    if options[:update_librarian_puppet]
-        prepare_library
-    end
+    debug "Spec filter: #{options[:filter_specs]}" if options[:filter_specs]
+    debug "Yaml filter: #{options[:filter_yamls]}" if options[:filter_yamls]
 
     success, result = for_every_astute_yaml do
       if options[:run_individually]
@@ -507,6 +523,7 @@ module NoopTests
     end
 
     show_results result
+    save_failed_log result
 
     exit 1 unless success
   end