Fix ral catalog and improve reporting

Change-Id: I66b24b57c5a8c7625fd8f339fe38897c1e62ebf5

doc/usage.rst

@@ -199,6 +199,10 @@ You can use **-r** and **-R** options to load the saved reports from the
 previous run and display the report again, or to load reports and run the tasks
 that have previously failed after you have tried to somehow fix them.
 
+You can use option **-o** to filter out only failed tasks and examples from
+the report and **-O** options to show only tasks without showing the individual
+examples. These options can be used together to show only failed tasks.
+
 The task manager can also generate a test report in *jUnit XML* format using
 the **-x** options. It will be saves to the **report.xml** file in the *reports*
 folder of the fixtures repository. This file can be used by many tools to

doc/utility.rst

@@ -19,11 +19,12 @@ Output:::
     Main options:
         -j, --jobs JOBS                  Parallel run RSpec jobs
         -g, --globals                    Run all globals tasks and update saved globals YAML files
-      -b, --bundle_setup               Setup Ruby environment using Bundle
-      -B, --bundle_exec                Use "bundle exec" to run rspec
+        -B, --bundle_setup               Setup Ruby environment using Bundle
+        -b, --bundle_exec                Use "bundle exec" to run rspec
         -l, --update-librarian           Run librarian-puppet update in the deployment directory prior to testing
         -L, --reset-librarian            Reset puppet modules to librarian versions in the deployment directory prior to testing
         -o, --report_only_failed         Show only failed tasks and examples in the report
+        -O, --report_only_tasks          Show only tasks, skip individual examples
         -r, --load_saved_reports         Read saved report JSON files from the previous run and show tasks report
         -R, --run_failed_tasks           Run the task that have previously failed again
         -M, --list_missing               List all task manifests without a spec file

lib/noop/manager/options.rb

@@ -32,6 +32,9 @@ module Noop
         opts.on('-o', '--report_only_failed', 'Show only failed tasks and examples in the report') do
           @options[:report_only_failed] = true
         end
+        opts.on('-O', '--report_only_tasks', 'Show only tasks, skip individual examples') do
+          @options[:report_only_tasks] = true
+        end
         opts.on('-r', '--load_saved_reports', 'Read saved report JSON files from the previous run and show tasks report') do
           @options[:load_saved_reports] = true
         end

lib/noop/manager/report.rb

@@ -2,6 +2,8 @@ require 'erb'
 require 'colorize'
 require 'rexml/document'
 
+# TODO: cli report should use data from tasks_report_structure instead of reimplementing it
+
 module Noop
   class Manager
     STATUS_STRING_LENGTH = 8
@@ -60,7 +62,7 @@ module Noop
       line += "#{task.file_base_facts.to_s.ljust max_length_facts + 1}"
       line += "#{task.file_base_hiera.to_s.ljust max_length_hiera + 1}"
       output line
-      output_task_examples task
+      output_task_examples task unless options[:report_only_tasks]
     end
 
     def output_task_examples(task)
@@ -131,21 +133,46 @@ module Noop
     end
 
     def output_task_totals
-      tasks = 0
-      failed = 0
-      pending = 0
+      count = {
+          :total => 0,
+          :failed => 0,
+          :pending => 0,
+      }
       task_list.each do |task|
-        pending += 1 if task.pending?
-        failed += 1 if task.failed?
-        tasks += 1
+        count[:pending] += 1 if task.pending?
+        count[:failed] += 1 if task.failed?
+        count[:total] += 1
       end
-      output "Tasks: #{tasks} Failed: #{failed} Pending: #{pending}"
+      output "Tasks: #{count[:total]} Failed: #{count[:failed]} Pending: #{count[:pending]}"
+    end
+
+    def output_examples_total
+      count = {
+          :total => 0,
+          :failed => 0,
+          :pending => 0,
+      }
+
+      task_list.each do |task|
+        examples = task.report['examples']
+        next unless examples.is_a? Array
+        examples.each do |example|
+          count[:total] += 1
+          if example['status'] == 'failed'
+            count[:failed] += 1
+          elsif example['status'] == 'pending'
+            count[:pending] += 1
+          end
+        end
+      end
+      output "Examples: #{count[:total]} Failed: #{count[:failed]} Pending: #{count[:pending]}"
     end
 
     def task_report
       task_list.each do |task|
         output_task_status task
       end
+      output_examples_total unless options[:report_only_tasks]
       output_task_totals
     end
 

lib/noop/task/helpers.rb

@@ -39,20 +39,27 @@ module Noop
     end
 
     # convert resource catalog to a RAL catalog
-    # and run "generate" hook for all resources
+    # and run both "generate" functions for each resource
+    # that has it and then add results to the catalog
+    # @return <Lambda>
     def create_ral_catalog(context)
       catalog = context.subject
       catalog = catalog.call if catalog.is_a? Proc
       ral_catalog = catalog.to_ral
+      generate_functions = [:generate, :eval_generate]
+
       ral_catalog.resources.each do |resource|
-        next unless resource.respond_to? :generate
-        generated = resource.generate
+        generate_functions.each do |function_name|
+          next unless resource.respond_to? function_name
+          generated = resource.send function_name
           next unless generated.is_a? Array
           generated.each do |generated_resource|
             next unless generated_resource.is_a? Puppet::Type
             ral_catalog.add_resource generated_resource
           end
         end
+      end
+
       lambda { ral_catalog }
     end