From 1168ef0b5f641cfdc388875d0e7cfca584702851 Mon Sep 17 00:00:00 2001 From: Dmitry Ilyin Date: Tue, 16 Feb 2016 20:10:17 +0300 Subject: [PATCH] Fix ral catalog and improve reporting Change-Id: I66b24b57c5a8c7625fd8f339fe38897c1e62ebf5 --- doc/usage.rst | 4 ++ doc/utility.rst | 83 +++++++++++++++++++------------------ lib/noop/manager/options.rb | 3 ++ lib/noop/manager/report.rb | 43 +++++++++++++++---- lib/noop/task/helpers.rb | 21 ++++++---- 5 files changed, 98 insertions(+), 56 deletions(-) diff --git a/doc/usage.rst b/doc/usage.rst index ca707c2..59ef789 100644 --- a/doc/usage.rst +++ b/doc/usage.rst @@ -199,6 +199,10 @@ You can use **-r** and **-R** options to load the saved reports from the previous run and display the report again, or to load reports and run the tasks that have previously failed after you have tried to somehow fix them. +You can use option **-o** to filter out only failed tasks and examples from +the report and **-O** options to show only tasks without showing the individual +examples. These options can be used together to show only failed tasks. + The task manager can also generate a test report in *jUnit XML* format using the **-x** options. It will be saves to the **report.xml** file in the *reports* folder of the fixtures repository. This file can be used by many tools to diff --git a/doc/utility.rst b/doc/utility.rst index 1d6c2d6..460ed03 100644 --- a/doc/utility.rst +++ b/doc/utility.rst @@ -15,47 +15,48 @@ First, you can use the **-h** options to get the help output.:: Output::: - Usage: noop_tests [options] - Main options: - -j, --jobs JOBS Parallel run RSpec jobs - -g, --globals Run all globals tasks and update saved globals YAML files - -b, --bundle_setup Setup Ruby environment using Bundle - -B, --bundle_exec Use "bundle exec" to run rspec - -l, --update-librarian Run librarian-puppet update in the deployment directory prior to testing - -L, --reset-librarian Reset puppet modules to librarian versions in the deployment directory prior to testing - -o, --report_only_failed Show only failed tasks and examples in the report - -r, --load_saved_reports Read saved report JSON files from the previous run and show tasks report - -R, --run_failed_tasks Run the task that have previously failed again - -M, --list_missing List all task manifests without a spec file - -x, --xunit_report Save report in xUnit format to a file - List options: - -Y, --list_hiera List all hiera yaml files - -S, --list_specs List all task spec files - -F, --list_facts List all facts yaml files - -T, --list_tasks List all task manifest files - Filter options: - -s, --specs SPEC1,SPEC2 Run only these spec files. Example: "hosts/hosts_spec.rb,apache/apache_spec.rb" - -y, --yamls YAML1,YAML2 Run only these hiera yamls. Example: "controller.yaml,compute.yaml" - -f, --facts FACTS1,FACTS2 Run only these facts yamls. Example: "ubuntu.yaml,centos.yaml" - Debug options: - -c, --task_console Run PRY console - -C, --rspec_console Run PRY console in the - -d, --task_debug Show framework debug messages - -D, --puppet_debug Show Puppet debug messages - --debug_log FILE Write all debug messages to this files - -t, --self-check Perform self-check and diagnostic procedures - -p, --pretend Show which tasks will be run without actually running them - Path options: - --dir_root DIR Path to the test root folder - --dir_deployment DIR Path to the test deployment folder - --dir_hiera_yamls DIR Path to the folder with hiera files - --dir_facts_yamls DIR Path to the folder with facts yaml files - --dir_spec_files DIR Path to the folder with task spec files (changing this may break puppet-rspec) - --dir_task_files DIR Path to the folder with task manifest files - --dir_puppet_modules DIR Path to the puppet modules - Spec options: - -A, --catalog_show Show catalog content debug output - -a, --spec_status Show spec status blocks + Usage: noop_tests [options] + Main options: + -j, --jobs JOBS Parallel run RSpec jobs + -g, --globals Run all globals tasks and update saved globals YAML files + -B, --bundle_setup Setup Ruby environment using Bundle + -b, --bundle_exec Use "bundle exec" to run rspec + -l, --update-librarian Run librarian-puppet update in the deployment directory prior to testing + -L, --reset-librarian Reset puppet modules to librarian versions in the deployment directory prior to testing + -o, --report_only_failed Show only failed tasks and examples in the report + -O, --report_only_tasks Show only tasks, skip individual examples + -r, --load_saved_reports Read saved report JSON files from the previous run and show tasks report + -R, --run_failed_tasks Run the task that have previously failed again + -M, --list_missing List all task manifests without a spec file + -x, --xunit_report Save report in xUnit format to a file + List options: + -Y, --list_hiera List all hiera yaml files + -S, --list_specs List all task spec files + -F, --list_facts List all facts yaml files + -T, --list_tasks List all task manifest files + Filter options: + -s, --specs SPEC1,SPEC2 Run only these spec files. Example: "hosts/hosts_spec.rb,apache/apache_spec.rb" + -y, --yamls YAML1,YAML2 Run only these hiera yamls. Example: "controller.yaml,compute.yaml" + -f, --facts FACTS1,FACTS2 Run only these facts yamls. Example: "ubuntu.yaml,centos.yaml" + Debug options: + -c, --task_console Run PRY console + -C, --rspec_console Run PRY console in the + -d, --task_debug Show framework debug messages + -D, --puppet_debug Show Puppet debug messages + --debug_log FILE Write all debug messages to this files + -t, --self-check Perform self-check and diagnostic procedures + -p, --pretend Show which tasks will be run without actually running them + Path options: + --dir_root DIR Path to the test root folder + --dir_deployment DIR Path to the test deployment folder + --dir_hiera_yamls DIR Path to the folder with hiera files + --dir_facts_yamls DIR Path to the folder with facts yaml files + --dir_spec_files DIR Path to the folder with task spec files (changing this may break puppet-rspec) + --dir_task_files DIR Path to the folder with task manifest files + --dir_puppet_modules DIR Path to the puppet modules + Spec options: + -A, --catalog_show Show catalog content debug output + -a, --spec_status Show spec status blocks Shortcut scripts ---------------- diff --git a/lib/noop/manager/options.rb b/lib/noop/manager/options.rb index 909fd10..aaba688 100644 --- a/lib/noop/manager/options.rb +++ b/lib/noop/manager/options.rb @@ -32,6 +32,9 @@ module Noop opts.on('-o', '--report_only_failed', 'Show only failed tasks and examples in the report') do @options[:report_only_failed] = true end + opts.on('-O', '--report_only_tasks', 'Show only tasks, skip individual examples') do + @options[:report_only_tasks] = true + end opts.on('-r', '--load_saved_reports', 'Read saved report JSON files from the previous run and show tasks report') do @options[:load_saved_reports] = true end diff --git a/lib/noop/manager/report.rb b/lib/noop/manager/report.rb index f67a62f..18fe87c 100644 --- a/lib/noop/manager/report.rb +++ b/lib/noop/manager/report.rb @@ -2,6 +2,8 @@ require 'erb' require 'colorize' require 'rexml/document' +# TODO: cli report should use data from tasks_report_structure instead of reimplementing it + module Noop class Manager STATUS_STRING_LENGTH = 8 @@ -60,7 +62,7 @@ module Noop line += "#{task.file_base_facts.to_s.ljust max_length_facts + 1}" line += "#{task.file_base_hiera.to_s.ljust max_length_hiera + 1}" output line - output_task_examples task + output_task_examples task unless options[:report_only_tasks] end def output_task_examples(task) @@ -131,21 +133,46 @@ module Noop end def output_task_totals - tasks = 0 - failed = 0 - pending = 0 + count = { + :total => 0, + :failed => 0, + :pending => 0, + } task_list.each do |task| - pending += 1 if task.pending? - failed += 1 if task.failed? - tasks += 1 + count[:pending] += 1 if task.pending? + count[:failed] += 1 if task.failed? + count[:total] += 1 end - output "Tasks: #{tasks} Failed: #{failed} Pending: #{pending}" + output "Tasks: #{count[:total]} Failed: #{count[:failed]} Pending: #{count[:pending]}" + end + + def output_examples_total + count = { + :total => 0, + :failed => 0, + :pending => 0, + } + + task_list.each do |task| + examples = task.report['examples'] + next unless examples.is_a? Array + examples.each do |example| + count[:total] += 1 + if example['status'] == 'failed' + count[:failed] += 1 + elsif example['status'] == 'pending' + count[:pending] += 1 + end + end + end + output "Examples: #{count[:total]} Failed: #{count[:failed]} Pending: #{count[:pending]}" end def task_report task_list.each do |task| output_task_status task end + output_examples_total unless options[:report_only_tasks] output_task_totals end diff --git a/lib/noop/task/helpers.rb b/lib/noop/task/helpers.rb index f4ac3e1..a1169f2 100644 --- a/lib/noop/task/helpers.rb +++ b/lib/noop/task/helpers.rb @@ -39,20 +39,27 @@ module Noop end # convert resource catalog to a RAL catalog - # and run "generate" hook for all resources + # and run both "generate" functions for each resource + # that has it and then add results to the catalog + # @return def create_ral_catalog(context) catalog = context.subject catalog = catalog.call if catalog.is_a? Proc ral_catalog = catalog.to_ral + generate_functions = [:generate, :eval_generate] + ral_catalog.resources.each do |resource| - next unless resource.respond_to? :generate - generated = resource.generate - next unless generated.is_a? Array - generated.each do |generated_resource| - next unless generated_resource.is_a? Puppet::Type - ral_catalog.add_resource generated_resource + generate_functions.each do |function_name| + next unless resource.respond_to? function_name + generated = resource.send function_name + next unless generated.is_a? Array + generated.each do |generated_resource| + next unless generated_resource.is_a? Puppet::Type + ral_catalog.add_resource generated_resource + end end end + lambda { ral_catalog } end