From 8bf290f4ef044c65e12486d0eb229e17d76a9a85 Mon Sep 17 00:00:00 2001 From: Sergey Novikov Date: Tue, 12 Apr 2016 16:28:55 +0300 Subject: [PATCH] Add test task coverage by lcm tests Implements: blueprint test-granular-task-idempotency Change-Id: I49dabcf0e9b08eec602ce2a7f5102556df10e343 --- doc/base_tests.rst | 5 + fuelweb_test/tests/tests_lcm/base_lcm_test.py | 3 +- .../{ => idempotency}/cinder.yaml | 0 .../{ => idempotency}/compute.yaml | 0 .../{ => idempotency}/controller.yaml | 0 .../{ => idempotency}/compute.yaml | 0 .../{ => idempotency}/controller.yaml | 0 .../{ => idempotency}/mongo.yaml | 0 .../{ => idempotency}/ceph-osd.yaml | 0 .../{ => idempotency}/compute.yaml | 0 .../{ => idempotency}/controller.yaml | 0 .../tests/tests_lcm/test_task_coverage.py | 123 ++++++++++++++++++ 12 files changed, 130 insertions(+), 1 deletion(-) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/{ => idempotency}/cinder.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/{ => idempotency}/compute.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/{ => idempotency}/controller.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/{ => idempotency}/compute.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/{ => idempotency}/controller.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/{ => idempotency}/mongo.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/{ => idempotency}/ceph-osd.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/{ => idempotency}/compute.yaml (100%) rename fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/{ => idempotency}/controller.yaml (100%) create mode 100644 fuelweb_test/tests/tests_lcm/test_task_coverage.py diff --git a/doc/base_tests.rst b/doc/base_tests.rst index 0dec1fd62..d1697e9f5 100644 --- a/doc/base_tests.rst +++ b/doc/base_tests.rst @@ -366,6 +366,11 @@ Test task ensurability .. automodule:: fuelweb_test.tests.tests_lcm.test_ensurability :members: +Test task coverage by LCM tests +------------------------------- +.. automodule:: fuelweb_test.tests.tests_lcm.test_task_coverage + :members: + Gating tests ============ diff --git a/fuelweb_test/tests/tests_lcm/base_lcm_test.py b/fuelweb_test/tests/tests_lcm/base_lcm_test.py index 87b400c54..f02a05493 100644 --- a/fuelweb_test/tests/tests_lcm/base_lcm_test.py +++ b/fuelweb_test/tests/tests_lcm/base_lcm_test.py @@ -41,6 +41,7 @@ def construct_ruby_sym(loader, node): TASKS_BLACKLIST = [ + "pre_hiera_config", "reboot_provisioned_nodes", "hiera", "configure_default_route", @@ -151,7 +152,7 @@ class LCMTestBasic(TestBasic): fixture is loaded :return: a dictionary with loaded fixture data """ - subdir = "" if idmp else "ensurability" + subdir = "idempotency" if idmp else "ensurability" fixture_path = os.path.join( os.path.dirname(__file__), "fixtures", deployment_type, subdir, "{}.yaml".format(role)) diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/cinder.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/cinder.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/cinder.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/cinder.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/compute.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/compute.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/compute.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/compute.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/controller.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/controller.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/controller.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_cinder/idempotency/controller.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/compute.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/compute.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/compute.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/compute.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/controller.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/controller.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/controller.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/controller.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/mongo.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/mongo.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/mongo.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_1_mongo/idempotency/mongo.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/ceph-osd.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/ceph-osd.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/ceph-osd.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/ceph-osd.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/compute.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/compute.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/compute.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/compute.yaml diff --git a/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/controller.yaml b/fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/controller.yaml similarity index 100% rename from fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/controller.yaml rename to fuelweb_test/tests/tests_lcm/fixtures/1_ctrl_1_cmp_3_ceph/idempotency/controller.yaml diff --git a/fuelweb_test/tests/tests_lcm/test_task_coverage.py b/fuelweb_test/tests/tests_lcm/test_task_coverage.py new file mode 100644 index 000000000..4bd86121d --- /dev/null +++ b/fuelweb_test/tests/tests_lcm/test_task_coverage.py @@ -0,0 +1,123 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from proboscis import asserts +from proboscis import test +import yaml + +from fuelweb_test import logger +from fuelweb_test.helpers.decorators import log_snapshot_after_test +from fuelweb_test.tests.base_test_case import SetupEnvironment +from fuelweb_test.tests.base_test_case import TestBasic +from fuelweb_test.tests.tests_lcm.base_lcm_test import TASKS_BLACKLIST + + +EXCLUDED_TASKS_FROM_COVERAGE = [ + "top-role-cinder-vmware", + "top-role-compute-vmware", + "generate_vms" +] + + +@test +class TaskLCMCoverage(TestBasic): + """Test suite for verification of task coverage by LCM tests""" + @staticmethod + def _load_from_file(path, tasks): + """Load fixture from the corresponding yaml file + + :param path: a string, a full path to fixture file + :return: a list of tasks + """ + with open(path) as f: + fixture = yaml.load(f) + for task in fixture['tasks']: + task_name, task_attr = task.items()[0] + if task_attr is None or 'type' not in task_attr: + tasks.append(task_name) + return tasks + + def load_tasks_fixture_file(self, path, subdir, tasks=None): + """Load task fixtures + + :param path: a string, relative path to fixture directory + :param subdir: a string, indicates whether idempotency or ensurability + fixture is uploaded + :param tasks: a list of taken into consideration tasks + :return: a list of tasks + """ + if not tasks: + tasks = [] + if os.path.isdir(path) and os.path.basename(path) == subdir: + for fl in os.listdir(path): + filepath = os.path.join(path, fl) + tasks = list(set(tasks)) + tasks.extend( + self._load_from_file(filepath, tasks) or []) + elif os.path.isdir(path): + for fl in os.listdir(path): + filepath = os.path.join(path, fl) + tasks.extend( + self.load_tasks_fixture_file( + filepath, subdir, tasks) or []) + return tasks + + @test(depends_on=[SetupEnvironment.prepare_release], + groups=['task_idempotency_coverage']) + @log_snapshot_after_test + def task_idempotency_coverage(self): + """Setup master node with custom manifests + + Scenario: + 1. Revert snapshot "ready" + 2. Download task graph + 3. Download task from existing fixture files + 4. Define coverage of fuel task by idempotency tests + + Duration 60m + """ + self.show_step(1) + self.env.revert_snapshot('ready') + + self.show_step(2) + release_id = self.fuel_web.client.get_release_id() + deployment_tasks = self.fuel_web.client.get_release_deployment_tasks( + release_id + ) + puppet_tasks = [task['id'] + for task in deployment_tasks + if task['type'] == 'puppet'] + puppet_tasks = set(puppet_tasks) + + self.show_step(3) + path = os.path.join(os.path.dirname(__file__), "fixtures") + fixture_tasks = set(self.load_tasks_fixture_file(path, 'idempotency')) + + self.show_step(4) + task_blacklist = (set(TASKS_BLACKLIST) | + set(EXCLUDED_TASKS_FROM_COVERAGE)) + general_tasks = puppet_tasks & fixture_tasks + extra_deployment_tasks = puppet_tasks - general_tasks - task_blacklist + extra_fixtures_tasks = fixture_tasks - general_tasks + if extra_fixtures_tasks: + logger.warning('There are extra fixture tasks which are not ' + ' included in the current deployment graph: ' + 'list of tasks: {}'.format(extra_fixtures_tasks)) + asserts.assert_equal(extra_deployment_tasks, set(), + 'There are new deployment tasks which ' + 'appeared in the current deployment graph and ' + 'are not included to test LCM fixtures: list ' + 'of tasks: {}'.format(extra_deployment_tasks))