Store config when snapshots are taken and restore it with revert

Now 'underlay', 'k8scluster' or 'ccpcluster' can restore
a snapshot if the snapshot name was specified in the
pytest.mark.revert_snapshot(name=<str>).

Snapshot is reverted *only* if:
  - name of the snapshot matches the name of the snapshot that
    is created by fixture:
    * 'underlay' fixture reverts only 'underlay' snapshot
    * 'k8scluster' reverts only 'k8s_deployed' snapshot
    * 'ccpcluster' reverts only 'ccp_deployed' snapshot
  - snapshot actualy exists at the moment

If fixture made changes to the environment, it takes a snapshot
and stores current config to a file 'config_<snapshot_name>.ini'

Other changes:

- Removed 'initial' snapshot. It doesn't makes sense because
  matches to the environment state 'never powered on yet'
- Add 'underlay' snapshot instead of 'initial', that stands for
  ready Underlay layer (operation systems are provisioned and
  configured)
- Move hardware.start() from 'hardware' fixture to 'underlay'
- Use for tests @pytest.mark.revert_snapshot with more suitable
  snapshot names

Change-Id: I15ac734736563e3e40cd850adc49e1b4a070ae20

fuel_ccp_tests/fixtures/ccp_fixtures.py

@@ -17,7 +17,7 @@ from fuel_ccp_tests.helpers import ext
 from fuel_ccp_tests.managers import ccpmanager
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope='function')
 def ccp_actions(config, underlay):
     """Fixture that provides various actions for CCP
 
@@ -30,8 +30,9 @@ def ccp_actions(config, underlay):
     return ccpmanager.CCPManager(config, underlay)
 
 
-@pytest.fixture(scope='session')
-def ccpcluster(config, hardware, underlay, k8scluster, ccp_actions):
+@pytest.fixture(scope='function')
+def ccpcluster(revert_snapshot, config, hardware,
+               underlay, k8scluster, ccp_actions):
     """Fixture to get or install fuel-ccp on k8s environment
 
     :param config: fixture provides oslo.config
@@ -54,9 +55,25 @@ def ccpcluster(config, hardware, underlay, k8scluster, ccp_actions):
     If you want to revert 'ccp_deployed' snapshot, please use mark:
     @pytest.mark.revert_snapshot("ccp_deployed")
     """
-    if config.ccp.os_host is None:
+    # Try to guess environment config for reverted snapshot
+    if revert_snapshot and config.ccp.os_host == '0.0.0.0':
+        config.ccp.os_host = config.k8s.kube_host
+
+    # Install CCP
+    if config.ccp.os_host == '0.0.0.0':
         ccp_actions.install_ccp()
-        config.ccp.os_host = "TODO: get OpenStack endpoints"
+        config.ccp.os_host = config.k8s.kube_host
+
         hardware.create_snapshot(ext.SNAPSHOT.ccp_deployed)
 
+    else:
+        # 1. hardware environment created and powered on
+        # 2. config.underlay.ssh contains SSH access to provisioned nodes
+        #    (can be passed from external config with TESTS_CONFIGS variable)
+        # 3. config.k8s.* options contain access credentials to the already
+        #    installed k8s API endpoint
+        # 4. config.ccp.os_host contains an IP address of CCP admin node
+        #    (not used yet)
+        pass
+
     return ccp_actions

fuel_ccp_tests/fixtures/config_fixtures.py

@@ -12,12 +12,14 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 import os
+
 import pytest
+
 from fuel_ccp_tests import settings_oslo
 
 
 @pytest.fixture(scope='session')
-def config(request):
+def config():
 
     config_files = []
 

fuel_ccp_tests/fixtures/k8s_fixtures.py

@@ -19,7 +19,7 @@ from fuel_ccp_tests import settings
 from fuel_ccp_tests.managers import k8smanager
 
 
-@pytest.fixture(scope='session')
+@pytest.fixture(scope='function')
 def k8s_actions(config, underlay):
     """Fixture that provides various actions for K8S
 
@@ -32,8 +32,9 @@ def k8s_actions(config, underlay):
     return k8smanager.K8SManager(config, underlay)
 
 
-@pytest.fixture(scope='session')
-def k8scluster(request, config, hardware, underlay, k8s_actions):
+@pytest.fixture(scope='function')
+def k8scluster(revert_snapshot, request, config,
+               hardware, underlay, k8s_actions):
     """Fixture to get or install k8s on environment
 
     :param request: fixture provides pytest data
@@ -56,11 +57,25 @@ def k8scluster(request, config, hardware, underlay, k8s_actions):
     If you want to revert 'k8s_deployed' snapshot, please use mark:
     @pytest.mark.revert_snapshot("k8s_deployed")
     """
+    # Try to guess environment config for reverted snapshot
+    if revert_snapshot and config.k8s.kube_host == '0.0.0.0':
+        config.k8s.kube_host = underlay.host_by_node_name(
+            underlay.node_names()[0])
 
-    if config.k8s.kube_host is None:
+    # Create k8s cluster
+    if config.k8s.kube_host == '0.0.0.0':
         kube_settings = getattr(request.instance, 'kube_settings',
                                 settings.DEFAULT_CUSTOM_YAML)
         k8s_actions.install_k8s(custom_yaml=kube_settings)
+
         hardware.create_snapshot(ext.SNAPSHOT.k8s_deployed)
 
+    else:
+        # 1. hardware environment created and powered on
+        # 2. config.underlay.ssh contains SSH access to provisioned nodes
+        #    (can be passed from external config with TESTS_CONFIGS variable)
+        # 3. config.k8s.* options contain access credentials to the already
+        #    installed k8s API endpoint
+        pass
+
     return k8s_actions

fuel_ccp_tests/fixtures/underlay_fixtures.py

@@ -13,6 +13,7 @@
 #    under the License.
 import pytest
 
+from fuel_ccp_tests.helpers import ext
 from fuel_ccp_tests import logger
 from fuel_ccp_tests import settings
 from fuel_ccp_tests.managers import envmanager_devops
@@ -90,27 +91,22 @@ def hardware(request, config):
     return env
 
 
-@pytest.fixture(scope='function', autouse=True)
+@pytest.fixture(scope='function')
 def revert_snapshot(request, hardware):
-    """Fixture to revert environment to snapshot
+    """Extract snapshot name from mark
 
     Marks:
         revert_snapshot - if used this mark with 'name' parameter,
-        use given name as result
+                          use given name as result
 
-    :param request: pytest.python.FixtureRequest
-    :param env: envmanager.EnvironmentManager
+    :rtype string: name of the reverted snapshot or None
     """
     revert_snapshot = request.keywords.get('revert_snapshot', None)
     snapshot_name = extract_name_from_mark(revert_snapshot)
-    if revert_snapshot and snapshot_name:
-        if hardware.has_snapshot(snapshot_name):
-            LOG.info("Reverting snapshot {0}".format(snapshot_name))
-            hardware.revert_snapshot(snapshot_name)
-        else:
-            if revert_snapshot.kwargs.get('strict', True):
-                pytest.fail("Environment doesn't have snapshot "
-                            "named '{}'".format(snapshot_name))
+
+    if snapshot_name and hardware.has_snapshot(snapshot_name):
+        hardware.revert_snapshot(snapshot_name)
+        return snapshot_name
 
 
 @pytest.fixture(scope='function', autouse=True)
@@ -153,8 +149,8 @@ def snapshot(request, hardware):
     request.addfinalizer(test_fin)
 
 
-@pytest.fixture(scope="session")
-def underlay(config, hardware):
+@pytest.fixture(scope="function")
+def underlay(revert_snapshot, config, hardware):
     """Fixture that should provide SSH access to underlay objects.
 
        Input data:
@@ -166,8 +162,27 @@ def underlay(config, hardware):
                                - provide SSH access to underlay nodes using
                                  node names or node IPs.
     """
-
-    if not config.underlay.ssh:
+    # Try to guess environment config for reverted snapshot
+    if revert_snapshot and not config.underlay.ssh:
         config.underlay.ssh = hardware.get_ssh_data()
 
+    # Create Underlay
+    if not config.underlay.ssh:
+        # for devops manager: power on nodes and wait for SSH
+        # for empty manager: do nothing
+        # for maas manager: provision nodes and wait for SSH
+        hardware.start()
+
+        # If config.underlay.ssh wasn't provided from external config, then
+        # try to get necessary data from hardware manager (fuel-devops)
+        config.underlay.ssh = hardware.get_ssh_data()
+
+        hardware.create_snapshot(ext.SNAPSHOT.underlay)
+
+    else:
+        # 1. hardware environment created and powered on
+        # 2. config.underlay.ssh contains SSH access to provisioned nodes
+        #    (can be passed from external config with TESTS_CONFIGS variable)
+        pass
+
     return underlay_ssh_manager.UnderlaySSHManager(config.underlay.ssh)

fuel_ccp_tests/helpers/ext.py

@@ -33,7 +33,7 @@ NETWORK_TYPE = enum(
 )
 
 SNAPSHOT = enum(
-    'initial',
+    'underlay',
     'k8s_deployed',
     'ccp_deployed',
 )

fuel_ccp_tests/managers/envmanager_devops.py

@@ -16,20 +16,23 @@ from devops import error
 from devops.helpers import helpers
 from devops import models
 from django import db
+from oslo_config import cfg
 
 from fuel_ccp_tests import settings
+from fuel_ccp_tests import settings_oslo
 from fuel_ccp_tests.helpers import env_config
 from fuel_ccp_tests.helpers import ext
 from fuel_ccp_tests.helpers import exceptions
 from fuel_ccp_tests import logger
 
-
 LOG = logger.logger
 
 
 class EnvironmentManager(object):
     """Class-helper for creating VMs via devops environments"""
 
+    __config = None
+
     def __init__(self, config=None):
         """Initializing class instance and create the environment
 
@@ -52,9 +55,6 @@ class EnvironmentManager(object):
         except error.DevopsObjNotFound:
             LOG.info("Environment doesn't exist, creating a new one")
             self._create_environment()
-            self.create_snapshot(config.hardware.current_snapshot)
-            LOG.info("Environment created with initial snapshot: {}"
-                     .format(config.hardware.current_snapshot))
 
     @property
     def _devops_config(self):
@@ -117,6 +117,9 @@ class EnvironmentManager(object):
     def create_snapshot(self, name, description=None):
         """Create named snapshot of current env.
 
+        - Create a libvirt snapshots for all nodes in the environment
+        - Save 'config' object to a file 'config_<name>.ini'
+
         :name: string
         """
         LOG.info("Creating snapshot named '{0}'".format(name))
@@ -127,10 +130,16 @@ class EnvironmentManager(object):
             self._env.resume()
         else:
             raise exceptions.EnvironmentIsNotSet()
+        settings_oslo.save_config(self.__config, name)
 
     def revert_snapshot(self, name):
         """Revert snapshot by name
 
+        - Revert a libvirt snapshots for all nodes in the environment
+        - Try to reload 'config' object from a file 'config_<name>.ini'
+          If the file not found, then pass with defaults.
+        - Set <name> as the current state of the environment after reload
+
         :param name: string
         """
         LOG.info("Reverting from snapshot named '{0}'".format(name))
@@ -140,6 +149,13 @@ class EnvironmentManager(object):
             self._env.resume()
         else:
             raise exceptions.EnvironmentIsNotSet()
+
+        try:
+            settings_oslo.reload_snapshot_config(self.__config, name)
+        except cfg.ConfigFilesNotFoundError as conf_err:
+            LOG.error("Config file(s) {0} not found!".format(
+                conf_err.config_files))
+
         self.__config.hardware.current_snapshot = name
 
     def _create_environment(self):
@@ -168,12 +184,12 @@ class EnvironmentManager(object):
             )
             raise exceptions.EnvironmentAlreadyExists(env_name)
         self._env.define()
-        self._start_environment()
+
         LOG.info(
             'Environment "{0}" created and started'.format(env_name)
         )
 
-    def _start_environment(self):
+    def start(self):
         """Method for start environment
 
         """

fuel_ccp_tests/managers/envmanager_empty.py

@@ -12,10 +12,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from oslo_config import cfg
+
+from fuel_ccp_tests import settings_oslo
+
 
 class EnvironmentManagerEmpty(object):
     """Class-helper for creating VMs via devops environments"""
 
+    __config = None
+
     def __init__(self, config=None):
         """Initializing class instance and create the environment
 
@@ -31,13 +37,38 @@ class EnvironmentManagerEmpty(object):
                         "Please provide SSH details in config.underlay.ssh")
 
     def create_snapshot(self, name, description=None):
+        """Store environmetn state into the config object
+
+        - Store the state of the environment <name> to the 'config' object
+        - Save 'config' object to a file 'config_<name>.ini'
+        """
         self.__config.hardware.current_snapshot = name
+        settings_oslo.save_config(self.__config, name)
 
     def revert_snapshot(self, name):
+        """Check the current state <name> of the environment
+
+        - Check that the <name> matches the current state of the environment
+          that is stored in the 'self.__config.hardware.current_snapshot'
+        - Try to reload 'config' object from a file 'config_<name>.ini'
+          If the file not found, then pass with defaults.
+        - Set <name> as the current state of the environment after reload
+
+        :param name: string
+        """
         if self.__config.hardware.current_snapshot != name:
             raise Exception(
                 "EnvironmentManagerEmpty cannot revert nodes from {} to {}"
                 .format(self.__config.hardware.current_snapshot, name))
+        try:
+            settings_oslo.reload_snapshot_config(self.__config, name)
+        except cfg.ConfigFilesNotFoundError:
+            pass
+        self.__config.hardware.current_snapshot = name
+
+    def start(self):
+        """Start environment"""
+        pass
 
     def resume(self):
         """Resume environment"""

fuel_ccp_tests/settings_oslo.py

@@ -12,11 +12,16 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 import copy
+import os
 import pkg_resources
 
 from oslo_config import cfg
-from fuel_ccp_tests.helpers import oslo_cfg_types as ct
+from oslo_config import generator
+
 from fuel_ccp_tests.helpers import ext
+from fuel_ccp_tests.helpers import oslo_cfg_types as ct
+from fuel_ccp_tests import settings
+
 
 _default_conf = pkg_resources.resource_filename(
     __name__, 'templates/default.yaml')
@@ -30,7 +35,7 @@ hardware_opts = [
 
     ct.Cfg('current_snapshot', ct.String(),
            help="Latest environment status name",
-           default=ext.SNAPSHOT.initial),
+           default=ext.SNAPSHOT.underlay),
 ]
 
 
@@ -65,7 +70,7 @@ k8s_deploy_opts = [
     ct.Cfg('deploy_script', ct.String(),
            help="", default=None),
     ct.Cfg('kube_settings', ct.JSONDict(),
-           help="", default=None),
+           help="", default={}),
 ]
 
 # Access credentials to a ready K8S cluster
@@ -75,7 +80,7 @@ k8s_opts = [
     ct.Cfg('kube_admin_pass', ct.String(),
            help="", default="changeme"),
     ct.Cfg('kube_host', ct.IPAddress(),
-           help="", default=None),
+           help="", default='0.0.0.0'),
 ]
 
 
@@ -91,7 +96,7 @@ ccp_deploy_opts = [
 ccp_opts = [
     # TODO: OpenStack endpoints, any other endpoints (galera? rabbit?)
     ct.Cfg('os_host', ct.IPAddress(),
-           help="", default=None),
+           help="", default='0.0.0.0'),
 ]
 
 
@@ -139,7 +144,43 @@ def load_config(config_files):
     return config
 
 
+def reload_snapshot_config(config, snapshot_name):
+    """Reset config to the state from test_config file"""
+    test_config_path = os.path.join(
+        settings.LOGS_DIR, 'config_{0}.ini'.format(snapshot_name))
+    config(args=[], default_config_files=[test_config_path])
+    return config
+
+
 def list_opts():
     """Return a list of oslo.config options available in the fuel-ccp-tests.
     """
     return [(group, copy.deepcopy(opts)) for group, opts in _group_opts]
+
+
+def list_current_opts(config):
+    """Return a list of oslo.config options available in the fuel-ccp-tests.
+    """
+    result_opts = []
+    for group, opts in _group_opts:
+        current_opts = copy.deepcopy(opts)
+        for opt in current_opts:
+            if hasattr(config, group):
+                if hasattr(config[group], opt.name):
+                    opt.default = getattr(config[group], opt.name)
+        result_opts.append((group, current_opts))
+    return result_opts
+
+
+def save_config(config, snapshot_name):
+    test_config_path = os.path.join(
+        settings.LOGS_DIR, 'config_{0}.ini'.format(snapshot_name))
+
+    with open(test_config_path, 'w') as output_file:
+        formatter = generator._OptFormatter(output_file=output_file)
+        for group, opts in list_current_opts(config):
+            formatter.format_group(group)
+            for opt in opts:
+                formatter.format(opt, group, minimal=True)
+                formatter.write('\n')
+            formatter.write('\n')

fuel_ccp_tests/tests/system/pre_commit/test_horizon.py

@@ -33,7 +33,7 @@ class TestServiceHorizon(object):
      /horizon-master/openstack_dashboard/test/integration_tests
     '''.format(registry=settings.REGISTRY, tag='ccp')
 
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
+    @pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
     @pytest.mark.horizon_component
     def test_horizon_component(self, config, underlay,
                                k8scluster, ccpcluster):

fuel_ccp_tests/tests/system/test_ccp_errexit_codes.py

@@ -14,10 +14,11 @@
 
 import pytest
 
+from fuel_ccp_tests.helpers import ext
 from fuel_ccp_tests.logger import logger
 
 
-@pytest.yield_fixture(scope='module')
+@pytest.yield_fixture(scope='function')
 def admin_node(config, underlay, ccpcluster):
     """Return <remote> object to k8s admin node"""
     logger.info("Get SSH access to admin node")
@@ -32,6 +33,7 @@ def clean_repos(node):
 
 @pytest.mark.ccp_cli_errexit_codes
 @pytest.mark.ccp_cli_error_in_fetch
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
 class TestCppCliErrorInFetch(object):
     """Check exit codes when fetch is failing
 
@@ -61,13 +63,13 @@ class TestCppCliErrorInFetch(object):
 
 @pytest.mark.ccp_cli_errexit_codes
 @pytest.mark.ccp_cli_build_exit_code
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
 class TestCppCliBuildExitCode(object):
     """Check exit codes when build is failing
 
        pytest.mark: ccp_cli_build_exit_code
        module pytest.mark: ccp_cli_errexit_codes
     """
-
     @pytest.mark.fail_snapshot
     def test_nonexistent_repo_name(self, admin_node):
         cmd = 'ccp build --components example'
@@ -88,6 +90,7 @@ class TestCppCliBuildExitCode(object):
 
 @pytest.mark.ccp_cli_errexit_codes
 @pytest.mark.ccp_cli_deploy_exit_code
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
 class TestCppCliDeploy(object):
     """Check exit codes when deploy is failing
 

fuel_ccp_tests/tests/system/test_ccp_install_k8s.py

@@ -77,7 +77,7 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
 
     @pytest.mark.k8s_installed_default
     @pytest.mark.snapshot_needed
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
+    @pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
     @pytest.mark.fail_snapshot
     def test_k8s_installed_default(self, underlay, k8s_actions):
         """Test for deploying an k8s environment and check it
@@ -108,7 +108,7 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
 
     @pytest.mark.k8s_installed_with_etcd_on_host
     @pytest.mark.snapshot_needed
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
+    @pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
     @pytest.mark.fail_snapshot
     def test_k8s_installed_with_etcd_on_host(self, underlay, k8s_actions):
         """Test for deploying an k8s environment and check it
@@ -150,7 +150,7 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
 
     @pytest.mark.k8s_installed_with_etcd_in_container
     @pytest.mark.snapshot_needed
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
+    @pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
     @pytest.mark.fail_snapshot
     def test_k8s_installed_with_etcd_in_container(self, underlay, k8s_actions):
         """Test for deploying an k8s environment and check it
@@ -192,7 +192,7 @@ class TestFuelCCPInstaller(base_test.SystemBaseTest,
 
     @pytest.mark.k8s_installed_with_ready_ssh_keys
     @pytest.mark.snapshot_needed
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
+    @pytest.mark.revert_snapshot(ext.SNAPSHOT.underlay)
     @pytest.mark.fail_snapshot
     def test_k8s_installed_with_ready_ssh_keys(self, ssh_keys_dir,
                                                underlay, k8s_actions):

fuel_ccp_tests/tests/system/test_deploy.py

@@ -26,6 +26,7 @@ LOG = logger.logger
 
 
 @pytest.mark.deploy_openstack
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.ccp_deployed)
 class TestDeployOpenstack(base_test.SystemBaseTest):
     """Deploy OpenStack with CCP
 
@@ -67,7 +68,6 @@ class TestDeployOpenstack(base_test.SystemBaseTest):
             assert result['exit_code'] == 0
 
     @pytest.mark.snapshot_needed(name=snapshot_microservices_deployed)
-    @pytest.mark.revert_snapshot(ext.SNAPSHOT.initial)
     @pytest.mark.fail_snapshot
     def test_fuel_ccp_deploy_microservices(self, config, underlay, ccpcluster,
                                            k8scluster):