Backup all data before upgrade and stop supervisor

Unfortunately, we have a puppet run inside containers on each start and
that's cause to the situation when old containers install new packages
on restart. We don't have this issue before because our containers won't
restarted during upgrade process and now they do (because of upgrading
Docker package).

As workaround, we have to:

* backup all data before starting upgrades
* stop supervisor before host system puppet run, so containers won't be
  restarted and hence won't install new packages

Still, it's hackish.. but I don't know better solution now :(

Closes-Bug: #1455419

Change-Id: I9522ab561718473873faa0f38c4cb34c3bee64b9
Signed-off-by: Igor Kalnitsky <igor@kalnitsky.org>

fuel_upgrade/clients/supervisor_client.py

@@ -119,6 +119,12 @@ class SupervisorClient(object):
             current_cfg_path)
         self.supervisor.reloadConfig()
 
+    def start_all_services(self):
+        """Stops all processes
+        """
+        logger.info(u'Start all services')
+        self.supervisor.startAllProcesses()
+
     def stop_all_services(self):
         """Stops all processes
         """

fuel_upgrade/config.py

@@ -219,6 +219,8 @@ def get_host_system(update_path, new_version):
             '/etc/yum.repos.d',
             '{0}_nailgun.repo'.format(new_version)),
 
+        'repo_aux_config_path': '/etc/yum.repos.d/auxiliary.repo',
+
         'repos': {
             'src': join(update_path, 'repos', '[0-9.-]*'),
             'dst': join('/var', 'www', 'nailgun')},

fuel_upgrade/engines/base.py

@@ -39,6 +39,11 @@ class UpgradeEngine(object):
         """Rollback all the changes, generally used in case of failed upgrade.
         """
 
+    def backup(self):
+        """Perform backup actions
+        """
+        return NotImplemented
+
     @abc.abstractproperty
     def required_free_space(self):
         """Required free space for upgarde

fuel_upgrade/engines/docker_engine.py

@@ -58,13 +58,13 @@ class DockerUpgrader(UpgradeEngine):
         self.from_version = self.config.from_version
         self.supervisor = SupervisorClient(self.config, self.from_version)
 
-    def upgrade(self):
-        """Method with upgarde logic
-        """
-        # Preapre env for upgarde
+    def backup(self):
         self.save_db()
         self.save_cobbler_configs()
 
+    def upgrade(self):
+        """Method with upgarde logic
+        """
         # Point to new supervisor configs and restart supervisor in
         # order to apply them
         self.switch_to_new_configs()

fuel_upgrade/engines/host_system.py

@@ -17,6 +17,7 @@
 import glob
 import os
 
+from fuel_upgrade.clients import SupervisorClient
 from fuel_upgrade.engines.base import UpgradeEngine
 from fuel_upgrade import utils
 
@@ -60,6 +61,9 @@ class HostSystemUpgrader(UpgradeEngine):
         #: packages to be installed before running puppet
         self.packages = self.host_system_config['install_packages']
 
+        self.supervisor = SupervisorClient(
+            self.config, self.config.from_version)
+
     @property
     def required_free_space(self):
         """Required free space to run upgrade
@@ -80,6 +84,14 @@ class HostSystemUpgrader(UpgradeEngine):
     def upgrade(self):
         """Run host system upgrade process
         """
+        # The workaround we need in order to fix [1]. In few words,
+        # when new Docker is installed the containers MUST NOT start
+        # again because in this case puppet inside them will install
+        # latest packages and breaks dependencies in some soft.
+        #
+        # [1]: https://bugs.launchpad.net/fuel/+bug/1455419
+        self.supervisor.stop_all_services()
+
         self.install_repos()
         self.update_repo()
         self.install_packages()
@@ -92,6 +104,8 @@ class HostSystemUpgrader(UpgradeEngine):
         self.remove_repo_config()
         self.remove_repos()
 
+        self.supervisor.start_all_services()
+
     def install_repos(self):
         sources = glob.glob(self.host_system_config['repos']['src'])
         for source in sources:
@@ -136,3 +150,10 @@ class HostSystemUpgrader(UpgradeEngine):
         """Remove yum repository config
         """
         utils.remove_if_exists(self.repo_config_path)
+
+        # One more damn hack! We have to remove auxiliary repo config
+        # if we're rollbacking to the Fuel version that doesn't have
+        # auxiliary repo at all.
+        if utils.compare_version(self.config.from_version, '6.1') > 0:
+            utils.remove_if_exists(
+                self.host_system_config['repo_aux_config_path'])

fuel_upgrade/tests/test_cli.py

@@ -24,6 +24,7 @@ from fuel_upgrade.cli import run_upgrade
 from fuel_upgrade.tests.base import BaseTestCase
 
 
+@mock.patch('fuel_upgrade.engines.host_system.SupervisorClient', mock.Mock())
 @mock.patch('fuel_upgrade.cli.CheckerManager', mock.Mock())
 @mock.patch('fuel_upgrade.cli.PreUpgradeHookManager', mock.Mock())
 @mock.patch('fuel_upgrade.cli.UpgradeManager', mock.Mock())

fuel_upgrade/tests/test_host_system_upgrader.py

@@ -25,7 +25,9 @@ from fuel_upgrade.tests.base import BaseTestCase
 class TestHostSystemUpgrader(BaseTestCase):
 
     def setUp(self):
-        self.upgrader = HostSystemUpgrader(self.fake_config)
+        with mock.patch('fuel_upgrade.engines.host_system.SupervisorClient'):
+            self.upgrader = HostSystemUpgrader(self.fake_config)
+        self.upgrader.supervisor = mock.Mock()
 
     @mock.patch(
         'fuel_upgrade.engines.host_system.HostSystemUpgrader.install_repos')
@@ -42,6 +44,7 @@ class TestHostSystemUpgrader(BaseTestCase):
         self.called_once(install_repos_mock)
         self.called_once(run_puppet_mock)
         self.called_once(update_repo_mock)
+        self.called_once(self.upgrader.supervisor.stop_all_services)
         mock_utils.exec_cmd.assert_called_with(
             'yum install -v -y fuel-9999.0.0')
 
@@ -74,11 +77,22 @@ class TestHostSystemUpgrader(BaseTestCase):
         self.upgrader.rollback()
         self.called_once(remove_repo_config_mock)
         self.called_once(remove_repos_mock)
+        self.called_once(self.upgrader.supervisor.start_all_services)
 
-    @mock.patch('fuel_upgrade.engines.host_system.utils')
-    def test_remove_repo_config(self, utils_mock):
+    @mock.patch('fuel_upgrade.engines.host_system.utils.remove_if_exists')
+    def test_remove_repo_config(self, remove_mock):
+        self.upgrader.config.from_version = '6.0'
         self.upgrader.remove_repo_config()
-        utils_mock.remove_if_exists.assert_called_once_with(
+        self.assertEqual(remove_mock.call_args_list, [
+            mock.call('/etc/yum.repos.d/9999_nailgun.repo'),
+            mock.call('/etc/yum.repos.d/auxiliary.repo'),
+        ])
+
+    @mock.patch('fuel_upgrade.engines.host_system.utils.remove_if_exists')
+    def test_remove_repo_config_for_fuel_ge_61(self, remove_mock):
+        self.upgrader.config.from_version = '6.1'
+        self.upgrader.remove_repo_config()
+        remove_mock.assert_called_once_with(
             '/etc/yum.repos.d/9999_nailgun.repo')
 
     @mock.patch('fuel_upgrade.engines.host_system.utils.copy')

fuel_upgrade/tests/test_upgrade_manager.py

@@ -76,6 +76,29 @@ class TestUpgradeManager(BaseTestCase):
         self.method_was_not_called(upgrader._upgraders[2].upgrade)
         self.method_was_not_called(upgrader._upgraders[2].rollback)
 
+    def test_run_backup_for_all_engines(self):
+        upgrader = UpgradeManager(**self.default_args(
+            upgraders=[mock.Mock(), mock.Mock()],
+        ))
+        upgrader.run()
+
+        self.called_once(upgrader._upgraders[0].backup)
+        self.called_once(upgrader._upgraders[1].backup)
+
+    def test_run_backup_fails(self):
+        upgrader = UpgradeManager(**self.default_args(
+            upgraders=[mock.Mock(), mock.Mock()],
+        ))
+        upgrader._upgraders[1].backup.side_effect = Exception('Backup fails')
+        self.assertRaisesRegexp(
+            Exception, 'Backup fails', upgrader.run)
+
+        self.called_once(upgrader._upgraders[0].backup)
+        self.called_once(upgrader._upgraders[1].backup)
+
+        self.method_was_not_called(upgrader._upgraders[0].rollback)
+        self.method_was_not_called(upgrader._upgraders[1].rollback)
+
     def test_run_upgrade_for_all_engines(self):
         upgrader = UpgradeManager(**self.default_args(
             upgraders=[mock.Mock(), mock.Mock()],
@@ -123,7 +146,8 @@ class TestUpgradeManager(BaseTestCase):
         upgrader._on_success.side_effect = Exception('error')
         upgrader.run()
 
-    def test_hostsystem_rollback_is_first(self):
+    @mock.patch('fuel_upgrade.engines.host_system.SupervisorClient')
+    def test_hostsystem_rollback_is_first(self, _):
         args = self.default_args()
 
         hostsystem = HostSystemUpgrader(args['config'])

fuel_upgrade/upgrade.py

@@ -60,10 +60,23 @@ class UpgradeManager(object):
         self._version_file.switch_to_new()
 
         for upgrader in self._upgraders:
+            logger.debug('%s: backuping...', upgrader.__class__.__name__)
+
+            try:
+                upgrader.backup()
+            except Exception as exc:
+                logger.exception(
+                    '%s: failed to backup: "%s"',
+                    upgrader.__class__.__name__, exc)
+
+                logger.error('*** UPGRADE FAILED')
+                raise
+
+        for upgrader in self._upgraders:
+            logger.debug('%s: upgrading...', upgrader.__class__.__name__)
+            self._used_upgraders.append(upgrader)
 
             try:
-                logger.debug('%s: upgrading...', upgrader.__class__.__name__)
-                self._used_upgraders.append(upgrader)
                 upgrader.upgrade()
             except Exception as exc:
                 logger.exception(