From 4b30cad7f8660879846fd2de4925fda04d6fb2f6 Mon Sep 17 00:00:00 2001 From: Igor Kalnitsky Date: Fri, 15 May 2015 17:24:33 +0300 Subject: [PATCH] Backup all data before upgrade and stop supervisor Unfortunately, we have a puppet run inside containers on each start and that's cause to the situation when old containers install new packages on restart. We don't have this issue before because our containers won't restarted during upgrade process and now they do (because of upgrading Docker package). As workaround, we have to: * backup all data before starting upgrades * stop supervisor before host system puppet run, so containers won't be restarted and hence won't install new packages Still, it's hackish.. but I don't know better solution now :( Closes-Bug: #1455419 Change-Id: I9522ab561718473873faa0f38c4cb34c3bee64b9 Signed-off-by: Igor Kalnitsky --- fuel_upgrade/clients/supervisor_client.py | 6 +++++ fuel_upgrade/config.py | 2 ++ fuel_upgrade/engines/base.py | 5 ++++ fuel_upgrade/engines/docker_engine.py | 8 +++--- fuel_upgrade/engines/host_system.py | 21 +++++++++++++++ fuel_upgrade/tests/test_cli.py | 1 + .../tests/test_host_system_upgrader.py | 22 +++++++++++++--- fuel_upgrade/tests/test_upgrade_manager.py | 26 ++++++++++++++++++- fuel_upgrade/upgrade.py | 17 ++++++++++-- 9 files changed, 97 insertions(+), 11 deletions(-) diff --git a/fuel_upgrade/clients/supervisor_client.py b/fuel_upgrade/clients/supervisor_client.py index ac1b88f..ac9f6f8 100644 --- a/fuel_upgrade/clients/supervisor_client.py +++ b/fuel_upgrade/clients/supervisor_client.py @@ -119,6 +119,12 @@ class SupervisorClient(object): current_cfg_path) self.supervisor.reloadConfig() + def start_all_services(self): + """Stops all processes + """ + logger.info(u'Start all services') + self.supervisor.startAllProcesses() + def stop_all_services(self): """Stops all processes """ diff --git a/fuel_upgrade/config.py b/fuel_upgrade/config.py index a3a7ed2..c0aaabb 100644 --- a/fuel_upgrade/config.py +++ b/fuel_upgrade/config.py @@ -219,6 +219,8 @@ def get_host_system(update_path, new_version): '/etc/yum.repos.d', '{0}_nailgun.repo'.format(new_version)), + 'repo_aux_config_path': '/etc/yum.repos.d/auxiliary.repo', + 'repos': { 'src': join(update_path, 'repos', '[0-9.-]*'), 'dst': join('/var', 'www', 'nailgun')}, diff --git a/fuel_upgrade/engines/base.py b/fuel_upgrade/engines/base.py index 64deb50..1aad3f7 100644 --- a/fuel_upgrade/engines/base.py +++ b/fuel_upgrade/engines/base.py @@ -39,6 +39,11 @@ class UpgradeEngine(object): """Rollback all the changes, generally used in case of failed upgrade. """ + def backup(self): + """Perform backup actions + """ + return NotImplemented + @abc.abstractproperty def required_free_space(self): """Required free space for upgarde diff --git a/fuel_upgrade/engines/docker_engine.py b/fuel_upgrade/engines/docker_engine.py index bab274a..21d82e5 100644 --- a/fuel_upgrade/engines/docker_engine.py +++ b/fuel_upgrade/engines/docker_engine.py @@ -58,13 +58,13 @@ class DockerUpgrader(UpgradeEngine): self.from_version = self.config.from_version self.supervisor = SupervisorClient(self.config, self.from_version) - def upgrade(self): - """Method with upgarde logic - """ - # Preapre env for upgarde + def backup(self): self.save_db() self.save_cobbler_configs() + def upgrade(self): + """Method with upgarde logic + """ # Point to new supervisor configs and restart supervisor in # order to apply them self.switch_to_new_configs() diff --git a/fuel_upgrade/engines/host_system.py b/fuel_upgrade/engines/host_system.py index 6f73b63..fbfbb3a 100644 --- a/fuel_upgrade/engines/host_system.py +++ b/fuel_upgrade/engines/host_system.py @@ -17,6 +17,7 @@ import glob import os +from fuel_upgrade.clients import SupervisorClient from fuel_upgrade.engines.base import UpgradeEngine from fuel_upgrade import utils @@ -60,6 +61,9 @@ class HostSystemUpgrader(UpgradeEngine): #: packages to be installed before running puppet self.packages = self.host_system_config['install_packages'] + self.supervisor = SupervisorClient( + self.config, self.config.from_version) + @property def required_free_space(self): """Required free space to run upgrade @@ -80,6 +84,14 @@ class HostSystemUpgrader(UpgradeEngine): def upgrade(self): """Run host system upgrade process """ + # The workaround we need in order to fix [1]. In few words, + # when new Docker is installed the containers MUST NOT start + # again because in this case puppet inside them will install + # latest packages and breaks dependencies in some soft. + # + # [1]: https://bugs.launchpad.net/fuel/+bug/1455419 + self.supervisor.stop_all_services() + self.install_repos() self.update_repo() self.install_packages() @@ -92,6 +104,8 @@ class HostSystemUpgrader(UpgradeEngine): self.remove_repo_config() self.remove_repos() + self.supervisor.start_all_services() + def install_repos(self): sources = glob.glob(self.host_system_config['repos']['src']) for source in sources: @@ -136,3 +150,10 @@ class HostSystemUpgrader(UpgradeEngine): """Remove yum repository config """ utils.remove_if_exists(self.repo_config_path) + + # One more damn hack! We have to remove auxiliary repo config + # if we're rollbacking to the Fuel version that doesn't have + # auxiliary repo at all. + if utils.compare_version(self.config.from_version, '6.1') > 0: + utils.remove_if_exists( + self.host_system_config['repo_aux_config_path']) diff --git a/fuel_upgrade/tests/test_cli.py b/fuel_upgrade/tests/test_cli.py index b6d10fa..e13f029 100644 --- a/fuel_upgrade/tests/test_cli.py +++ b/fuel_upgrade/tests/test_cli.py @@ -24,6 +24,7 @@ from fuel_upgrade.cli import run_upgrade from fuel_upgrade.tests.base import BaseTestCase +@mock.patch('fuel_upgrade.engines.host_system.SupervisorClient', mock.Mock()) @mock.patch('fuel_upgrade.cli.CheckerManager', mock.Mock()) @mock.patch('fuel_upgrade.cli.PreUpgradeHookManager', mock.Mock()) @mock.patch('fuel_upgrade.cli.UpgradeManager', mock.Mock()) diff --git a/fuel_upgrade/tests/test_host_system_upgrader.py b/fuel_upgrade/tests/test_host_system_upgrader.py index d3a3640..c638de1 100644 --- a/fuel_upgrade/tests/test_host_system_upgrader.py +++ b/fuel_upgrade/tests/test_host_system_upgrader.py @@ -25,7 +25,9 @@ from fuel_upgrade.tests.base import BaseTestCase class TestHostSystemUpgrader(BaseTestCase): def setUp(self): - self.upgrader = HostSystemUpgrader(self.fake_config) + with mock.patch('fuel_upgrade.engines.host_system.SupervisorClient'): + self.upgrader = HostSystemUpgrader(self.fake_config) + self.upgrader.supervisor = mock.Mock() @mock.patch( 'fuel_upgrade.engines.host_system.HostSystemUpgrader.install_repos') @@ -42,6 +44,7 @@ class TestHostSystemUpgrader(BaseTestCase): self.called_once(install_repos_mock) self.called_once(run_puppet_mock) self.called_once(update_repo_mock) + self.called_once(self.upgrader.supervisor.stop_all_services) mock_utils.exec_cmd.assert_called_with( 'yum install -v -y fuel-9999.0.0') @@ -74,11 +77,22 @@ class TestHostSystemUpgrader(BaseTestCase): self.upgrader.rollback() self.called_once(remove_repo_config_mock) self.called_once(remove_repos_mock) + self.called_once(self.upgrader.supervisor.start_all_services) - @mock.patch('fuel_upgrade.engines.host_system.utils') - def test_remove_repo_config(self, utils_mock): + @mock.patch('fuel_upgrade.engines.host_system.utils.remove_if_exists') + def test_remove_repo_config(self, remove_mock): + self.upgrader.config.from_version = '6.0' self.upgrader.remove_repo_config() - utils_mock.remove_if_exists.assert_called_once_with( + self.assertEqual(remove_mock.call_args_list, [ + mock.call('/etc/yum.repos.d/9999_nailgun.repo'), + mock.call('/etc/yum.repos.d/auxiliary.repo'), + ]) + + @mock.patch('fuel_upgrade.engines.host_system.utils.remove_if_exists') + def test_remove_repo_config_for_fuel_ge_61(self, remove_mock): + self.upgrader.config.from_version = '6.1' + self.upgrader.remove_repo_config() + remove_mock.assert_called_once_with( '/etc/yum.repos.d/9999_nailgun.repo') @mock.patch('fuel_upgrade.engines.host_system.utils.copy') diff --git a/fuel_upgrade/tests/test_upgrade_manager.py b/fuel_upgrade/tests/test_upgrade_manager.py index 56f918f..3cfbcc6 100644 --- a/fuel_upgrade/tests/test_upgrade_manager.py +++ b/fuel_upgrade/tests/test_upgrade_manager.py @@ -76,6 +76,29 @@ class TestUpgradeManager(BaseTestCase): self.method_was_not_called(upgrader._upgraders[2].upgrade) self.method_was_not_called(upgrader._upgraders[2].rollback) + def test_run_backup_for_all_engines(self): + upgrader = UpgradeManager(**self.default_args( + upgraders=[mock.Mock(), mock.Mock()], + )) + upgrader.run() + + self.called_once(upgrader._upgraders[0].backup) + self.called_once(upgrader._upgraders[1].backup) + + def test_run_backup_fails(self): + upgrader = UpgradeManager(**self.default_args( + upgraders=[mock.Mock(), mock.Mock()], + )) + upgrader._upgraders[1].backup.side_effect = Exception('Backup fails') + self.assertRaisesRegexp( + Exception, 'Backup fails', upgrader.run) + + self.called_once(upgrader._upgraders[0].backup) + self.called_once(upgrader._upgraders[1].backup) + + self.method_was_not_called(upgrader._upgraders[0].rollback) + self.method_was_not_called(upgrader._upgraders[1].rollback) + def test_run_upgrade_for_all_engines(self): upgrader = UpgradeManager(**self.default_args( upgraders=[mock.Mock(), mock.Mock()], @@ -123,7 +146,8 @@ class TestUpgradeManager(BaseTestCase): upgrader._on_success.side_effect = Exception('error') upgrader.run() - def test_hostsystem_rollback_is_first(self): + @mock.patch('fuel_upgrade.engines.host_system.SupervisorClient') + def test_hostsystem_rollback_is_first(self, _): args = self.default_args() hostsystem = HostSystemUpgrader(args['config']) diff --git a/fuel_upgrade/upgrade.py b/fuel_upgrade/upgrade.py index 4f7879d..5c7df59 100644 --- a/fuel_upgrade/upgrade.py +++ b/fuel_upgrade/upgrade.py @@ -60,10 +60,23 @@ class UpgradeManager(object): self._version_file.switch_to_new() for upgrader in self._upgraders: + logger.debug('%s: backuping...', upgrader.__class__.__name__) + + try: + upgrader.backup() + except Exception as exc: + logger.exception( + '%s: failed to backup: "%s"', + upgrader.__class__.__name__, exc) + + logger.error('*** UPGRADE FAILED') + raise + + for upgrader in self._upgraders: + logger.debug('%s: upgrading...', upgrader.__class__.__name__) + self._used_upgraders.append(upgrader) try: - logger.debug('%s: upgrading...', upgrader.__class__.__name__) - self._used_upgraders.append(upgrader) upgrader.upgrade() except Exception as exc: logger.exception(