Set upgrade level after each upgrade-node

The upgrade_levels:compute parameter have to be set after each deploy
changes operation on controllers otherwise live-migration of instances
will not work properly.

Change-Id: I98a89b51102371f7a50d8a182dc61d671dbf7151
Partial-Bug: #1624341

octane/commands/upgrade_node.py

@@ -92,6 +92,7 @@ def upgrade_node(env_id, node_ids, isolated=False, provision=True, roles=None,
     else:
         env_util.deploy_changes(env, nodes)
     call_handlers('postdeploy')
+    env_util.set_upgrade_levels_for_controllers(env)
 
 
 def copy_patches_folder_to_nailgun():

octane/handlers/upgrade/compute.py

@@ -56,7 +56,7 @@ class ComputeUpgrade(upgrade.UpgradeHandler):
                 controller, False)
 
         seed_version = self.env.data["fuel_version"]
-        openstack_release = magic_consts.UPGRADE_LEVELS[seed_version]
+        openstack_release = nova.get_upgrade_levels(seed_version)
         node_util.add_compute_upgrade_levels(self.node, openstack_release)
 
         ssh.call(["service", "nova-compute", "restart"], node=self.node)

octane/handlers/upgrade/controller.py

@@ -17,7 +17,6 @@ from octane.handlers import upgrade
 from octane.helpers import transformations
 from octane import magic_consts
 from octane.util import env as env_util
-from octane.util import node as node_util
 from octane.util import ssh
 
 LOG = logging.getLogger(__name__)
@@ -62,12 +61,6 @@ class ControllerUpgrade(upgrade.UpgradeHandler):
         return magic_consts.SKIP_CONTROLLER_TASKS
 
     def postdeploy(self):
-        seed_version = self.env.data["fuel_version"]
-        openstack_release = magic_consts.UPGRADE_LEVELS[seed_version]
-        node_util.add_compute_upgrade_levels(self.node, openstack_release)
-
-        node_util.restart_nova_services(self.node)
-
         if self.isolated and self.gateway:
             # From restore_default_gateway
             LOG.info("Deleting default route at node %s",

octane/tests/test_env.py

@@ -348,3 +348,21 @@ def test_util_env(mocker, api, modified, nodes):
     env.get_default_facts.return_value = api
     assert modified == env_util.get_node_default_facts(env, nodes)
     env.get_default_facts.assert_called_once_with('deployment', nodes=nodes)
+
+
+def test_set_upgrade_levels_for_controllers(mocker):
+    env = mock.Mock(data={"fuel_version": "9.1"})
+    node = mock.Mock()
+
+    mock_get_levels = mocker.patch("octane.util.nova.get_upgrade_levels")
+    mock_get_levels.return_value = "liberty"
+    mock_get_conts = mocker.patch("octane.util.env.get_controllers")
+    mock_get_conts.return_value = [node]
+    mock_add = mocker.patch("octane.util.node.add_compute_upgrade_levels")
+    mock_restart = mocker.patch("octane.util.node.restart_nova_services")
+
+    env_util.set_upgrade_levels_for_controllers(env)
+
+    mock_get_levels.assert_called_once_with("9.1")
+    mock_add.assert_called_once_with(node, "liberty")
+    mock_restart.assert_called_once_with(node)

octane/tests/test_upgrade_node.py

@@ -76,6 +76,8 @@ def test_upgrade_node(mocker, node_ids, isolated, provision, roles,
     mock_deploy_nodes = mocker.patch(
         "octane.util.env.deploy_nodes_without_tasks"
     )
+    mock_upgrade_levels = mocker.patch(
+        "octane.util.env.set_upgrade_levels_for_controllers")
     mock_deploy_changes = mocker.patch("octane.util.env.deploy_changes")
     mock_check_isolation = mocker.patch(
         "octane.commands.upgrade_node.check_isolation")
@@ -102,6 +104,7 @@ def test_upgrade_node(mocker, node_ids, isolated, provision, roles,
     else:
         mock_deploy_changes.assert_called_once_with(mock_env, mock_nodes_list)
         not mock_deploy_nodes.called
+    mock_upgrade_levels.assert_called_once_with(mock_env)
 
 
 @pytest.mark.parametrize('node_data,expected_error', [

octane/tests/test_util_nova.py

@@ -218,3 +218,19 @@ def test_get_active_instances(mocker, cmd_out, result, node_fqdn):
         "--status", "ACTIVE",
         "--minimal"],
         controller)
+
+
+@pytest.mark.parametrize(("levels", "version", "result", "error"), [
+    ({"9.1": "liberty"}, "9.1", "liberty", False),
+    ({"9.1": "liberty", "8.0": "kilo"}, "8.0", "kilo", False),
+    ({}, "9.1", None, True),
+])
+def test_get_upgrade_levels(mocker, levels, version, result, error):
+    mocker.patch.dict("octane.magic_consts.UPGRADE_LEVELS", levels, clear=True)
+    if error:
+        msg = ("Could not find suitable upgrade_levels for the "
+               "{version} release.".format(version=version))
+        with pytest.raises(KeyError, message=msg):
+            nova.get_upgrade_levels(version)
+    else:
+        assert nova.get_upgrade_levels(version) == result

octane/util/env.py

@@ -30,6 +30,8 @@ from octane.helpers import transformations
 from octane import magic_consts
 from octane.util import disk
 from octane.util import helpers
+from octane.util import node as node_util
+from octane.util import nova
 from octane.util import ssh
 from octane.util import subprocess
 
@@ -412,3 +414,11 @@ def copy_fuel_keys(source_env_id, seed_env_id):
 def get_generated(env_id):
     return environment_obj.Environment.connection.get_request(
         'clusters/{0}/generated'.format(env_id))
+
+
+def set_upgrade_levels_for_controllers(env):
+    version = env.data["fuel_version"]
+    openstack_release = nova.get_upgrade_levels(version)
+    for node in get_controllers(env):
+        node_util.add_compute_upgrade_levels(node, openstack_release)
+        node_util.restart_nova_services(node)

octane/util/nova.py

@@ -13,6 +13,7 @@
 import logging
 import time
 
+from octane import magic_consts
 from octane.util import ssh
 
 LOG = logging.getLogger(__name__)
@@ -103,3 +104,14 @@ def get_active_instances(controller, node_fqdn):
         controller)
     instances = nova_stdout_parser(instances_stdout)
     return [i["ID"] for i in instances]
+
+
+def get_upgrade_levels(version):
+    try:
+        release = magic_consts.UPGRADE_LEVELS[version]
+    except KeyError:
+        LOG.error("Could not find suitable upgrade_levels for the "
+                  "{version} release.".format(version=version))
+        raise
+    else:
+        return release