Add cleanup command

Remove upgrade_levels for each node running on environment and restart
all nova running services on each node.

Change-Id: Ic5294b481f3c60156346ccec6451a17548ea0b7e
(cherry picked from commit 6aff9a9bb7)

octane/commands/cleanup.py

@@ -0,0 +1,45 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+from cliff import command as cmd
+from fuelclient import objects
+
+from octane.util import env as env_util
+from octane.util import node as node_util
+
+LOG = logging.getLogger(__name__)
+
+
+def cleanup_environment(env_id):
+    env = objects.Environment(env_id)
+
+    nodes = env_util.get_nodes(env, ['controller', 'compute'])
+    for node in nodes:
+        node_util.remove_compute_upgrade_levels(node)
+        node_util.restart_nova_services(node)
+
+
+class CleanupCommand(cmd.Command):
+    """Cleanup upgraded environment"""
+
+    def get_parser(self, prog_name):
+        parser = super(CleanupCommand, self).get_parser(prog_name)
+
+        parser.add_argument(
+            'env', type=int, metavar='ENV_ID',
+            help="ID of environment to cleanup")
+        return parser
+
+    def take_action(self, parsed_args):
+        cleanup_environment(parsed_args.env)

octane/handlers/upgrade/controller.py

@@ -83,15 +83,7 @@ class ControllerUpgrade(upgrade.UpgradeHandler):
         openstack_release = magic_consts.VERSIONS[orig_version]
         node_util.add_compute_upgrade_levels(self.node, openstack_release)
 
-        nova_services = ssh.call_output(
-            ["bash", "-c",
-                "initctl list | "
-                "awk '/nova/ && /start/ {print $1}' | tr '\n' ' '"],
-            node=self.node
-        )
-
-        for nova_service in nova_services.split():
-            ssh.call(["service", nova_service, "restart"], node=self.node)
+        node_util.restart_nova_services(self.node)
 
         if self.isolated and self.gateway:
             # From restore_default_gateway

octane/tests/test_cleanup.py

@@ -0,0 +1,37 @@
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import mock
+import pytest
+
+from octane.commands import cleanup
+
+
+@pytest.mark.parametrize("env_id", [1])
+@pytest.mark.parametrize("node_count", [0, 1, 10])
+def test_cleanup_env(mocker, env_id, node_count):
+    env = mock.Mock()
+    get_env_mock = mocker.patch(
+        "fuelclient.objects.Environment", return_value=env)
+    nodes = [mock.MagicMock() for _ in range(node_count)]
+    get_nodes_mock = mocker.patch(
+        "octane.util.env.get_nodes", return_value=nodes)
+    remove_compute_mock = mocker.patch(
+        "octane.util.node.remove_compute_upgrade_levels")
+    restart_service_mock = mocker.patch(
+        "octane.util.node.restart_nova_services")
+    cleanup.cleanup_environment(env_id)
+    for node in nodes:
+        remove_compute_mock.assert_any_call(node)
+        restart_service_mock.assert_any_call(node)
+    get_nodes_mock.assert_called_once_with(env, ["controller", "compute"])
+    get_env_mock.assert_called_once_with(env_id)

octane/tests/test_util_node.py

@@ -174,3 +174,29 @@ def test_get_nova_node_handle(mocker, node_data, fuel_version, expected_name):
     else:
         with pytest.raises(Exception):
             node_util.get_nova_node_handle(node)
+
+
+@pytest.mark.parametrize('stdout,nova_services_to_restart', [
+    (
+        "   [ + ] nova-service-1\n"
+        "   [ + ] nova-service-2\n"
+        "   [ - ] nova-service-3\n"
+        "   [ - ] not-nova-service-1\n"
+        "   [ + ] not-nova-service-2\n"
+        "   [ + ] nova-service-4\n",
+        [
+            'nova-service-1',
+            'nova-service-2',
+            'nova-service-4',
+        ]
+    )
+])
+def test_restart_nova_services(mocker, node, stdout, nova_services_to_restart):
+    call_output_mock = mocker.patch(
+        "octane.util.ssh.call_output", return_value=stdout)
+    call_mock = mocker.patch("octane.util.ssh.call")
+    node_util.restart_nova_services(node)
+    for service in nova_services_to_restart:
+        call_mock.assert_any_call(["service", service, "restart"], node=node)
+    call_output_mock.assert_called_once_with(
+        ["service", "--status-all"], node=node)

octane/util/node.py

@@ -173,3 +173,12 @@ def is_live_migration_supported(node):
                     and "VIR_MIGRATE_LIVE" in line:
                 return True
     return False
+
+
+def restart_nova_services(node):
+    nova_services = ssh.call_output(["service", "--status-all"], node=node)
+    for service_line in nova_services.splitlines():
+        service_line = service_line.strip()
+        _, status, _, service = service_line.split()
+        if status == "+" and service.startswith("nova"):
+            ssh.call(["service", service, "restart"], node=node)

setup.cfg

@@ -42,6 +42,7 @@ octane =
     update-bootstrap-centos = octane.commands.update_bootstrap:UpdateCentos
     enable-release = octane.commands.enable_release:EnableReleaseCommand
     patch-active-img = octane.commands.patch_active_image:PatchImgCommand
+    cleanup = octane.commands.cleanup:CleanupCommand
 octane.handlers.upgrade =
     controller = octane.handlers.upgrade.controller:ControllerUpgrade
     compute = octane.handlers.upgrade.compute:ComputeUpgrade