From 0b4977a8d1603528f7b7a35c8e103bbdefbae1ac Mon Sep 17 00:00:00 2001 From: suresh kumar Date: Thu, 13 Apr 2017 09:45:01 +0200 Subject: [PATCH] Adds Resource specific purge support Implements `--resource` arg for purging the specific resource type. Change-Id: I9a493d767df3283b8ccd4531109cd5c15e2a3b50 --- ospurge/main.py | 31 +++++++++++++++++++------ ospurge/tests/test_main.py | 45 ++++++++++++++++++++++++++++++++++++- ospurge/tests/test_utils.py | 17 ++++++++++++++ ospurge/utils.py | 25 +++++++++++++++++++++ 4 files changed, 110 insertions(+), 8 deletions(-) diff --git a/ospurge/main.py b/ospurge/main.py index a34b869..f84f603 100644 --- a/ospurge/main.py +++ b/ospurge/main.py @@ -62,6 +62,13 @@ def create_argument_parser() -> argparse.ArgumentParser: "temporarily granted on the project to purge to the " "authenticated user." ) + parser.add_argument( + "--resource", choices=["Networks", "Ports", "SecurityGroups", + "FloatingIPs", "Routers", "RouterInterfaces", + "Servers", "Images", "Backups", "Snapshots", + "Volumes"], + help="Name of Resource type to be checked, instead of all resources " + ) group = parser.add_mutually_exclusive_group(required=True) group.add_argument( @@ -160,7 +167,7 @@ def runner( ) -> None: try: - if not options.dry_run: + if not (options.dry_run or options.resource): resource_mngr.wait_for_check_prerequisite(exit) for resource in resource_mngr.list(): @@ -175,7 +182,11 @@ def runner( if options.dry_run: continue - utils.call_and_ignore_notfound(resource_mngr.delete, resource) + if options.resource: + utils.call_and_ignore_all(resource_mngr.delete, resource) + else: + utils.call_and_ignore_notfound(resource_mngr.delete, + resource) except Exception as exc: log = logging.error @@ -196,7 +207,6 @@ def runner( if is_exception_recoverable(exc): log = logging.info recoverable = True - log("Can't deal with %s: %r", resource_mngr.__class__.__name__, exc) if not recoverable: exit.set() @@ -216,10 +226,17 @@ def main() -> None: creds_manager.ensure_enabled_project() creds_manager.ensure_role_on_project() - resource_managers = sorted( - [cls(creds_manager) for cls in utils.get_all_resource_classes()], - key=operator.methodcaller('order') - ) + if options.resource: + resource_managers = sorted( + [cls(creds_manager) + for cls in utils.get_resource_classes(options.resource)], + key=operator.methodcaller('order') + ) + else: + resource_managers = sorted( + [cls(creds_manager) for cls in utils.get_all_resource_classes()], + key=operator.methodcaller('order') + ) # This is an `Event` used to signal whether one of the threads encountered # an unrecoverable error, at which point all threads should exit because diff --git a/ospurge/tests/test_main.py b/ospurge/tests/test_main.py index f694c5a..78e70c2 100644 --- a/ospurge/tests/test_main.py +++ b/ospurge/tests/test_main.py @@ -56,10 +56,23 @@ class TestFunctions(unittest.TestCase): self.assertEqual(False, options.delete_shared_resources) self.assertEqual(True, options.purge_own_project) + def test_create_argument_parser_with_resource(self): + parser = main.create_argument_parser() + self.assertIsInstance(parser, argparse.ArgumentParser) + + options = parser.parse_args([ + '--resource', 'Networks', '--purge-project', 'foo' + ]) + self.assertEqual(False, options.verbose) + self.assertEqual(False, options.dry_run) + self.assertEqual(False, options.delete_shared_resources) + self.assertEqual('foo', options.purge_project) + self.assertEqual('Networks', options.resource) + def test_runner(self): resources = [mock.Mock(), mock.Mock(), mock.Mock()] resource_manager = mock.Mock(list=mock.Mock(return_value=resources)) - options = mock.Mock(dry_run=False) + options = mock.Mock(dry_run=False, resource=False) exit = mock.Mock(is_set=mock.Mock(side_effect=[False, False, True])) main.runner(resource_manager, options, exit) @@ -88,6 +101,15 @@ class TestFunctions(unittest.TestCase): resource_manager.wait_for_check_prerequisite.assert_not_called() resource_manager.delete.assert_not_called() + def test_runner_resource(self): + resources = [mock.Mock()] + resource_manager = mock.Mock(list=mock.Mock(return_value=resources)) + options = mock.Mock(dry_run=False, resource=True) + exit = mock.Mock(is_set=mock.Mock(return_value=False)) + main.runner(resource_manager, options, exit) + resource_manager.wait_for_check_prerequisite.assert_not_called() + resource_manager.delete.assert_called_once_with(mock.ANY) + def test_runner_with_unrecoverable_exception(self): resource_manager = mock.Mock(list=mock.Mock(side_effect=Exception)) exit = mock.Mock() @@ -125,6 +147,7 @@ class TestFunctions(unittest.TestCase): m_tpe.return_value.__enter__.return_value.map.side_effect = \ KeyboardInterrupt m_parse_args.return_value.purge_own_project = False + m_parse_args.return_value.resource = None m_shade.operator_cloud().get_project().enabled = False main.main() @@ -148,6 +171,26 @@ class TestFunctions(unittest.TestCase): m_event.return_value.is_set.assert_called_once_with() self.assertIsInstance(m_sys_exit.call_args[0][0], int) + @mock.patch.object(main, 'os_client_config', autospec=True) + @mock.patch.object(main, 'shade') + @mock.patch('argparse.ArgumentParser.parse_args') + @mock.patch('threading.Event', autospec=True) + @mock.patch('concurrent.futures.ThreadPoolExecutor', autospec=True) + @mock.patch('sys.exit', autospec=True) + def test_main_resource(self, m_sys_exit, m_tpe, m_event, m_parse_args, + m_shade, m_oscc): + m_tpe.return_value.__enter__.return_value.map.side_effect = \ + KeyboardInterrupt + m_parse_args.return_value.purge_own_project = False + m_parse_args.return_value.resource = "Networks" + m_shade.operator_cloud().get_project().enabled = False + main.main() + m_tpe.return_value.__enter__.assert_called_once_with() + executor = m_tpe.return_value.__enter__.return_value + map_args = executor.map.call_args[0] + for obj in map_args[1]: + self.assertIsInstance(obj, ServiceResource) + @mock.patch.object(main, 'shade') class TestCredentialsManager(unittest.TestCase): diff --git a/ospurge/tests/test_utils.py b/ospurge/tests/test_utils.py index 1424217..ccad2a9 100644 --- a/ospurge/tests/test_utils.py +++ b/ospurge/tests/test_utils.py @@ -49,6 +49,13 @@ class TestUtils(unittest.TestCase): for klass in classes: self.assertTrue(issubclass(klass, ServiceResource)) + def test_get_resource_classes(self): + config = "Networks" + classes = utils.get_resource_classes(config) + self.assertIsInstance(classes, typing.List) + for klass in classes: + self.assertTrue(issubclass(klass, ServiceResource)) + def test_call_and_ignore_notfound(self): def raiser(): raise shade.exc.OpenStackCloudResourceNotFound("") @@ -59,6 +66,16 @@ class TestUtils(unittest.TestCase): utils.call_and_ignore_notfound(m, 42) self.assertEqual([mock.call(42)], m.call_args_list) + def test_call_and_ignore_all(self): + def raiser(): + raise shade.exc.OpenStackCloudException("") + + self.assertIsNone(utils.call_and_ignore_all(raiser)) + + m = mock.Mock() + utils.call_and_ignore_all(m, 42) + self.assertEqual([mock.call(42)], m.call_args_list) + @mock.patch('logging.getLogger', autospec=True) def test_monkeypatch_oscc_logging_warning(self, mock_getLogger): oscc_target = 'os_client_config.cloud_config' diff --git a/ospurge/utils.py b/ospurge/utils.py index f01473f..b2b6440 100644 --- a/ospurge/utils.py +++ b/ospurge/utils.py @@ -14,6 +14,7 @@ import functools import importlib import logging import pkgutil +import sys from typing import Any from typing import Callable from typing import cast @@ -42,6 +43,22 @@ def get_all_resource_classes() -> List: return base.ServiceResource.__subclasses__() +def get_resource_classes(resources) -> List: + """ + Import the modules by subclass name and return the subclasses + of `ServiceResource` Abstract Base Class. + """ + iter_modules = pkgutil.iter_modules( + ['ospurge/resources'], prefix='ospurge.resources.' + ) + for (_, name, ispkg) in iter_modules: + if not ispkg: + importlib.import_module(name) + + return [s for s in base.ServiceResource.__subclasses__() + if s.__name__ in resources] + + F = TypeVar('F', bound=Callable[..., Any]) @@ -77,6 +94,14 @@ def call_and_ignore_notfound(f: Callable, *args: List) -> None: pass +def call_and_ignore_all(f: Callable, *args: List) -> None: + try: + f(*args) + except shade.exc.OpenStackCloudException: + print(sys.exc_info()) + pass + + def replace_project_info(config: Dict, new_project_id: str) -> Dict[str, Any]: """ Replace all tenant/project info in a `os_client_config` config dict with