diff --git a/muranoclient/osc/v1/package.py b/muranoclient/osc/v1/package.py index e3e88bc1..f598cdfb 100644 --- a/muranoclient/osc/v1/package.py +++ b/muranoclient/osc/v1/package.py @@ -12,9 +12,12 @@ """Application-catalog v1 package action implementation""" +import collections import itertools import os import shutil +import six +import sys import tempfile import zipfile @@ -24,12 +27,16 @@ from osc_lib import utils from oslo_log import log as logging from muranoclient.apiclient import exceptions +from muranoclient.common import exceptions as common_exceptions +from muranoclient.common import utils as murano_utils from muranoclient.v1.package_creator import hot_package from muranoclient.v1.package_creator import mpl_package LOG = logging.getLogger(__name__) +DEFAULT_REPO_URL = "http://apps.openstack.org/api/v1/murano_repo/liberty/" + class CreatePackage(command.Command): """Create an application package.""" @@ -320,3 +327,191 @@ class DeletePackage(command.Lister): columns, ) for s in data) ) + + +def _handle_package_exists(mc, data, package, exists_action): + name = package.manifest['FullName'] + version = package.manifest.get('Version', '0') + while True: + print("Importing package {0}".format(name)) + try: + return mc.packages.create(data, {name: package.file()}) + except common_exceptions.HTTPConflict: + print("Importing package {0} failed. Package with the same" + " name/classes is already registered.".format(name)) + allowed_results = ['s', 'u', 'a'] + res = exists_action + if not res: + while True: + print("What do you want to do? (s)kip, (u)pdate, (a)bort") + res = six.moves.input() + if res in allowed_results: + break + if res == 's': + print("Skipping.") + return None + elif res == 'a': + print("Exiting.") + sys.exit() + elif res == 'u': + pkgs = list(mc.packages.filter(fqn=name, version=version, + owned=True)) + if not pkgs: + msg = ( + "Got a conflict response, but could not find the " + "package '{0}' in the current tenant.\nThis probably " + "means the conflicting package is in another tenant.\n" + "Please delete it manually." + ).format(name) + raise exceptions.CommandError(msg) + elif len(pkgs) > 1: + msg = ( + "Got {0} packages with name '{1}'.\nI do not trust " + "myself, please delete the package manually." + ).format(len(pkgs), name) + raise exceptions.CommandError(msg) + print("Deleting package {0}({1})".format(name, pkgs[0].id)) + mc.packages.delete(pkgs[0].id) + continue + + +class ImportPackage(command.Lister): + """Import a package.""" + + def get_parser(self, prog_name): + parser = super(ImportPackage, self).get_parser(prog_name) + parser.add_argument( + 'filename', + metavar='', + nargs='+', + help='URL of the murano zip package, FQPN, path to zip package' + ' or path to directory with package.' + ) + parser.add_argument( + '--categories', + metavar='', + nargs='*', + help='Category list to attach.', + ) + parser.add_argument( + '--is-public', + action='store_true', + default=False, + help="Make the package available for users from other tenants.", + ) + parser.add_argument( + '--package-version', + default='', + help='Version of the package to use from repository ' + '(ignored when importing with multiple packages).' + ) + parser.add_argument( + '--exists-action', + default='', + choices=['a', 's', 'u'], + help='Default action when a package already exists: ' + '(s)kip, (u)pdate, (a)bort.' + ) + parser.add_argument( + '--dep-exists-action', + default='', + choices=['a', 's', 'u'], + help='Default action when a dependency package already exists: ' + '(s)kip, (u)pdate, (a)bort.' + ) + parser.add_argument('--murano-repo-url', + default=murano_utils.env( + 'MURANO_REPO_URL', + default=DEFAULT_REPO_URL), + help=('Defaults to env[MURANO_REPO_URL] ' + 'or {0}'.format(DEFAULT_REPO_URL))) + + return parser + + def take_action(self, parsed_args): + LOG.debug("take_action({0})".format(parsed_args)) + + client = self.app.client_manager.application_catalog + + data = {"is_public": parsed_args.is_public} + version = parsed_args.package_version + if version and len(parsed_args.filename) >= 2: + print("Requested to import more than one package, " + "ignoring version.") + version = '' + + if parsed_args.categories: + data["categories"] = parsed_args.categories + + total_reqs = collections.OrderedDict() + main_packages_names = [] + for filename in parsed_args.filename: + if os.path.isfile(filename) or os.path.isdir(filename): + _file = filename + else: + print("Package file '{0}' does not exist, attempting to " + "download".format(filename)) + _file = murano_utils.to_url( + filename, + version=version, + base_url=parsed_args.murano_repo_url, + extension='.zip', + path='apps/', + ) + try: + package = murano_utils.Package.from_file(_file) + except Exception as e: + print("Failed to create package for '{0}', reason: {1}".format( + filename, e)) + continue + total_reqs.update( + package.requirements(base_url=parsed_args.murano_repo_url)) + main_packages_names.append(package.manifest['FullName']) + + imported_list = [] + + dep_exists_action = parsed_args.dep_exists_action + if dep_exists_action == '': + dep_exists_action = parsed_args.exists_action + + for name, package in six.iteritems(total_reqs): + image_specs = package.images() + if image_specs: + print("Inspecting required images") + try: + imgs = murano_utils.ensure_images( + glance_client=client.glance_client, + image_specs=image_specs, + base_url=parsed_args.murano_repo_url, + is_package_public=parsed_args.is_public) + for img in imgs: + print("Added {0}, {1} image".format( + img['name'], img['id'])) + except Exception as e: + print("Error {0} occurred while installing " + "images for {1}".format(e, name)) + + if name in main_packages_names: + exists_action = parsed_args.exists_action + else: + exists_action = dep_exists_action + try: + imported_package = _handle_package_exists( + client, data, package, exists_action) + if imported_package: + imported_list.append(imported_package) + except Exception as e: + print("Error {0} occurred while installing package {1}".format( + e, name)) + + columns = ('id', 'name', 'fully_qualified_name', 'author', 'active', + 'is public', 'type', 'version') + column_headers = [c.capitalize() for c in columns] + + return ( + column_headers, + list(utils.get_item_properties( + s, + columns, + ) for s in imported_list) + ) diff --git a/muranoclient/tests/unit/osc/v1/test_package.py b/muranoclient/tests/unit/osc/v1/test_package.py index 8b110a0d..568ae3a5 100644 --- a/muranoclient/tests/unit/osc/v1/test_package.py +++ b/muranoclient/tests/unit/osc/v1/test_package.py @@ -17,13 +17,19 @@ import tempfile from testtools import matchers +from muranoclient.common import exceptions as common_exceptions +from muranoclient.common import utils as mc_utils from muranoclient.osc.v1 import package as osc_pkg from muranoclient.tests.unit.osc.v1 import fakes +from muranoclient.tests.unit import test_utils from muranoclient.v1 import packages import mock from osc_lib import exceptions as exc from osc_lib import utils +import requests_mock + +make_pkg = test_utils.make_pkg FIXTURE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), 'fixture_data')) @@ -222,3 +228,323 @@ class TestPackageDelete(TestPackage): expected_data = [('1234', 'Core library', 'io.murano', 'murano.io', '', 'True', 'Library', '0.0.0')] self.assertEqual(expected_data, data) + + +class TestPackageImport(TestPackage): + def setUp(self): + super(TestPackageImport, self).setUp() + self.package_mock.filter.return_value = \ + [packages.Package(None, DATA)] + + # Command to test + self.cmd = osc_pkg.ImportPackage(self.app, None) + + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import(self, from_file): + with tempfile.NamedTemporaryFile() as f: + RESULT_PACKAGE = f.name + categories = ['Cat1', 'Cat2 with space'] + + pkg = make_pkg({'FullName': RESULT_PACKAGE}) + from_file.return_value = mc_utils.Package(mc_utils.File(pkg)) + + arglist = [RESULT_PACKAGE, '--categories', + categories, '--is-public'] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + self.package_mock.create.assert_called_once_with({ + 'categories': [categories], + 'is_public': True + }, {RESULT_PACKAGE: mock.ANY},) + + def _test_conflict(self, + packages, from_file, raw_input_mock, + input_action, exists_action=''): + packages.create = mock.MagicMock( + side_effect=[common_exceptions.HTTPConflict("Conflict"), None]) + + packages.filter.return_value = [mock.Mock(id='test_id')] + + raw_input_mock.return_value = input_action + with tempfile.NamedTemporaryFile() as f: + pkg = make_pkg({'FullName': f.name}) + from_file.return_value = mc_utils.Package(mc_utils.File(pkg)) + if exists_action: + arglist = [f.name, '--exists-action', exists_action] + else: + arglist = [f.name] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + return f.name + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_skip(self, from_file, raw_input_mock): + + name = self._test_conflict( + self.package_mock, + from_file, + raw_input_mock, + 's', + ) + + self.package_mock.create.assert_called_once_with({ + 'is_public': False, + }, {name: mock.ANY},) + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_skip_ea(self, from_file, raw_input_mock): + + name = self._test_conflict( + self.package_mock, + from_file, + raw_input_mock, + '', + exists_action='s', + ) + + self.package_mock.create.assert_called_once_with({ + 'is_public': False, + }, {name: mock.ANY},) + self.assertFalse(raw_input_mock.called) + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_abort(self, from_file, raw_input_mock): + + self.assertRaises(SystemExit, self._test_conflict, + self.package_mock, + from_file, + raw_input_mock, + 'a', + ) + + self.package_mock.create.assert_called_once_with({ + 'is_public': False, + }, mock.ANY,) + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_abort_ea(self, + from_file, raw_input_mock): + + self.assertRaises(SystemExit, self._test_conflict, + self.package_mock, + from_file, + raw_input_mock, + '', + exists_action='a', + ) + + self.package_mock.create.assert_called_once_with({ + 'is_public': False, + }, mock.ANY,) + self.assertFalse(raw_input_mock.called) + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_update(self, from_file, raw_input_mock): + + name = self._test_conflict( + self.package_mock, + from_file, + raw_input_mock, + 'u', + ) + + self.assertEqual(2, self.package_mock.create.call_count) + self.package_mock.delete.assert_called_once_with('test_id') + + self.package_mock.create.assert_has_calls( + [ + mock.call({'is_public': False}, {name: mock.ANY},), + mock.call({'is_public': False}, {name: mock.ANY},) + ], any_order=True, + ) + + @mock.patch('six.moves.input') + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_update_ea(self, + from_file, raw_input_mock): + + name = self._test_conflict( + self.package_mock, + from_file, + raw_input_mock, + '', + exists_action='u', + ) + + self.assertEqual(2, self.package_mock.create.call_count) + self.package_mock.delete.assert_called_once_with('test_id') + + self.package_mock.create.assert_has_calls( + [ + mock.call({'is_public': False}, {name: mock.ANY},), + mock.call({'is_public': False}, {name: mock.ANY},) + ], any_order=True, + ) + self.assertFalse(raw_input_mock.called) + + def _test_conflict_dep(self, + packages, from_file, + dep_exists_action=''): + packages.create = mock.MagicMock( + side_effect=[common_exceptions.HTTPConflict("Conflict"), + common_exceptions.HTTPConflict("Conflict"), + None]) + + packages.filter.return_value = [mock.Mock(id='test_id')] + + pkg1 = make_pkg( + {'FullName': 'first_app', 'Require': {'second_app': '1.0'}, }) + pkg2 = make_pkg({'FullName': 'second_app', }) + + def side_effect(name): + if 'first_app' in name: + return mc_utils.Package(mc_utils.File(pkg1)) + if 'second_app' in name: + return mc_utils.Package(mc_utils.File(pkg2)) + + from_file.side_effect = side_effect + + arglist = ['first_app', '--exists-action', 's', + '--dep-exists-action', dep_exists_action] + + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_dep_skip_ea(self, from_file): + self._test_conflict_dep( + self.package_mock, + from_file, + dep_exists_action='s', + ) + + self.assertEqual(2, self.package_mock.create.call_count) + self.package_mock.create.assert_has_calls( + [ + mock.call({'is_public': False}, {'first_app': mock.ANY}), + mock.call({'is_public': False}, {'second_app': mock.ANY}), + ], any_order=True, + ) + + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_dep_abort_ea(self, from_file): + self.assertRaises(SystemExit, self._test_conflict_dep, + self.package_mock, + from_file, + dep_exists_action='a', + ) + + self.package_mock.create.assert_called_with({ + 'is_public': False, + }, {'second_app': mock.ANY},) + + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_conflict_dep_update_ea(self, from_file): + self._test_conflict_dep( + self.package_mock, + from_file, + dep_exists_action='u', + ) + + self.assertGreater(self.package_mock.create.call_count, 2) + self.assertLess(self.package_mock.create.call_count, 5) + + self.assertTrue(self.package_mock.delete.called) + + self.package_mock.create.assert_has_calls( + [ + mock.call({'is_public': False}, {'first_app': mock.ANY}), + mock.call({'is_public': False}, {'second_app': mock.ANY}), + mock.call({'is_public': False}, {'second_app': mock.ANY}), + ], any_order=True, + ) + + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_no_categories(self, from_file): + with tempfile.NamedTemporaryFile() as f: + RESULT_PACKAGE = f.name + pkg = make_pkg({'FullName': RESULT_PACKAGE}) + from_file.return_value = mc_utils.Package(mc_utils.File(pkg)) + + arglist = [RESULT_PACKAGE] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + self.package_mock.create.assert_called_once_with( + {'is_public': False}, + {RESULT_PACKAGE: mock.ANY}, + ) + + @requests_mock.mock() + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_url(self, rm, from_file): + filename = "http://127.0.0.1/test_package.zip" + + pkg = make_pkg({'FullName': 'test_package'}) + from_file.return_value = mc_utils.Package(mc_utils.File(pkg)) + + rm.get(filename, body=make_pkg({'FullName': 'test_package'})) + + arglist = [filename] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + self.package_mock.create.assert_called_once_with( + {'is_public': False}, + {'test_package': mock.ANY}, + ) + + @requests_mock.mock() + @mock.patch('muranoclient.common.utils.Package.from_file') + def test_package_import_by_name(self, rm, from_file): + filename = "io.test.apps.test_application" + murano_repo_url = "http://127.0.0.1" + + pkg = make_pkg({'FullName': filename}) + from_file.return_value = mc_utils.Package(mc_utils.File(pkg)) + + rm.get(murano_repo_url + '/apps/' + filename + '.zip', + body=make_pkg({'FullName': 'first_app'})) + + arglist = [filename, '--murano-repo-url', murano_repo_url] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + self.assertTrue(self.package_mock.create.called) + self.package_mock.create.assert_called_once_with( + {'is_public': False}, + {filename: mock.ANY}, + ) + + @requests_mock.mock() + def test_package_import_multiple(self, rm): + filename = ["io.test.apps.test_application", + "http://127.0.0.1/test_app2.zip", ] + murano_repo_url = "http://127.0.0.1" + + rm.get(murano_repo_url + '/apps/' + filename[0] + '.zip', + body=make_pkg({'FullName': 'first_app'})) + + rm.get(filename[1], + body=make_pkg({'FullName': 'second_app'})) + + arglist = [filename[0], filename[1], + '--murano-repo-url', murano_repo_url] + parsed_args = self.check_parser(self.cmd, arglist, []) + self.cmd.take_action(parsed_args) + + self.assertEqual(2, self.package_mock.create.call_count) + + self.package_mock.create.assert_has_calls( + [ + mock.call({'is_public': False}, {'first_app': mock.ANY}), + mock.call({'is_public': False}, {'second_app': mock.ANY}), + ], any_order=True, + ) diff --git a/setup.cfg b/setup.cfg index 73c5ea3e..895086c2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -55,6 +55,7 @@ openstack.application_catalog.v1 = package_create = muranoclient.osc.v1.package:CreatePackage package_list = muranoclient.osc.v1.package:ListPackages package_delete = muranoclient.osc.v1.package:DeletePackage + package_import = muranoclient.osc.v1.package:ImportPackage static-action_call = muranoclient.osc.v1.action:StaticActionCall class-schema = muranoclient.osc.v1.schema:ShowSchema