diff --git a/glare/engine.py b/glare/engine.py index 83e639f..5bb885e 100644 --- a/glare/engine.py +++ b/glare/engine.py @@ -15,6 +15,7 @@ from copy import deepcopy +from eventlet import tpool import jsonpatch from oslo_config import cfg from oslo_log import log as logging @@ -574,10 +575,11 @@ class Engine(object): if hasattr(af, 'validate_upload'): LOG.warning("Method 'validate_upload' was deprecated. " "Please use 'pre_upload_hook' instead.") - fd, path = af.validate_upload(context, af, field_name, fd) + fd, path = tpool.execute( + af.validate_upload, context, af, field_name, fd) else: - fd = af.pre_upload_hook( - context, af, field_name, blob_key, fd) + fd = tpool.execute(af.pre_upload_hook, + context, af, field_name, blob_key, fd) except exception.GlareException: raise except Exception as e: diff --git a/glare/tests/unit/base.py b/glare/tests/unit/base.py index 815043e..1b0c7eb 100644 --- a/glare/tests/unit/base.py +++ b/glare/tests/unit/base.py @@ -79,12 +79,13 @@ class BaseTestCase(testtools.TestCase): self.config( custom_artifact_types_modules=[ 'glare.tests.sample_artifact', - 'glare.tests.hooks_artifact' + 'glare.tests.hooks_artifact', + 'glare.tests.unpacking_artifact' ], enabled_artifact_types=[ - 'hooks_artifact', 'sample_artifact', 'images', - 'heat_templates', 'heat_environments', 'murano_packages', - 'tosca_templates'] + 'unpacking_artifact', 'hooks_artifact', 'sample_artifact', + 'images', 'heat_templates', 'heat_environments', + 'murano_packages', 'tosca_templates'] ) location.SCHEME_TO_CLS_MAP = {} diff --git a/glare/tests/unit/test_multistore.py b/glare/tests/unit/test_multistore.py index 697d4d8..89e4c77 100644 --- a/glare/tests/unit/test_multistore.py +++ b/glare/tests/unit/test_multistore.py @@ -26,7 +26,8 @@ class TestMultistore(base.BaseTestCase): 'tosca_templates': 'sheepdog', 'murano_packages': 'vsphere', 'sample_artifact': 'database', - 'hooks_artifact': 'database'} + 'hooks_artifact': 'database', + 'unpacking_artifact': 'database'} # create engine and register new artifact types engine.Engine() diff --git a/glare/tests/unit/test_unpacking.py b/glare/tests/unit/test_unpacking.py new file mode 100644 index 0000000..b70569e --- /dev/null +++ b/glare/tests/unit/test_unpacking.py @@ -0,0 +1,44 @@ +# Copyright 2017 - Nokia Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from glare.tests.unit import base + + +class TestArtifactHooks(base.BaseTestArtifactAPI): + + def setUp(self): + super(TestArtifactHooks, self).setUp() + values = {'name': 'ttt', 'version': '1.0'} + self.unpacking_artifact = self.controller.create( + self.req, 'unpacking_artifact', values) + + def test_unpacking(self): + var_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + '../', 'var')) + data_path = os.path.join(var_dir, 'hooks.zip') + with open(data_path, "rb") as data: + self.controller.upload_blob( + self.req, 'unpacking_artifact', self.unpacking_artifact['id'], + 'zip', data, 'application/octet-stream') + artifact = self.controller.show(self.req, 'unpacking_artifact', + self.unpacking_artifact['id']) + self.assertEqual(818, artifact['zip']['size']) + self.assertEqual('active', artifact['zip']['status']) + + self.assertEqual(11, artifact['content']['aaa.txt']['size']) + self.assertEqual(11, artifact['content']['folder1/bbb.txt']['size']) + self.assertEqual( + 11, artifact['content']['folder1/folder2/ccc.txt']['size']) diff --git a/glare/tests/unpacking_artifact.py b/glare/tests/unpacking_artifact.py new file mode 100644 index 0000000..2de5847 --- /dev/null +++ b/glare/tests/unpacking_artifact.py @@ -0,0 +1,58 @@ +# Copyright 2017 - Nokia Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +import zipfile + +from glare.common import exception +from glare.common import utils +from glare.objects import base +from glare.objects.meta import file_utils +from glare.objects.meta import wrappers + +Blob = wrappers.BlobField.init +Folder = wrappers.FolderField.init + + +class Unpacker(base.BaseArtifact): + MAX_BLOB_SIZE = 100000 + + fields = { + 'zip': Blob(description="Original zipped data.", + required_on_activate=False), + 'content': Folder(system=True, required_on_activate=False), + } + + @classmethod + def get_type_name(cls): + return "unpacking_artifact" + + @classmethod + def pre_upload_hook(cls, context, af, field_name, blob_key, fd): + flobj = io.BytesIO(fd.read(cls.MAX_BLOB_SIZE)) + + # Raise exception if something left in the stream + if fd.read(1): + msg = ("The file you are trying to upload is too big. " + "The system upper limit is %s.") % cls.MAX_BLOB_SIZE + raise exception.RequestEntityTooLarge(msg) + + zip_ref = zipfile.ZipFile(flobj, 'r') + for name in zip_ref.namelist(): + if not name.endswith('/'): + file_utils.upload_content_file( + context, af, utils.BlobIterator(zip_ref.read(name)), + 'content', name) + flobj.seek(0) + return flobj