From 59e862e4226edadf01b84b6072c3cfecb4241415 Mon Sep 17 00:00:00 2001 From: Ke Wu Date: Fri, 1 Jun 2012 12:06:50 -0700 Subject: [PATCH] Add Swift pseudo-folder support to Horizon. Implements blueprint swift-folders. Change-Id: If29ad3cc1fcfb9b7bdb66d915a667f3363d38da0 --- .gitignore | 1 + horizon/api/swift.py | 17 +++- horizon/dashboards/nova/containers/forms.py | 78 +++++++++++++--- horizon/dashboards/nova/containers/tables.py | 55 +++++++++++- horizon/dashboards/nova/containers/tests.py | 10 ++- horizon/dashboards/nova/containers/urls.py | 33 ++++--- horizon/dashboards/nova/containers/views.py | 90 ++++++++++++++----- .../templates/nova/containers/create.html | 4 - .../nova/templates/nova/containers/index.html | 4 +- .../nova/templates/nova/objects/_copy.html | 2 +- .../nova/templates/nova/objects/index.html | 9 +- horizon/tests/api_tests/swift_tests.py | 4 +- 12 files changed, 240 insertions(+), 67 deletions(-) diff --git a/.gitignore b/.gitignore index 5453cb6222..2d0c4f506e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ build dist AUTHORS ChangeLog +tags diff --git a/horizon/api/swift.py b/horizon/api/swift.py index 9af3aa9c64..0568399878 100644 --- a/horizon/api/swift.py +++ b/horizon/api/swift.py @@ -87,12 +87,15 @@ def swift_delete_container(request, name): swift_api(request).delete_container(name) -def swift_get_objects(request, container_name, prefix=None, marker=None): +def swift_get_objects(request, container_name, prefix=None, path=None, + marker=None): limit = getattr(settings, 'API_RESULT_LIMIT', 1000) container = swift_api(request).get_container(container_name) objects = container.get_objects(prefix=prefix, marker=marker, - limit=limit + 1) + limit=limit + 1, + delimiter="/", + path=path) if(len(objects) > limit): return (objects[0:-1], True) else: @@ -122,6 +125,16 @@ def swift_copy_object(request, orig_container_name, orig_object_name, return orig_obj.copy_to(new_container_name, new_object_name) +def swift_create_subfolder(request, container_name, folder_name): + container = swift_api(request).get_container(container_name) + obj = container.create_object(folder_name) + obj.headers = {'content-type': 'application/directory', + 'content-length': 0} + obj.send('') + obj.sync_metadata() + return obj + + def swift_upload_object(request, container_name, object_name, object_file): container = swift_api(request).get_container(container_name) obj = container.create_object(object_name) diff --git a/horizon/dashboards/nova/containers/forms.py b/horizon/dashboards/nova/containers/forms.py index 1aeba8c11b..e0bfc9b3d4 100644 --- a/horizon/dashboards/nova/containers/forms.py +++ b/horizon/dashboards/nova/containers/forms.py @@ -41,21 +41,47 @@ no_slash_validator = validators.RegexValidator(r'^(?u)[^/]+$', class CreateContainer(forms.SelfHandlingForm): - name = forms.CharField(max_length="255", + parent = forms.CharField(max_length=255, + required=False, + widget=forms.HiddenInput) + name = forms.CharField(max_length=255, label=_("Container Name"), validators=[no_slash_validator]) def handle(self, request, data): try: - api.swift_create_container(request, data['name']) - messages.success(request, _("Container created successfully.")) + if not data['parent']: + # Create a container + api.swift_create_container(request, data["name"]) + messages.success(request, _("Container created successfully.")) + else: + # Create a pseudo-folder + container, slash, remainder = data['parent'].partition("/") + remainder = remainder.rstrip("/") + subfolder_name = "/".join([bit for bit + in (remainder, data['name']) + if bit]) + api.swift_create_subfolder(request, + container, + subfolder_name) + messages.success(request, _("Folder created successfully.")) + url = "horizon:nova:containers:object_index" + if remainder: + remainder = remainder.rstrip("/") + remainder += "/" + return shortcuts.redirect(url, container, remainder) + except: exceptions.handle(request, _('Unable to create container.')) + return shortcuts.redirect("horizon:nova:containers:index") class UploadObject(forms.SelfHandlingForm): - name = forms.CharField(max_length="255", + path = forms.CharField(max_length=255, + required=False, + widget=forms.HiddenInput) + name = forms.CharField(max_length=255, label=_("Object Name"), validators=[no_slash_validator]) object_file = forms.FileField(label=_("File")) @@ -63,10 +89,14 @@ class UploadObject(forms.SelfHandlingForm): def handle(self, request, data): object_file = self.files['object_file'] + if data['path']: + object_path = "/".join([data['path'].rstrip("/"), data['name']]) + else: + object_path = data['name'] try: obj = api.swift_upload_object(request, data['container_name'], - data['name'], + object_path, object_file) obj.metadata['orig-filename'] = object_file.name obj.sync_metadata() @@ -74,13 +104,14 @@ class UploadObject(forms.SelfHandlingForm): except: exceptions.handle(request, _("Unable to upload object.")) return shortcuts.redirect("horizon:nova:containers:object_index", - data['container_name']) + data['container_name'], data['path']) class CopyObject(forms.SelfHandlingForm): new_container_name = forms.ChoiceField(label=_("Destination container"), validators=[no_slash_validator]) - new_object_name = forms.CharField(max_length="255", + path = forms.CharField(max_length=255, required=False) + new_object_name = forms.CharField(max_length=255, label=_("Destination object name"), validators=[no_slash_validator]) orig_container_name = forms.CharField(widget=forms.HiddenInput()) @@ -97,15 +128,38 @@ class CopyObject(forms.SelfHandlingForm): orig_object = data['orig_object_name'] new_container = data['new_container_name'] new_object = data['new_object_name'] + new_path = "%s%s" % (data['path'], new_object) + + # Iteratively make sure all the directory markers exist. + if data['path']: + path_component = "" + for bit in data['path'].split("/"): + path_component += bit + try: + api.swift.swift_create_subfolder(request, + new_container, + path_component) + except: + redirect = reverse(object_index, args=(orig_container,)) + exceptions.handle(request, + _("Unable to copy object."), + redirect=redirect) + path_component += "/" + + # Now copy the object itself. try: api.swift_copy_object(request, orig_container, orig_object, new_container, - new_object) - vals = {"container": new_container, "obj": new_object} - messages.success(request, _('Object "%(obj)s" copied to container ' - '"%(container)s".') % vals) + new_path) + dest = "%s/%s" % (new_container, data['path']) + vals = {"dest": dest.rstrip("/"), + "orig": orig_object.split("/")[-1], + "new": new_object} + messages.success(request, + _('Copied "%(orig)s" to "%(dest)s" as "%(new)s".') + % vals) except exceptions.HorizonException, exc: messages.error(request, exc) return shortcuts.redirect(object_index, orig_container) @@ -114,4 +168,4 @@ class CopyObject(forms.SelfHandlingForm): exceptions.handle(request, _("Unable to copy object."), redirect=redirect) - return shortcuts.redirect(object_index, new_container) + return shortcuts.redirect(object_index, new_container, data['path']) diff --git a/horizon/dashboards/nova/containers/tables.py b/horizon/dashboards/nova/containers/tables.py index c41b43db53..5b03be8962 100644 --- a/horizon/dashboards/nova/containers/tables.py +++ b/horizon/dashboards/nova/containers/tables.py @@ -52,7 +52,7 @@ class CreateContainer(tables.LinkAction): class ListObjects(tables.LinkAction): name = "list_objects" - verbose_name = _("List Objects") + verbose_name = _("View Container") url = "horizon:nova:containers:object_index" classes = ("btn-list",) @@ -71,7 +71,10 @@ class UploadObject(tables.LinkAction): else: # This is a table action and we already have the container name container_name = self.table.kwargs['container_name'] - return reverse(self.url, args=(container_name,)) + subfolders = self.table.kwargs.get('subfolder_path', '') + args = (http.urlquote(bit) for bit in + (container_name, subfolders) if bit) + return reverse(self.url, args=args) def update(self, request, obj): # This will only be called for the row, so we can remove the button @@ -129,7 +132,6 @@ class DownloadObject(tables.LinkAction): classes = ("btn-download",) def get_link_url(self, obj): - #assert False, obj.__dict__['_apiresource'].__dict__ return reverse(self.url, args=(http.urlquote(obj.container.name), http.urlquote(obj.name))) @@ -147,12 +149,18 @@ class ObjectFilterAction(tables.FilterAction): return filter(comp, objects) +def sanitize_name(name): + return name.split("/")[-1] + + def get_size(obj): return filesizeformat(obj.size) class ObjectsTable(tables.DataTable): - name = tables.Column("name", verbose_name=_("Object Name")) + name = tables.Column("name", + verbose_name=_("Object Name"), + filters=(sanitize_name,)) size = tables.Column(get_size, verbose_name=_('Size')) def get_object_id(self, obj): @@ -163,3 +171,42 @@ class ObjectsTable(tables.DataTable): verbose_name = _("Objects") table_actions = (ObjectFilterAction, UploadObject, DeleteObject) row_actions = (DownloadObject, CopyObject, DeleteObject) + + +def get_link_subfolder(subfolder): + return reverse("horizon:nova:containers:object_index", + args=(http.urlquote(subfolder.container.name), + http.urlquote(subfolder.name + "/"))) + + +class CreateSubfolder(CreateContainer): + verbose_name = _("Create Folder") + url = "horizon:nova:containers:create" + + def get_link_url(self): + container = self.table.kwargs['container_name'] + subfolders = self.table.kwargs['subfolder_path'] + parent = "/".join((bit for bit in [container, subfolders] if bit)) + parent = parent.rstrip("/") + return reverse(self.url, args=(http.urlquote(parent + "/"),)) + + +class DeleteSubfolder(DeleteObject): + data_type_singular = _("Folder") + data_type_plural = _("Folders") + + +class ContainerSubfoldersTable(tables.DataTable): + name = tables.Column("name", + link=get_link_subfolder, + verbose_name=_("Subfolder Name"), + filters=(sanitize_name,)) + + def get_object_id(self, obj): + return obj.name + + class Meta: + name = "subfolders" + verbose_name = _("Subfolders") + table_actions = (CreateSubfolder, DeleteSubfolder) + row_actions = (DeleteSubfolder,) diff --git a/horizon/dashboards/nova/containers/tests.py b/horizon/dashboards/nova/containers/tests.py index 5256fe62c5..e58ba48b00 100644 --- a/horizon/dashboards/nova/containers/tests.py +++ b/horizon/dashboards/nova/containers/tests.py @@ -102,16 +102,18 @@ class ObjectViewTests(test.TestCase): ret = (self.objects.list(), False) api.swift_get_objects(IsA(http.HttpRequest), self.containers.first().name, - marker=None).AndReturn(ret) + marker=None, + path=None).AndReturn(ret) self.mox.ReplayAll() res = self.client.get(reverse('horizon:nova:containers:object_index', args=[self.containers.first().name])) self.assertTemplateUsed(res, 'nova/objects/index.html') - expected = [obj.name for obj in self.objects.list()] - self.assertQuerysetEqual(res.context['table'].data, + # UTF8 encoding here to ensure there aren't problems with Nose output. + expected = [obj.name.encode('utf8') for obj in self.objects.list()] + self.assertQuerysetEqual(res.context['objects_table'].data, expected, - lambda obj: obj.name) + lambda obj: obj.name.encode('utf8')) def test_upload_index(self): res = self.client.get(reverse('horizon:nova:containers:object_upload', diff --git a/horizon/dashboards/nova/containers/urls.py b/horizon/dashboards/nova/containers/urls.py index 9ba215946f..c72f1fc1d9 100644 --- a/horizon/dashboards/nova/containers/urls.py +++ b/horizon/dashboards/nova/containers/urls.py @@ -23,16 +23,29 @@ from django.conf.urls.defaults import patterns, url from .views import IndexView, CreateView, UploadView, ObjectIndexView, CopyView -OBJECTS = r'^(?P[^/]+)/%s$' - - # Swift containers and objects. urlpatterns = patterns('horizon.dashboards.nova.containers.views', url(r'^$', IndexView.as_view(), name='index'), - url(r'^create/$', CreateView.as_view(), name='create'), - url(OBJECTS % r'$', ObjectIndexView.as_view(), name='object_index'), - url(OBJECTS % r'upload$', UploadView.as_view(), name='object_upload'), - url(OBJECTS % r'(?P[^/]+)/copy$', - CopyView.as_view(), name='object_copy'), - url(OBJECTS % r'(?P[^/]+)/download$', - 'object_download', name='object_download')) + + url(r'^(?P(.+/)+)?create$', + CreateView.as_view(), + name='create'), + + url(r'^(?P[^/]+)/(?P(.+/)+)?$', + ObjectIndexView.as_view(), + name='object_index'), + + url(r'^(?P[^/]+)/(?P(.+/)+)?upload$', + UploadView.as_view(), + name='object_upload'), + + url(r'^(?P[^/]+)/' + r'(?P(.+/)+)?' + r'(?P.+)/copy$', + CopyView.as_view(), + name='object_copy'), + + url(r'^(?P[^/]+)/(?P.+)/download$', + 'object_download', + name='object_download') +) diff --git a/horizon/dashboards/nova/containers/views.py b/horizon/dashboards/nova/containers/views.py index c8720319fb..eaac1a50ba 100644 --- a/horizon/dashboards/nova/containers/views.py +++ b/horizon/dashboards/nova/containers/views.py @@ -33,7 +33,8 @@ from horizon import exceptions from horizon import forms from horizon import tables from .forms import CreateContainer, UploadObject, CopyObject -from .tables import ContainersTable, ObjectsTable +from .tables import ContainersTable, ObjectsTable,\ + ContainerSubfoldersTable LOG = logging.getLogger(__name__) @@ -63,31 +64,68 @@ class CreateView(forms.ModalFormView): form_class = CreateContainer template_name = 'nova/containers/create.html' + def get_initial(self): + initial = super(CreateView, self).get_initial() + initial['parent'] = self.kwargs['container_name'] + return initial -class ObjectIndexView(tables.DataTableView): - table_class = ObjectsTable + +class ObjectIndexView(tables.MultiTableView): + table_classes = (ObjectsTable, ContainerSubfoldersTable) template_name = 'nova/objects/index.html' def has_more_data(self, table): return self._more - def get_data(self): - objects = [] - self._more = None - marker = self.request.GET.get('marker', None) - container_name = self.kwargs['container_name'] - try: - objects, self._more = api.swift_get_objects(self.request, - container_name, - marker=marker) - except: - msg = _('Unable to retrieve object list.') - exceptions.handle(self.request, msg) - return objects + @property + def objects(self): + """ Returns a list of objects given the subfolder's path. + + The path is from the kwargs of the request + """ + if not hasattr(self, "_objects"): + objects = [] + self._more = None + marker = self.request.GET.get('marker', None) + container_name = self.kwargs['container_name'] + subfolders = self.kwargs['subfolder_path'] + if subfolders: + prefix = subfolders.rstrip("/") + else: + prefix = None + try: + objects, self._more = api.swift_get_objects(self.request, + container_name, + marker=marker, + path=prefix) + except: + objects = [] + msg = _('Unable to retrieve object list.') + exceptions.handle(self.request, msg) + self._objects = objects + return self._objects + + def get_objects_data(self): + """ Returns the objects within the in the current folder. + + These objects are those whose names don't contain '/' after + striped the path out + """ + filtered_objects = [item for item in self.objects if + item.content_type != "application/directory"] + return filtered_objects + + def get_subfolders_data(self): + """ Returns a list of subfolders given the current folder path. + """ + filtered_objects = [item for item in self.objects if + item.content_type == "application/directory"] + return filtered_objects def get_context_data(self, **kwargs): context = super(ObjectIndexView, self).get_context_data(**kwargs) context['container_name'] = self.kwargs["container_name"] + context['subfolder_path'] = self.kwargs["subfolder_path"] return context @@ -96,7 +134,8 @@ class UploadView(forms.ModalFormView): template_name = 'nova/objects/upload.html' def get_initial(self): - return {"container_name": self.kwargs["container_name"]} + return {"container_name": self.kwargs["container_name"], + "path": self.kwargs['subfolder_path']} def get_context_data(self, **kwargs): context = super(UploadView, self).get_context_data(**kwargs) @@ -104,25 +143,25 @@ class UploadView(forms.ModalFormView): return context -def object_download(request, container_name, object_name): - obj = api.swift.swift_get_object(request, container_name, object_name) +def object_download(request, container_name, object_path): + obj = api.swift.swift_get_object(request, container_name, object_path) # Add the original file extension back on if it wasn't preserved in the # name given to the object. - filename = object_name + filename = object_path.rsplit("/")[-1] if not os.path.splitext(obj.name)[1]: name, ext = os.path.splitext(obj.metadata.get('orig-filename', '')) - filename = "%s%s" % (object_name, ext) + filename = "%s%s" % (filename, ext) try: object_data = api.swift_get_object_data(request, container_name, - object_name) + object_path) except: redirect = reverse("horizon:nova:containers:index") exceptions.handle(request, _("Unable to retrieve object."), redirect=redirect) response = http.HttpResponse() - safe_name = filename.encode('utf-8') + safe_name = filename.replace(",", "").encode('utf-8') response['Content-Disposition'] = 'attachment; filename=%s' % safe_name response['Content-Type'] = 'application/octet-stream' for data in object_data: @@ -147,9 +186,12 @@ class CopyView(forms.ModalFormView): return kwargs def get_initial(self): + path = self.kwargs["subfolder_path"] + orig = "%s%s" % (path or '', self.kwargs["object_name"]) return {"new_container_name": self.kwargs["container_name"], "orig_container_name": self.kwargs["container_name"], - "orig_object_name": self.kwargs["object_name"], + "orig_object_name": orig, + "path": path, "new_object_name": "%s copy" % self.kwargs["object_name"]} def get_context_data(self, **kwargs): diff --git a/horizon/dashboards/nova/templates/nova/containers/create.html b/horizon/dashboards/nova/templates/nova/containers/create.html index 1facb05d89..d9e1b561b2 100644 --- a/horizon/dashboards/nova/templates/nova/containers/create.html +++ b/horizon/dashboards/nova/templates/nova/containers/create.html @@ -9,7 +9,3 @@ {% block dash_main %} {% include "nova/containers/_create.html" %} {% endblock %} - - - - diff --git a/horizon/dashboards/nova/templates/nova/containers/index.html b/horizon/dashboards/nova/templates/nova/containers/index.html index c3f027c41e..54140a92f9 100644 --- a/horizon/dashboards/nova/templates/nova/containers/index.html +++ b/horizon/dashboards/nova/templates/nova/containers/index.html @@ -3,9 +3,7 @@ {% block title %}Containers{% endblock %} {% block page_header %} - {% url horizon:nova:images_and_snapshots:images:index as refresh_link %} - {# to make searchable false, just remove it from the include statement #} - {% include "horizon/common/_page_header.html" with title=_("Containers") refresh_link=refresh_link searchable="true" %} + {% include "horizon/common/_page_header.html" with title=_("Containers") %} {% endblock page_header %} {% block dash_main %} diff --git a/horizon/dashboards/nova/templates/nova/objects/_copy.html b/horizon/dashboards/nova/templates/nova/objects/_copy.html index 0a8e808d23..870c8689f3 100644 --- a/horizon/dashboards/nova/templates/nova/objects/_copy.html +++ b/horizon/dashboards/nova/templates/nova/objects/_copy.html @@ -14,7 +14,7 @@

{% trans "Description" %}:

-

{% trans "You may make a new copy of an existing object to store in this or another container." %}

+

{% trans "Make a new copy of an existing object to store in this or another container. You may also specify a path at which the new copy should live inside of the selected container." %}

{% endblock %} diff --git a/horizon/dashboards/nova/templates/nova/objects/index.html b/horizon/dashboards/nova/templates/nova/objects/index.html index 986a254519..cd44c7a837 100644 --- a/horizon/dashboards/nova/templates/nova/objects/index.html +++ b/horizon/dashboards/nova/templates/nova/objects/index.html @@ -4,10 +4,15 @@ {% block page_header %}
-

Objects Container: {{ container_name }}

+

{% trans "Container" %}: {{ container_name }}/{{ subfolder_path|default:"" }}

{% endblock page_header %} {% block dash_main %} - {{ table.render }} +
+ {{ subfolders_table.render }} +
+
+ {{ objects_table.render }} +
{% endblock %} diff --git a/horizon/tests/api_tests/swift_tests.py b/horizon/tests/api_tests/swift_tests.py index 89902041b2..afe8b69c45 100644 --- a/horizon/tests/api_tests/swift_tests.py +++ b/horizon/tests/api_tests/swift_tests.py @@ -68,7 +68,9 @@ class SwiftApiTests(test.APITestCase): self.mox.StubOutWithMock(container, 'get_objects') container.get_objects(limit=1001, marker=None, - prefix=None).AndReturn(objects) + prefix=None, + delimiter='/', + path=None).AndReturn(objects) self.mox.ReplayAll() (objs, more) = api.swift_get_objects(self.request, container.name)