Return caching to AppCatalog

Cache Application logo and dynamic_ui markup on local file-system
using `cache.with_cache` decorator. Objects are cached using cPickle
module to allow the function cached by `cache.with_cache` to return
arbitrary objects, not only string or buffer.

Change-Id: Ia0d89b38be1f0776d6e910e3126d89c978db0f10
Implements: blueprint ui-local-cache
Partial-Bug: #1312251

muranodashboard/catalog/image.py

@@ -1,21 +0,0 @@
-#    Copyright (c) 2014 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-from django.http import HttpResponse
-from muranodashboard.environments import api
-
-
-def get_image(request, app_id):
-    content = api.muranoclient(request).packages.get_logo(app_id)
-    return HttpResponse(content=content, content_type='image/png')

muranodashboard/catalog/urls.py

@@ -13,8 +13,8 @@
 #    under the License.
 
 from django.conf.urls import patterns, url
+
 from muranodashboard.catalog import views
-from muranodashboard.catalog import image
 from muranodashboard.dynamic_ui import services
 
 VIEW_MOD = 'muranodashboard.catalog.views'
@@ -43,5 +43,5 @@ urlpatterns = patterns(
         name='quick_add'),
     url(r'^details/(?P<application_id>[^/]+)$',
         views.AppDetailsView.as_view(), name='application_details'),
-    url(r'^images/(?P<app_id>[^/]*)', image.get_image, name="images")
+    url(r'^images/(?P<app_id>[^/]*)', 'get_image', name="images")
 )

muranodashboard/catalog/views.py

@@ -16,8 +16,8 @@ import copy
 import functools
 import json
 import logging
-
 import re
+
 from django.conf import settings
 from django.contrib import auth
 from django.contrib.auth import decorators as auth_dec
@@ -33,10 +33,12 @@ from horizon import messages
 from horizon import tabs
 from horizon.forms import views
 from horizon import exceptions
+
 from muranoclient.common import exceptions as exc
 from muranodashboard.catalog import tabs as catalog_tabs
-from muranodashboard.dynamic_ui import services
+from muranodashboard.common import cache
 from muranodashboard.dynamic_ui import helpers
+from muranodashboard.dynamic_ui import services
 from muranodashboard.environments import api
 from muranodashboard.environments import consts
 from muranodashboard import utils
@@ -124,6 +126,15 @@ def quick_deploy(request, app_id):
         raise
 
 
+def get_image(request, app_id):
+    @cache.with_cache('logo', 'logo.png')
+    def _get(_request, _app_id):
+        return api.muranoclient(_request).packages.get_logo(_app_id)
+
+    content = _get(request, app_id)
+    return http.HttpResponse(content=content, content_type='image/png')
+
+
 class LazyWizard(wizard_views.SessionWizardView):
     """The class which defers evaluation of form_list and condition_dict
     until view method is called. So, each time we load a page with a dynamic

muranodashboard/common/cache.py

@@ -0,0 +1,80 @@
+#    Copyright (c) 2014 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import cPickle as pickle
+import functools
+import logging
+import os
+
+from muranodashboard.environments.consts import CACHE_DIR
+
+
+LOG = logging.getLogger(__name__)
+OBJS_PATH = os.path.join(CACHE_DIR, 'apps')
+
+if not os.path.exists(OBJS_PATH):
+    os.makedirs(OBJS_PATH)
+    LOG.info('Creating apps cache directory located at {dir}'.
+             format(dir=OBJS_PATH))
+
+LOG.info('Using apps cache directory located at {dir}'.
+         format(dir=OBJS_PATH))
+
+
+def _get_entry_path(app_id):
+    head, tail = app_id[:2], app_id[2:]
+    head = os.path.join(OBJS_PATH, head)
+    if not os.path.exists(head):
+        os.mkdir(head)
+    tail = os.path.join(head, tail)
+    if not os.path.exists(tail):
+        os.mkdir(tail)
+    return tail
+
+
+def _load_from_file(file_name):
+    if os.path.isfile(file_name):
+        with open(file_name, 'rb') as f:
+            return pickle.load(f)
+    else:
+        return None
+
+
+def _save_to_file(file_name, content):
+    dir_path = os.path.dirname(file_name)
+    if not os.path.exists(dir_path):
+        os.makedirs(dir_path)
+    with open(file_name, 'wb') as f:
+        pickle.dump(content, f)
+
+
+def with_cache(*dst_parts):
+    def _decorator(func):
+        @functools.wraps(func)
+        def __inner(request, app_id):
+            path = os.path.join(_get_entry_path(app_id), *dst_parts)
+            content = _load_from_file(path)
+            if content is None:
+                content = func(request, app_id)
+                if content:
+                    LOG.debug('Caching value at {0}.'.format(path))
+                    _save_to_file(path, content)
+            else:
+                LOG.debug('Using cached value from {0}.'.format(path))
+
+            return content
+
+        return __inner
+
+    return _decorator

muranodashboard/dynamic_ui/services.py

@@ -14,11 +14,11 @@
 
 
 import logging
-
 import os
 import re
 import yaml
 import yaql
+
 from muranodashboard.dynamic_ui import helpers
 from .helpers import decamelize
 from muranodashboard.environments.consts import CACHE_DIR
@@ -26,6 +26,7 @@ from muranodashboard.dynamic_ui import version
 from muranodashboard.dynamic_ui import yaql_expression
 from muranodashboard.dynamic_ui import yaql_functions
 from muranodashboard.catalog import forms as catalog_forms
+from muranodashboard.common import cache
 
 LOG = logging.getLogger(__name__)
 
@@ -169,8 +170,12 @@ def import_app(request, app_id):
 
     app = services.get(app_id)
     if not app:
-        ui_desc = api.muranoclient(request).packages.get_ui(
-            app_id, make_loader_cls())
+        @cache.with_cache('ui', 'ui.yaml')
+        def _get(_request, _app_id):
+            return api.muranoclient(_request).packages.get_ui(
+                _app_id, make_loader_cls())
+
+        ui_desc = _get(request, app_id)
         version.check_version(ui_desc.pop('Version', 1))
         service = dict((decamelize(k), v) for (k, v) in ui_desc.iteritems())
         services[app_id] = Service(**service)

muranodashboard/environments/consts.py

@@ -24,9 +24,6 @@ ARCHIVE_PKG_NAME = 'archive.tar.gz'
 CACHE_DIR = getattr(settings, 'METADATA_CACHE_DIR',
                     os.path.join(tempfile.gettempdir(),
                                  'muranodashboard-cache'))
-IMAGE_CACHE_DIR = getattr(settings, 'IMAGE_CACHE_DIR',
-                          os.path.join(tempfile.gettempdir(),
-                                       'muranodashboard-image-cache'))
 
 CACHE_REFRESH_SECONDS_INTERVAL = 5
 

muranodashboard/image_cache/cache.py

@@ -1,82 +0,0 @@
-#    Copyright (c) 2014 Mirantis, Inc.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-import logging
-import os
-import time
-
-from muranodashboard.environments.consts import IMAGE_CACHE_DIR
-
-
-LOG = logging.getLogger(__name__)
-
-CHUNK_SIZE = 2048
-
-if not os.path.exists(IMAGE_CACHE_DIR):
-    os.mkdir(IMAGE_CACHE_DIR)
-    LOG.info('Creating image cache directory located at {dir}'.
-             format(dir=IMAGE_CACHE_DIR))
-LOG.info('Using image cache directory located at {dir}'.
-         format(dir=IMAGE_CACHE_DIR))
-
-
-def load_from_file(file_name):
-    if os.path.exists(file_name):
-        data = []
-        with open(file_name, 'rb') as f:
-            buf = f.read(CHUNK_SIZE)
-            while buf:
-                data.append(buf)
-                buf = f.read(CHUNK_SIZE)
-        return data
-    else:
-        return None
-
-
-class ImageCache(object):
-    """
-    Cache is a map which keeps the filenames for images
-    """
-    #TODO (gokrokve) define this as a config parameter
-    max_cached_count = 120
-    #TODO (gokrokve) define this as a config parameter
-    cache_age = 120
-
-    def __init__(self, **kwargs):
-        self.cache = {}
-        self.last_clean = time.time()
-
-    def get_entry(self, name):
-        entry = self.cache.get(name, None)
-        if entry is None:
-            return
-        else:
-            LOG.debug("Cached entry: %s" % name)
-            data = load_from_file(entry['file_name'])
-            entry['last_updated'] = time.time()
-            return data
-
-    def put_cache(self, name, file_name):
-        self.cache[name] = {
-            'file_name': file_name,
-            'last_updated': time.time()
-        }
-
-    def clean(self):
-        LOG.debug("Clean cache entries.")
-        current_time = time.time()
-        for k, v in self.cache.iteritems():
-            if current_time - v['last_updated'] > self.cache_age:
-                LOG.debug("Remove old entry %s" % k)
-                self.cache.pop(k)