Merge "Pre-populate the Angular template cache and allow template overrides"

doc/source/topics/settings.rst

@@ -1686,13 +1686,21 @@ Template loaders defined here will have their output cached if DEBUG
 is set to False.
 
 ``ADD_TEMPLATE_LOADERS``
----------------------------
+------------------------
 
 .. versionadded:: 10.0.0(Newton)
 
 Template loaders defined here will be be loaded at the end of TEMPLATE_LOADERS,
 after the CACHED_TEMPLATE_LOADERS and will never have a cached output.
 
+``NG_TEMPLATE_CACHE_AGE``
+-------------------------
+
+.. versionadded:: 10.0.0(Newton)
+
+Angular Templates are cached using this duration (in seconds) if DEBUG
+is set to False.  Default value is ``2592000`` (or 30 days).
+
 ``SECRET_KEY``
 --------------
 

horizon/templatetags/angular.py

@@ -0,0 +1,120 @@
+# Copyright 2016 Hewlett Packard Enterprise Development Company LP
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from compressor.signals import post_compress
+from django.contrib.staticfiles import finders
+from django.core.cache import caches
+from django.core.cache.utils import make_template_fragment_key
+from django.dispatch import receiver
+from django import template
+
+register = template.Library()
+
+
+@receiver(post_compress)
+def update_angular_template_hash(sender, **kwargs):
+    """Listen for compress events. If the angular templates
+    have been re-compressed, also clear them from the Django
+    cache backend.  This is important to allow deployers to
+    change a template file, re-compress, and not accidentally
+    serve the old Django cached version of that content to
+    clients.
+    """
+    context = kwargs['context']  # context the compressor is working with
+    theme = context['THEME']  # current theme being compressed
+    compressed = context['compressed']  # the compressed content
+    compressed_name = compressed['name']  # name of the compressed content
+    if compressed_name == 'angular_template_cache_preloads':
+        # The compressor has modified the angular template cache preloads
+        # which are cached in the 'default' Django cache. Fetch that cache.
+        cache = caches['default']
+
+        # generate the same key as used in _scripts.html when caching the
+        # preloads
+        key = make_template_fragment_key(
+            "angular",
+            ['template_cache_preloads', theme]
+        )
+
+        # if template preloads have been cached, clear them
+        if cache.get(key):
+            cache.delete(key)
+
+
+@register.filter(name='angular_escapes')
+def angular_escapes(value):
+    """Djangos 'escapejs' is too aggressive and inserts unicode. Provide
+    a basic filter to allow angular template content to be used within
+    javascript strings.
+
+    Args:
+        value: a string
+
+    Returns:
+        string with escaped values
+    """
+    return value \
+        .replace('"', '\\"') \
+        .replace("'", "\\'") \
+        .replace("\n", "\\n") \
+        .replace("\r", "\\r")
+
+
+@register.inclusion_tag('angular/angular_templates.html', takes_context=True)
+def angular_templates(context):
+    """For all static HTML templates, generate a dictionary of template
+    contents. If the template has been overridden by a theme, load the
+    override contents instead of the original HTML file.  One use for
+    this is to pre-populate the angular template cache.
+
+    Args:
+        context: the context of the current Django template
+
+    Returns: an object containing
+     angular_templates: dictionary of angular template contents
+      - key is the template's static path,
+      - value is a string of HTML template contents
+    """
+    template_paths = context['HORIZON_CONFIG']['external_templates']
+    all_theme_static_files = context['HORIZON_CONFIG']['theme_static_files']
+    this_theme_static_files = all_theme_static_files[context['THEME']]
+    template_overrides = this_theme_static_files['template_overrides']
+    angular_templates = {}
+
+    for relative_path in template_paths:
+
+        template_static_path = context['STATIC_URL'] + relative_path
+
+        # If the current theme overrides this template, use the theme
+        # content instead of the original file content
+        if relative_path in template_overrides:
+            relative_path = template_overrides[relative_path]
+
+        result = []
+        for finder in finders.get_finders():
+            result.extend(finder.find(relative_path, True))
+        path = result[-1]
+        try:
+            with open(path) as template_file:
+                angular_templates[template_static_path] = template_file.read()
+        except (OSError, IOError):
+            # Failed to read template, leave the template dictionary blank
+            # If the caller is using this dictionary to pre-populate a cache
+            # there will simply be no pre-loaded version for this template.
+            pass
+
+    return {
+        'angular_templates': angular_templates
+    }

horizon/templatetags/horizon.py

@@ -211,6 +211,11 @@ def datepicker_locale():
     return locale_mapping.get(translation.get_language(), 'en')
 
 
+@register.assignment_tag
+def template_cache_age():
+    return getattr(settings, 'NG_TEMPLATE_CACHE_AGE', 0)
+
+
 @register.tag
 def minifyspace(parser, token):
     """Removes whitespace including tab and newline characters. Do not use this

horizon/themes.py

@@ -149,6 +149,9 @@ class ThemeTemplateLoader(tLoaderCls):
             elif template_name.find(template_path) != -1:
                 yield template_name
 
+        except SuspiciousFileOperation:
+            # In case we are loading a theme outside of Django, pass along
+            pass
         except UnicodeDecodeError:
             # The template dir name wasn't valid UTF-8.
             raise

openstack_dashboard/settings.py

@@ -56,6 +56,7 @@ MEDIA_URL = None
 STATIC_ROOT = None
 STATIC_URL = None
 INTEGRATION_TESTS_SUPPORT = False
+NG_TEMPLATE_CACHE_AGE = 2592000
 
 ROOT_URLCONF = 'openstack_dashboard.urls'
 
@@ -307,6 +308,7 @@ try:
 except ImportError:
     logging.warning("No local_settings file found.")
 
+# Template loaders
 if DEBUG:
     TEMPLATE_LOADERS += CACHED_TEMPLATE_LOADERS + tuple(ADD_TEMPLATE_LOADERS)
 else:
@@ -314,6 +316,8 @@ else:
         ('django.template.loaders.cached.Loader', CACHED_TEMPLATE_LOADERS),
     ) + tuple(ADD_TEMPLATE_LOADERS)
 
+NG_TEMPLATE_CACHE_AGE = NG_TEMPLATE_CACHE_AGE if not DEBUG else 0
+
 # allow to drop settings snippets into a local_settings_dir
 LOCAL_SETTINGS_DIR_PATH = os.path.join(ROOT_PATH, "local", "local_settings.d")
 if os.path.exists(LOCAL_SETTINGS_DIR_PATH):
@@ -376,7 +380,8 @@ if DEFAULT_THEME_PATH is not None:
 
 # populate HORIZON_CONFIG with auto-discovered JavaScript sources, mock files,
 # specs files and external templates.
-find_static_files(HORIZON_CONFIG)
+find_static_files(HORIZON_CONFIG, AVAILABLE_THEMES,
+                  THEME_COLLECTION_DIR, ROOT_PATH)
 
 # Ensure that we always have a SECRET_KEY set, even when no local_settings.py
 # file is present. See local_settings.py.example for full documentation on the
@@ -416,12 +421,15 @@ def check(actions, request, target=None):
 if POLICY_CHECK_FUNCTION is None:
     POLICY_CHECK_FUNCTION = check
 
+NG_TEMPLATE_CACHE_AGE = NG_TEMPLATE_CACHE_AGE if not DEBUG else 0
+
 # This base context objects gets added to the offline context generator
 # for each theme configured.
 HORIZON_COMPRESS_OFFLINE_CONTEXT_BASE = {
     'WEBROOT': WEBROOT,
     'STATIC_URL': STATIC_URL,
-    'HORIZON_CONFIG': HORIZON_CONFIG
+    'HORIZON_CONFIG': HORIZON_CONFIG,
+    'NG_TEMPLATE_CACHE_AGE': NG_TEMPLATE_CACHE_AGE,
 }
 
 if DEBUG:

openstack_dashboard/static_settings.py

@@ -47,6 +47,8 @@ import xstatic.pkg.termjs
 
 from horizon.utils import file_discovery
 
+from openstack_dashboard import theme_settings
+
 
 def get_staticfiles_dirs(webroot='/'):
     STATICFILES_DIRS = [
@@ -139,7 +141,11 @@ def get_staticfiles_dirs(webroot='/'):
     return STATICFILES_DIRS
 
 
-def find_static_files(HORIZON_CONFIG):
+def find_static_files(
+        HORIZON_CONFIG,
+        AVAILABLE_THEMES,
+        THEME_COLLECTION_DIR,
+        ROOT_PATH):
     import horizon
     import openstack_dashboard
     os_dashboard_home_dir = openstack_dashboard.__path__[0]
@@ -163,3 +169,48 @@ def find_static_files(HORIZON_CONFIG):
         os.path.join(os_dashboard_home_dir, 'static/'),
         sub_path='app/'
     )
+
+    # Discover theme static resources, and in particular any
+    # static HTML (client-side) that the theme overrides
+    theme_static_files = {}
+    theme_info = theme_settings.get_theme_static_dirs(
+        AVAILABLE_THEMES,
+        THEME_COLLECTION_DIR,
+        ROOT_PATH)
+
+    for url, path in theme_info:
+        discovered_files = {}
+
+        # discover static files provided by the theme
+        file_discovery.populate_horizon_config(
+            discovered_files,
+            path
+        )
+
+        # Get the theme name from the theme url
+        theme_name = url.split('/')[-1]
+
+        # build a dictionary of this theme's static HTML templates.
+        # For each overridden template, strip off the '/templates/' part of the
+        # theme filename then use that name as the key, and the location in the
+        # theme directory as the value. This allows the quick lookup of
+        # theme path for any file overridden by a theme template
+        template_overrides = {}
+        for theme_file in discovered_files['external_templates']:
+            # Example:
+            #   external_templates_dict[
+            #       'framework/widgets/help-panel/help-panel.html'
+            #   ] = 'themes/material/templates/framework/widgets/\
+            #        help-panel/help-panel.html'
+            (templates_part, override_path) = theme_file.split('/templates/')
+            template_overrides[override_path] = 'themes/' +\
+                                                theme_name + theme_file
+
+        discovered_files['template_overrides'] = template_overrides
+
+        # Save all of the discovered file info for this theme in our
+        # 'theme_files' object using the theme name as the key
+        theme_static_files[theme_name] = discovered_files
+
+    # Add the theme file info to the horizon config for use by template tags
+    HORIZON_CONFIG['theme_static_files'] = theme_static_files

openstack_dashboard/templates/angular/angular_templates.html

@@ -0,0 +1,5 @@
+{% for static_path, template_html in angular_templates.items %}
+  <script type='text/javascript' id='{{ static_path }}'>
+    {% include 'angular/angular_templates.js' %}
+  </script>
+{% endfor %}

openstack_dashboard/templates/angular/angular_templates.js

@@ -0,0 +1,11 @@
+{% autoescape off %}
+{% load angular_escapes from angular %}
+angular
+ .module('horizon.app')
+ .run(['$templateCache', function($templateCache) {
+   $templateCache.put(
+     "{{ static_path }}",
+     "{{ template_html|angular_escapes }}"
+   );
+}]);
+{% endautoescape %}

openstack_dashboard/templates/horizon/_scripts.html

@@ -1,7 +1,13 @@
 {% load compress %}
 {% load datepicker_locale from horizon %}
+{% load template_cache_age from horizon %}
+{% load themes %}
+{% load cache %}
+{% load angular_templates from angular %}
 
 {% datepicker_locale as DATEPICKER_LOCALE %}
+{% current_theme as THEME %}
+{% template_cache_age as NG_TEMPLATE_CACHE_AGE %}
 
 {% include "horizon/_script_i18n.html" %}
 
@@ -66,6 +72,13 @@
 {% endfor %}
 
 {% block custom_js_files %}{% endblock %}
+
+{% endcompress %}
+
+{% compress js file angular_template_cache_preloads %}
+  {% cache NG_TEMPLATE_CACHE_AGE angular 'template_cache_preloads' THEME %}
+    {% angular_templates %}
+  {% endcache %}
 {% endcompress %}
 
 {% comment %} Client-side Templates (These should *not* be inside the "compress" tag.) {% endcomment %}

openstack_dashboard/test/settings.py

@@ -12,6 +12,7 @@
 
 import os
 
+from django.utils.translation import pgettext_lazy
 from horizon.test.settings import *  # noqa
 from horizon.utils import secret_key
 from openstack_dashboard import exceptions
@@ -43,6 +44,22 @@ TEMPLATE_DIRS = (
 
 CUSTOM_THEME_PATH = 'themes/default'
 
+# 'key', 'label', 'path'
+AVAILABLE_THEMES = [
+    (
+        'default',
+        pgettext_lazy('Default style theme', 'Default'),
+        'themes/default'
+    ), (
+        'material',
+        pgettext_lazy("Google's Material Design style theme", "Material"),
+        'themes/material'
+    ),
+]
+
+# Theme Static Directory
+THEME_COLLECTION_DIR = 'themes'
+
 TEMPLATE_CONTEXT_PROCESSORS += (
     'openstack_dashboard.context_processors.openstack',
 )
@@ -98,7 +115,8 @@ settings.update_dashboards(
 # the stacks MappingsTests are updated with the new URL path.
 HORIZON_CONFIG['swift_panel'] = 'legacy'
 
-find_static_files(HORIZON_CONFIG)
+find_static_files(HORIZON_CONFIG, AVAILABLE_THEMES,
+                  THEME_COLLECTION_DIR, ROOT_PATH)
 
 # Set to True to allow users to upload images to glance via Horizon server.
 # When enabled, a file form field will appear on the create image form.

releasenotes/notes/bp-angular-template-overrides-9f05ffd61367245a.yaml

@@ -0,0 +1,11 @@
+---
+features:
+  - >
+    [`blueprint angular-template-overrides <https://blueprints.launchpad.net/horizon/+spec/angular-template-overrides>`_]
+    This blueprint provides a way for deployers to use a theme to override HTML
+    fragments used by Angular code in Horizon.  For example, to override the
+    launch instance help panel when the 'material' theme is used, create
+    openstack_dashboard/themes/material/static/templates/framework
+    /widgets/help-panel/help-panel.html. All of the client side templates are
+    now compiled into a single JavaScript file that is minified and is given
+    as an additional file in the manifest.json file.