From b93fb120b7f5ab2c190cb021f127c39631f0f5b3 Mon Sep 17 00:00:00 2001 From: Ekaterina Fedorova Date: Thu, 9 Jan 2014 14:53:40 +0400 Subject: [PATCH] Cherry-pick the following commits from master: * Extend exception messages. Closes-bug: #1264980 Ia57821cd906a203b18546b9e93e38f3e1bc71025 * Remove extra methods of interaction with API: and use environment_get instead it. Closes-Bug: #1265165 Ib49aecff4773773b6cf305d51db29a17bcf813f3 * Move most of code for dynamic UI form creation into metaclass. Implements: blueprint dynamic-ui-optimization I9b2617527b410abb7c60df978f9c00f7cef491d3 * Minor refactoring of dynamic_ui. Move more functions to dynamic_ui.helpers. Ib578a24159dda4de5fecf5df35ff71bc7d704215 * Rewrite dynamic_ui to store Services data per-session. Thus set of Services (and cleaned_data for them) for each user/tenant will be isolated supporting per-tenant isolation blueprint. Closes-bug: #1264289 I3d0b46463470912cf6d7a36fddd84292689775da * Update local_settings.py.example to stable/havana. If56e74338449eb14b6b8a581863502287e654517 * Hide "Upload UI file" btn in manage service table There is should be only one ui definition in service So need to show "Upload UI file" only ic case there is no any Closes-bug: #1263052 Ibe8c74f20062cd213d8a53ff46d9db9d41a2e08d * Support per-tenant isolation for service metadata files. Partially implements: blueprint per-tenant-isolation I7393e748216ddaa59d6e90249b263514d08f9d34 * Added empty line in KeyPair fields. Implements: blueprint rewrite-key-pair-for-linux-based-services If7dcf19084422c76d3bd1b075e4d5080254d003b Change-Id: I10b920a3b7cdd9b9a19d37243be81f6aa6aafa9b --- muranodashboard/dynamic_ui/fields.py | 93 ++-- muranodashboard/dynamic_ui/forms.py | 279 ++++++------ muranodashboard/dynamic_ui/helpers.py | 45 ++ muranodashboard/dynamic_ui/metadata.py | 32 +- muranodashboard/dynamic_ui/services.py | 149 ++++-- muranodashboard/environments/api.py | 26 +- muranodashboard/environments/consts.py | 1 - muranodashboard/environments/forms.py | 9 +- muranodashboard/environments/tables.py | 13 +- muranodashboard/environments/views.py | 21 +- .../local/local_settings.py.example | 429 +++++++++++++++--- muranodashboard/service_catalog/forms.py | 11 +- muranodashboard/service_catalog/tables.py | 53 +-- muranodashboard/service_catalog/utils.py | 5 + muranodashboard/service_catalog/views.py | 8 +- muranodashboard/settings.py | 8 +- setup.sh | 90 ++-- 17 files changed, 820 insertions(+), 452 deletions(-) diff --git a/muranodashboard/dynamic_ui/fields.py b/muranodashboard/dynamic_ui/fields.py index 9b2b331c4..656d09270 100644 --- a/muranodashboard/dynamic_ui/fields.py +++ b/muranodashboard/dynamic_ui/fields.py @@ -36,18 +36,6 @@ from django.template.loader import render_to_string log = logging.getLogger(__name__) -YAQL_FUNCTIONS = { - 'test': lambda self, pattern: re.match(pattern(), self()) is not None, -} - - -def create_yaql_context(functions=YAQL_FUNCTIONS): - context = yaql.create_context() - for name, func in functions.iteritems(): - context.register_function(func, name) - return context - - def with_request(func): """The decorator is meant to be used together with `UpdatableFieldsForm': apply it to the `update' method of fields inside that form. @@ -75,7 +63,7 @@ def make_yaql_validator(field, form, key, validator_property): def validator_func(value): data = getattr(form, 'cleaned_data', {}) data[key] = value - form.service.update_cleaned_data(form, data) + form.service.update_cleaned_data(data, form=form) if not validator_property['expr'].__get__(field): raise forms.ValidationError( _(validator_property.get('message', ''))) @@ -130,8 +118,31 @@ def get_murano_images(request): return murano_images +class RawProperty(object): + def __init__(self, key, spec): + self.key = key + self.spec = spec + + def finalize(self, form_name, service): + def _get(field): + data_ready, value = service.get_data(form_name, self.spec) + return value if data_ready else field.__dict__[self.key] + + def _set(field, value): + field.__dict__[self.key] = value + + def _del(field): + del field.__dict__[self.key] + return property(_get, _set, _del) + + class CustomPropertiesField(forms.Field): - def __init__(self, form=None, key=None, *args, **kwargs): + def __init__(self, description=None, description_title=None, + *args, **kwargs): + self.description = description + self.description_title = (description_title or + unicode(kwargs.get('label', ''))) + validators = [] for validator in kwargs.get('validators', []): if hasattr(validator, '__call__'): # single regexpValidator @@ -142,31 +153,33 @@ class CustomPropertiesField(forms.Field): if regex_validator: validators.append(wrap_regex_validator( regex_validator, validator.get('message', ''))) - elif isinstance(expr, property): - if form: - validators.append( - make_yaql_validator(self, form, key, validator)) - + elif isinstance(expr, RawProperty): + validators.append(validator) kwargs['validators'] = validators + super(CustomPropertiesField, self).__init__(*args, **kwargs) def clean(self, value): """Skip all validators if field is disabled.""" + # form is assigned in ServiceConfigurationForm.finalize_fields() + form = self.form + # the only place to ensure that Service object has up-to-date + # cleaned_data + form.service.update_cleaned_data(form.cleaned_data, form=form) if getattr(self, 'enabled', True): return super(CustomPropertiesField, self).clean(value) else: return super(CustomPropertiesField, self).to_python(value) @classmethod - def push_properties(cls, kwargs): + def finalize_properties(cls, kwargs, form_name, service): props = {} - for key, value in kwargs.iteritems(): - if isinstance(value, property): - props[key] = value - for key in props.keys(): - del kwargs[key] + for key, value in kwargs.items(): + if isinstance(value, RawProperty): + props[key] = value.finalize(form_name, service) + del kwargs[key] if props: - return type('cls_with_props', (cls,), props) + return type(cls.__name__, (cls,), props) else: return cls @@ -213,14 +226,17 @@ class PasswordField(CharField): if err_msg.get('required'): error_messages['required'] = err_msg.get('required') - super(PasswordField, self).__init__( - min_length=7, - max_length=255, - validators=[self.validate_password], - label=label, - error_messages=error_messages, - help_text=help_text, - widget=self.PasswordInput(render_value=True)) + kwargs.update({ + 'min_length': 7, + 'max_length': 255, + 'validators': [self.validate_password], + 'label': label, + 'error_messages': error_messages, + 'help_text': help_text, + 'widget': self.PasswordInput(render_value=True), + }) + + super(PasswordField, self).__init__(*args, **kwargs) def __deepcopy__(self, memo): result = super(PasswordField, self).__deepcopy__(memo) @@ -372,7 +388,7 @@ class TableWidget(floppyforms.widgets.Input): self.max_sync = max_sync # FixME: we need to use this hack because TableField passes all kwargs # to TableWidget - for kwarg in ('widget', 'key', 'form'): + for kwarg in ('widget', 'description', 'description_title'): ignorable = kwargs.pop(kwarg, None) super(TableWidget, self).__init__(*args, **kwargs) @@ -480,11 +496,12 @@ class FlavorChoiceField(ChoiceField): class KeyPairChoiceField(ChoiceField): - " This widget allows to select Key Pair for VMs " + " This widget allows to select keypair for VMs " @with_request def update(self, request, **kwargs): - self.choices = [(keypair.name, keypair.name) for keypair in - novaclient(request).keypairs.list()] + self.choices = [('', _('No keypair'))] + for keypair in novaclient(request).keypairs.list(): + self.choices.append((keypair.name, keypair.name)) class ImageChoiceField(ChoiceField): diff --git a/muranodashboard/dynamic_ui/forms.py b/muranodashboard/dynamic_ui/forms.py index 1d04c3fa4..0e7e31f02 100644 --- a/muranodashboard/dynamic_ui/forms.py +++ b/muranodashboard/dynamic_ui/forms.py @@ -12,12 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. -import re import logging import types from django import forms -from django.core.validators import RegexValidator from django.utils.translation import ugettext_lazy as _ import muranodashboard.dynamic_ui.fields as fields import muranodashboard.dynamic_ui.helpers as helpers @@ -26,6 +24,105 @@ import yaql log = logging.getLogger(__name__) +TYPES = { + 'string': fields.CharField, + 'boolean': fields.BooleanField, + 'instance': fields.InstanceCountField, + 'clusterip': fields.ClusterIPField, + 'domain': fields.DomainChoiceField, + 'password': fields.PasswordField, + 'integer': fields.IntegerField, + 'databaselist': fields.DatabaseListField, + 'table': fields.TableField, + 'flavor': fields.FlavorChoiceField, + 'keypair': fields.KeyPairChoiceField, + 'image': fields.ImageChoiceField, + 'azone': fields.AZoneChoiceField, + 'text': (fields.CharField, forms.Textarea) +} + + +def _collect_fields(field_specs, form_name, service): + def process_widget(kwargs, cls, widget): + widget = kwargs.get('widget', widget) + if widget is None: + widget = cls.widget + if 'widget_media' in kwargs: + media = kwargs['widget_media'] + del kwargs['widget_media'] + + class Widget(widget): + class Media: + js = media.get('js', ()) + css = media.get('css', {}) + widget = Widget + + if 'widget_attrs' in kwargs: + widget = widget(attrs=kwargs['widget_attrs']) + del kwargs['widget_attrs'] + return widget + + def parse_spec(spec, keys=None): + if keys is None: + keys = [] + + if not isinstance(keys, types.ListType): + keys = [keys] + key = keys and keys[-1] or None + if helpers.get_yaql_expr(spec): + return key, fields.RawProperty(key, spec) + elif isinstance(spec, types.DictType): + items = [] + for k, v in spec.iteritems(): + if not k in ('type', 'name'): + k = helpers.decamelize(k) + new_key, v = parse_spec(v, keys + [k]) + if new_key: + k = new_key + items.append((k, v)) + return key, dict(items) + elif isinstance(spec, types.ListType): + return key, [parse_spec(_spec, keys)[1] for _spec in spec] + elif isinstance(spec, basestring) and helpers.is_localizable(keys): + return key, _(spec) + else: + if key == 'type': + return key, TYPES[spec] + elif key == 'hidden' and spec is True: + return 'widget', forms.HiddenInput + elif key == 'regexp_validator': + return 'validators', [helpers.prepare_regexp(spec)] + else: + return key, spec + + def make_field(field_spec, form_name, service): + cls = parse_spec(field_spec['type'], 'type')[1] + widget = None + if isinstance(cls, types.TupleType): + cls, widget = cls + kwargs = parse_spec(field_spec)[1] + kwargs['widget'] = process_widget(kwargs, cls, widget) + cls = cls.finalize_properties(kwargs, form_name, service) + + attribute_names = kwargs.pop('attribute_names', None) + field = cls(**kwargs) + field.attribute_names = attribute_names + + return field_spec['name'], field + + return [make_field(spec, form_name, service) for spec in field_specs] + + +class DynamicFormMetaclass(forms.forms.DeclarativeFieldsMetaclass): + def __new__(meta, name, bases, dct): + name = dct.pop('name', name) + field_specs = dct.pop('field_specs', []) + service = dct['service'] + for field_name, field in _collect_fields(field_specs, name, service): + dct[field_name] = field + return super(DynamicFormMetaclass, meta).__new__( + meta, name, bases, dct) + class UpdatableFieldsForm(forms.Form): """This class is supposed to be a base for forms belonging to a FormWizard @@ -59,39 +156,30 @@ class UpdatableFieldsForm(forms.Form): class ServiceConfigurationForm(UpdatableFieldsForm): - types = { - 'string': fields.CharField, - 'boolean': fields.BooleanField, - 'instance': fields.InstanceCountField, - 'clusterip': fields.ClusterIPField, - 'domain': fields.DomainChoiceField, - 'password': fields.PasswordField, - 'integer': fields.IntegerField, - 'databaselist': fields.DatabaseListField, - 'table': fields.TableField, - 'flavor': fields.FlavorChoiceField, - 'keypair': fields.KeyPairChoiceField, - 'image': fields.ImageChoiceField, - 'azone': fields.AZoneChoiceField, - 'text': (fields.CharField, forms.Textarea) - } - - localizable_keys = set(['label', 'help_text', 'error_messages']) - def __init__(self, *args, **kwargs): log.info("Creating form {0}".format(self.__class__.__name__)) super(ServiceConfigurationForm, self).__init__(*args, **kwargs) self.attribute_mappings = {} - self.context = fields.create_yaql_context() - self.insert_fields(self.fields_template) + self.context = helpers.create_yaql_context() + self.finalize_fields() self.initial = kwargs.get('initial', self.initial) self.update_fields() - @staticmethod - def get_yaql_expr(expr): - return isinstance(expr, types.DictType) and expr.get('YAQL', None) + def finalize_fields(self): + for field_name, field in self.fields.iteritems(): + field.form = self - def init_attribute_mappings(self, field_name, kwargs): + validators = [] + for v in field.validators: + expr = isinstance(v, types.DictType) and v.get('expr') + if expr and isinstance(expr, fields.RawProperty): + v = fields.make_yaql_validator(field, self, field_name, v) + validators.append(v) + field.validators = validators + + self.init_attribute_mapping(field_name, field) + + def init_attribute_mapping(self, field_name, field): def set_mapping(name, value): """Spawns new dictionaries for each dot found in name.""" bits = name.split('.') @@ -102,8 +190,8 @@ class ServiceConfigurationForm(UpdatableFieldsForm): head, tail, mapping = tail[0], tail[1:], mapping[head] mapping[head] = value - if 'attribute_names' in kwargs: - attr_names = kwargs['attribute_names'] + attr_names = field.attribute_names + if attr_names is not None: if isinstance(attr_names, types.ListType): # allow pushing field value to multiple attributes for attr_name in attr_names: @@ -111,131 +199,17 @@ class ServiceConfigurationForm(UpdatableFieldsForm): elif attr_names: # if attributeNames = false, do not push field value set_mapping(attr_names, field_name) - del kwargs['attribute_names'] else: # default mapping: field to attr with same name # do not spawn new dictionaries for any dot in field_name self.attribute_mappings[field_name] = field_name - - def init_field_descriptions(self, kwargs): - if 'description' in kwargs: - del kwargs['description'] - if 'description_title' in kwargs: - del kwargs['description_title'] - - def insert_fields(self, field_specs): - def process_widget(kwargs, cls, widget): - widget = kwargs.get('widget', widget) - if widget is None: - widget = cls.widget - if 'widget_media' in kwargs: - media = kwargs['widget_media'] - del kwargs['widget_media'] - - class Widget(widget): - class Media: - js = media.get('js', ()) - css = media.get('css', {}) - widget = Widget - - if 'widget_attrs' in kwargs: - widget = widget(attrs=kwargs['widget_attrs']) - del kwargs['widget_attrs'] - return widget - - def append_field(field_spec): - cls = parse_spec(field_spec['type'], 'type')[1] - widget = None - if isinstance(cls, types.TupleType): - cls, widget = cls - kwargs = parse_spec(field_spec)[1] - kwargs.update({ - 'widget': process_widget(kwargs, cls, widget), - 'form': self, - 'key': field_spec['name'] - }) - cls = cls.push_properties(kwargs) - - self.init_attribute_mappings(field_spec['name'], kwargs) - self.init_field_descriptions(kwargs) - self.fields.insert(len(self.fields), - field_spec['name'], - cls(**kwargs)) - - def prepare_regexp(regexp): - if regexp[0] == '/': - groups = re.match(r'^/(.*)/([A-Za-z]*)$', regexp).groups() - regexp, flags_str = groups - flags = 0 - for flag in helpers.explode(flags_str): - flag = flag.upper() - if hasattr(re, flag): - flags |= getattr(re, flag) - return RegexValidator(re.compile(regexp, flags)) - else: - return RegexValidator(re.compile(regexp)) - - def is_localizable(keys): - return set(keys).intersection(self.localizable_keys) - - def make_property(key, spec): - def _get(field): - data_ready, value = self.get_data(spec) - return value if data_ready else field.__dict__[key] - - def _set(field, value): - field.__dict__[key] = value - - def _del(field): - del field.__dict__[key] - return property(_get, _set, _del) - - def parse_spec(spec, keys=[]): - if not isinstance(keys, types.ListType): - keys = [keys] - key = keys and keys[-1] or None - if self.get_yaql_expr(spec): - return key, make_property(key, spec) - elif isinstance(spec, types.DictType): - items = [] - for k, v in spec.iteritems(): - if not k in ('type', 'name'): - k = helpers.decamelize(k) - newKey, v = parse_spec(v, keys + [k]) - if newKey: - k = newKey - items.append((k, v)) - return key, dict(items) - elif isinstance(spec, types.ListType): - return key, [parse_spec(_spec, keys)[1] for _spec in spec] - elif isinstance(spec, basestring) and is_localizable(keys): - return key, _(spec) - else: - if key == 'type': - return key, self.types[spec] - elif key == 'hidden' and spec is True: - return 'widget', forms.HiddenInput - elif key == 'regexp_validator': - return 'validators', [prepare_regexp(spec)] - else: - return key, spec - - for spec in field_specs: - append_field(spec) - - def get_data(self, expr, data=None): - """First try to get value from cleaned data, if none - found, use raw data.""" - data = self.service.update_cleaned_data( - self, data or getattr(self, 'cleaned_data', None)) - expr = self.get_yaql_expr(expr) - value = data and yaql.parse(expr).evaluate(data, self.context) - return data != {}, value + del field.attribute_names def get_unit_templates(self, data): def parse_spec(spec): - if self.get_yaql_expr(spec): - data_ready, value = self.get_data(spec, data) + if helpers.get_yaql_expr(spec): + data_ready, value = self.service.get_data( + self.__class__.__name__, spec, data) return value elif isinstance(spec, types.ListType): return [parse_spec(_spec) for _spec in spec] @@ -245,8 +219,8 @@ class ServiceConfigurationForm(UpdatableFieldsForm): for (k, v) in spec.iteritems()) else: return spec - #TODO: add try-except if unit_templates is not in service description - return [parse_spec(spec) for spec in self.service.unit_templates] + unit_templates = getattr(self.service, 'unit_templates', []) + return [parse_spec(spec) for spec in unit_templates] def extract_attributes(self, attributes): def get_attr(name): @@ -262,10 +236,11 @@ class ServiceConfigurationForm(UpdatableFieldsForm): return self.cleaned_data else: cleaned_data = super(ServiceConfigurationForm, self).clean() - all_data = self.service.update_cleaned_data(self, cleaned_data) + all_data = self.service.update_cleaned_data( + cleaned_data, form=self) error_messages = [] for validator in self.validators: - expr = self.get_yaql_expr(validator['expr']) + expr = helpers.get_yaql_expr(validator['expr']) if not yaql.parse(expr).evaluate(all_data, self.context): error_messages.append(_(validator.get('message', ''))) if error_messages: @@ -282,5 +257,5 @@ class ServiceConfigurationForm(UpdatableFieldsForm): cleaned_data[name] = value log.debug("Update cleaned data in postclean method") - self.service.update_cleaned_data(self, cleaned_data) + self.service.update_cleaned_data(cleaned_data, form=self) return cleaned_data diff --git a/muranodashboard/dynamic_ui/helpers.py b/muranodashboard/dynamic_ui/helpers.py index 1292064cc..afefd68a8 100644 --- a/muranodashboard/dynamic_ui/helpers.py +++ b/muranodashboard/dynamic_ui/helpers.py @@ -13,13 +13,28 @@ # under the License. import re +from django.core.validators import RegexValidator +import types +import yaql + +_LOCALIZABLE_KEYS = set(['label', 'help_text', 'error_messages']) + +YAQL_FUNCTIONS = { + 'test': lambda self, pattern: re.match(pattern(), self()) is not None, +} + + +def is_localizable(keys): + return set(keys).intersection(_LOCALIZABLE_KEYS) def camelize(name): + """Turns snake_case name into SnakeCase.""" return ''.join([bit.capitalize() for bit in name.split('_')]) def decamelize(name): + """Turns CamelCase/camelCase name into camel_case.""" pat = re.compile(r'([A-Z]*[^A-Z]*)(.*)') bits = [] while True: @@ -33,6 +48,7 @@ def decamelize(name): def explode(string): + """Explodes a string into a list of one-character strings.""" if not string: return string bits = [] @@ -44,3 +60,32 @@ def explode(string): else: break return bits + + +def prepare_regexp(regexp): + """Converts regular expression string pattern into RegexValidator object. + + Also /regexp/flags syntax is allowed, where flags is a string of + one-character flags that will be appended to the compiled regexp.""" + if regexp.startswith('/'): + groups = re.match(r'^/(.*)/([A-Za-z]*)$', regexp).groups() + regexp, flags_str = groups + flags = 0 + for flag in explode(flags_str): + flag = flag.upper() + if hasattr(re, flag): + flags |= getattr(re, flag) + return RegexValidator(re.compile(regexp, flags)) + else: + return RegexValidator(re.compile(regexp)) + + +def get_yaql_expr(expr): + return isinstance(expr, types.DictType) and expr.get('YAQL', None) + + +def create_yaql_context(functions=YAQL_FUNCTIONS): + context = yaql.create_context() + for name, func in functions.iteritems(): + context.register_function(func, name) + return context diff --git a/muranodashboard/dynamic_ui/metadata.py b/muranodashboard/dynamic_ui/metadata.py index 1f050af1a..25625cb25 100644 --- a/muranodashboard/dynamic_ui/metadata.py +++ b/muranodashboard/dynamic_ui/metadata.py @@ -20,7 +20,7 @@ import logging import shutil import hashlib from muranodashboard.environments.consts import CHUNK_SIZE, CACHE_DIR, \ - ARCHIVE_PKG_PATH + ARCHIVE_PKG_NAME log = logging.getLogger(__name__) @@ -65,8 +65,12 @@ def metadataclient(request): return Client(endpoint=endpoint, token=token_id) -def get_existing_hash(): - for item in os.listdir(CACHE_DIR): +def get_tenant_dir(tenant_id): + return os.path.join(CACHE_DIR, tenant_id) + + +def get_existing_hash(tenant_dir): + for item in os.listdir(tenant_dir): if re.match(r'[a-f0-9]{40}', item): return item return None @@ -94,11 +98,11 @@ def get_hash(archive_path): return None -def unpack_ui_package(archive_path): +def unpack_ui_package(archive_path, tenant_dir): if not tarfile.is_tarfile(archive_path): raise RuntimeError('{0} is not valid tarfile!'.format(archive_path)) hash = get_hash(archive_path) - dst_dir = os.path.join(CACHE_DIR, hash) + dst_dir = os.path.join(tenant_dir, hash) if not os.path.exists(dst_dir): os.mkdir(dst_dir) else: @@ -138,9 +142,14 @@ def get_ui_metadata(request): occurred, returns None. """ log.debug("Retrieving metadata from Repository") - hash = get_existing_hash() + tenant_dir = get_tenant_dir(request.user.tenant_id) + if not os.path.exists(tenant_dir): + os.makedirs(tenant_dir) + + hash = get_existing_hash(tenant_dir) + metadata_dir = None if hash: - metadata_dir = os.path.join(CACHE_DIR, hash) + metadata_dir = os.path.join(tenant_dir, hash) data = None with metadata_exceptions(request): @@ -158,19 +167,20 @@ def get_ui_metadata(request): with tempfile.NamedTemporaryFile(delete=False) as out: for chunk in body_iter: out.write(chunk) - shutil.move(out.name, ARCHIVE_PKG_PATH) + archive_pkg_path = os.path.join(tenant_dir, ARCHIVE_PKG_NAME) + shutil.move(out.name, archive_pkg_path) log.info("Successfully downloaded new metadata package to {0}".format( - ARCHIVE_PKG_PATH)) + archive_pkg_path)) if hash: log.debug('Removing outdated metadata: {0}'.format(metadata_dir)) shutil.rmtree(metadata_dir) - return unpack_ui_package(ARCHIVE_PKG_PATH), True + return unpack_ui_package(archive_pkg_path, tenant_dir), True elif code == 304: log.info("Metadata package hash-sum hasn't changed, doing nothing") return metadata_dir, False else: msg = 'Unexpected response received: {0}'.format(code) - if hash: + if metadata_dir: log.error('Using existing version of metadata ' 'which may be outdated due to: {0}'.format(msg)) return metadata_dir, False diff --git a/muranodashboard/dynamic_ui/services.py b/muranodashboard/dynamic_ui/services.py index bf4a53aa6..b5bcd8383 100644 --- a/muranodashboard/dynamic_ui/services.py +++ b/muranodashboard/dynamic_ui/services.py @@ -18,12 +18,14 @@ import re import time import logging from muranodashboard.dynamic_ui import metadata -from muranodashboard.dynamic_ui.helpers import decamelize +from .helpers import decamelize, get_yaql_expr, create_yaql_context + try: from collections import OrderedDict except ImportError: # python2.6 from ordereddict import OrderedDict +import yaql import yaml from yaml.scanner import ScannerError from django.utils.translation import ugettext_lazy as _ @@ -31,44 +33,98 @@ import copy from muranodashboard.environments.consts import CACHE_REFRESH_SECONDS_INTERVAL log = logging.getLogger(__name__) -_all_services = OrderedDict() -_last_check_time = 0 -_current_cache_hash = None class Service(object): - def __init__(self, **kwargs): - import muranodashboard.dynamic_ui.forms as services - for key, value in kwargs.iteritems(): - if key == 'forms': - self.forms = [] - for form_data in value: - form_name, form_data = self.extract_form_data(form_data) - self.forms.append( - type(form_name, (services.ServiceConfigurationForm,), - {'service': self, - 'fields_template': form_data['fields'], - 'validators': form_data.get('validators', [])})) - else: - setattr(self, key, value) + """Class for keeping service persistent data, the most important are two: + ``self.forms`` list of service's steps (as Django form classes) and + ``self.cleaned_data`` dictionary of data from service validated steps. + + Attribute ``self.cleaned_data`` is needed for, e.g. ServiceA.Step2, be + able to reference data at ServiceA.Step1 while actual form instance + representing Step1 is already gone. + + Because the need to store this data per-user, sessions must be employed + (actually, they are not the _only_ way of doing this, but the most simple + one), and because every Django session backend uses pickle serialization, + __getstate__/__setstate__ methods for custom pickle serialization must be + implemented. + """ + NON_SERIALIZABLE_ATTRS = ('forms', 'context') + + def __init__(self, forms=None, **kwargs): + self.context = create_yaql_context() self.cleaned_data = {} + self.forms = [] + self._forms = [] + for key, value in kwargs.iteritems(): + setattr(self, key, value) + + if forms: + for data in forms: + name, field_specs, validators = self.extract_form_data(data) + self._add_form(name, field_specs, validators) + + # for pickling/unpickling + self._forms.append((name, field_specs, validators)) + + def __getstate__(self): + log.debug("Pickling service '{service.type}'".format( + service=self)) + dct = dict((k, v) for (k, v) in self.__dict__.iteritems() + if not k in self.NON_SERIALIZABLE_ATTRS) + return dct + + def __setstate__(self, d): + log.debug("Unpickling service '{type}'".format(**d)) + for k, v in d.iteritems(): + setattr(self, k, v) + # dealing with the attributes which cannot be serialized (see + # http://tinyurl.com/kxx3tam on pickle restrictions ) + # yaql context is not serializable because it contains lambda functions + self.context = create_yaql_context() + # form classes are not serializable 'cause they are defined dynamically + self.forms = [] + for name, field_specs, validators in d.get('_forms', []): + self._add_form(name, field_specs, validators) + + def _add_form(self, _name, _specs, _validators): + import muranodashboard.dynamic_ui.forms as forms + + class Form(forms.ServiceConfigurationForm): + __metaclass__ = forms.DynamicFormMetaclass + + service = self + name = _name + field_specs = _specs + validators = _validators + + self.forms.append(Form) @staticmethod def extract_form_data(form_data): form_name = form_data.keys()[0] - return form_name, form_data[form_name] + form_data = form_data[form_name] + return form_name, form_data['fields'], form_data.get('validators', []) - def update_cleaned_data(self, form, data): + def get_data(self, form_name, expr, data=None): + """First try to get value from cleaned data, if none + found, use raw data.""" if data: - # match = re.match('^.*-(\d)+$', form.prefix) - # index = int(match.group(1)) if match else None - # if index is not None: - # self.cleaned_data[index] = data - self.cleaned_data[form.__class__.__name__] = data + self.update_cleaned_data(data, form_name=form_name) + expr = get_yaql_expr(expr) + data = self.cleaned_data + value = data and yaql.parse(expr).evaluate(data, self.context) + return data != {}, value + + def update_cleaned_data(self, data, form=None, form_name=None): + form_name = form_name or form.__class__.__name__ + if data: + self.cleaned_data[form_name] = data return self.cleaned_data -def import_service(full_service_name, service_file): +def import_service(services, full_service_name, service_file): try: with open(service_file) as stream: yaml_desc = yaml.load(stream) @@ -77,17 +133,9 @@ def import_service(full_service_name, service_file): " reason: {1!s}".format(service_file, e)) else: service = dict((decamelize(k), v) for (k, v) in yaml_desc.iteritems()) - _all_services[full_service_name] = Service(**service) + services[full_service_name] = Service(**service) log.info("Added service '{0}' from '{1}'".format( - _all_services[full_service_name].name, service_file)) - - -def are_caches_in_sync(): - are_in_sync = (_current_cache_hash == metadata.get_existing_hash()) - if not are_in_sync: - log.debug('In-memory and on-disk caches are not in sync, ' - 'invalidating in-memory cache') - return are_in_sync + services[full_service_name].name, service_file)) def import_all_services(request): @@ -105,30 +153,30 @@ def import_all_services(request): If there is no YAMLs with form definitions inside dir, then won't be shown in Create Service first step. """ - global _last_check_time - global _all_services - global _current_cache_hash - if time.time() - _last_check_time > CACHE_REFRESH_SECONDS_INTERVAL: - _last_check_time = time.time() + last_check_time = request.session.get('last_check_time', 0) + if time.time() - last_check_time > CACHE_REFRESH_SECONDS_INTERVAL: + request.session['last_check_time'] = time.time() directory, modified = metadata.get_ui_metadata(request) + session_is_empty = not request.session.get('services', {}) # check directory here in case metadata service is not available # and None is returned as directory value. # TODO: it is better to use redirect for that purpose (if possible) - if modified or (directory and not are_caches_in_sync()): - _all_services = {} + if directory is not None and (modified or session_is_empty): + request.session['services'] = {} for full_service_name in os.listdir(directory): final_dir = os.path.join(directory, full_service_name) if os.path.isdir(final_dir) and len(os.listdir(final_dir)): filename = os.listdir(final_dir)[0] if filename.endswith('.yaml'): - import_service(full_service_name, + import_service(request.session['services'], + full_service_name, os.path.join(final_dir, filename)) - _current_cache_hash = metadata.get_existing_hash() def iterate_over_services(request): import_all_services(request) - for service in sorted(_all_services.values(), key=lambda v: v.name): + services = request.session.get('services', {}) + for service in sorted(services.values(), key=lambda v: v.name): yield service.type, service @@ -166,10 +214,11 @@ def get_service_field_descriptions(request, service_id, index): def get_descriptions(service): form_cls = service.forms[index] descriptions = [] - for field in form_cls.fields_template: - if 'description' in field: - title = field.get('descriptionTitle', field.get('label', '')) - descriptions.append((title, field['description'])) + for field in form_cls.base_fields.itervalues(): + title = field.description_title + description = field.description + if description: + descriptions.append((title, description)) return descriptions return with_service(request, service_id, get_descriptions, []) diff --git a/muranodashboard/environments/api.py b/muranodashboard/environments/api.py index 25afeb0fa..0b803b761 100644 --- a/muranodashboard/environments/api.py +++ b/muranodashboard/environments/api.py @@ -195,7 +195,8 @@ def environment_delete(request, environment_id): def environment_get(request, environment_id): session_id = Session.get(request, environment_id) - log.debug('Environment::Get '.format(environment_id)) + log.debug('Environment::Get '. + format(environment_id, session_id)) env = muranoclient(request).environments.get(environment_id, session_id) log.debug('Environment::Get {0}'.format(env)) return env @@ -214,29 +215,6 @@ def environment_update(request, environment_id, name): return muranoclient(request).environments.update(environment_id, name) -def get_environment_name(request, environment_id): - session_id = Session.get(request, environment_id) - environment = muranoclient(request).environments.get(environment_id, - session_id) - log.debug('Return environment name') - return environment.name - - -def get_environment_data(request, environment_id, *args): - """ - For given list of environment attributes return a values - :return list - """ - - session_id = Session.get(request, environment_id) - environment = muranoclient(request).environments.get(environment_id, - session_id) - result = [] - for attr in args: - result.append(getattr(environment, attr, None)) - return result - - def services_list(request, environment_id): def strip(msg, to=100): return '%s...' % msg[:to] if len(msg) > to else msg diff --git a/muranodashboard/environments/consts.py b/muranodashboard/environments/consts.py index 65a9e32df..5f6607b49 100644 --- a/muranodashboard/environments/consts.py +++ b/muranodashboard/environments/consts.py @@ -22,7 +22,6 @@ CACHE_DIR = getattr(settings, 'METADATA_CACHE_DIR', os.path.join(tempfile.gettempdir(), 'muranodashboard-cache')) -ARCHIVE_PKG_PATH = os.path.join(CACHE_DIR, ARCHIVE_PKG_NAME) CACHE_REFRESH_SECONDS_INTERVAL = 5 #---- Forms Consts ----# diff --git a/muranodashboard/environments/forms.py b/muranodashboard/environments/forms.py index d48c2320f..0fa0afe72 100644 --- a/muranodashboard/environments/forms.py +++ b/muranodashboard/environments/forms.py @@ -17,15 +17,16 @@ import json from django import forms from django.utils.translation import ugettext_lazy as _ from muranodashboard.dynamic_ui.services import get_service_choices -from muranodashboard.dynamic_ui.fields import get_murano_images +from muranodashboard.dynamic_ui.fields import get_murano_images, \ + ImageChoiceField log = logging.getLogger(__name__) def filter_service_by_image_type(service, request): def find_image_field(): for form_cls in service.forms: - for field in form_cls.fields_template: - if field.get('type') == 'image': + for field in form_cls.base_fields.itervalues(): + if isinstance(field, ImageChoiceField): return field return None @@ -34,7 +35,7 @@ def filter_service_by_image_type(service, request): if not image_field: message = "Please provide Image field description in UI definition" return filtered, message - specified_image_type = image_field.get('imageType') + specified_image_type = getattr(image_field, 'image_type', None) if not specified_image_type: message = "Please provide 'imageType' parameter in Image field " \ "description in UI definition" diff --git a/muranodashboard/environments/tables.py b/muranodashboard/environments/tables.py index 2fab85313..d9d31a880 100644 --- a/muranodashboard/environments/tables.py +++ b/muranodashboard/environments/tables.py @@ -36,7 +36,9 @@ class CreateService(tables.LinkAction): def allowed(self, request, environment): environment_id = self.table.kwargs['environment_id'] - status, = api.get_environment_data(request, environment_id, 'status') + env = api.environment_get(request, environment_id) + status = getattr(env, 'status', None) + if status not in [STATUS_ID_DEPLOYING]: return True return False @@ -94,7 +96,8 @@ class DeleteService(tables.DeleteAction): def allowed(self, request, service=None): environment_id = self.table.kwargs.get('environment_id') - status, = api.get_environment_data(request, environment_id, 'status') + env = api.environment_get(request, environment_id) + status = getattr(env, 'status', None) return False if status == STATUS_ID_DEPLOYING else True @@ -145,8 +148,10 @@ class DeployThisEnvironment(tables.Action): def allowed(self, request, service): environment_id = self.table.kwargs['environment_id'] - status, version = api.get_environment_data(request, environment_id, - 'status', 'version') + env = api.environment_get(request, environment_id) + status = getattr(env, 'status', None) + version = getattr(env, 'version', None) + if status == STATUS_ID_DEPLOYING: return False services = self.table.data diff --git a/muranodashboard/environments/views.py b/muranodashboard/environments/views.py index 25a1dfe36..a989342d0 100644 --- a/muranodashboard/environments/views.py +++ b/muranodashboard/environments/views.py @@ -201,10 +201,8 @@ class Services(tables.DataTableView): context = super(Services, self).get_context_data(**kwargs) try: - environment_name = api.get_environment_name( - self.request, - self.environment_id) - context['environment_name'] = environment_name + env = api.environment_get(self.request, self.environment_id) + context['environment_name'] = env.name except: msg = _("Sorry, this environment does't exist anymore") @@ -244,8 +242,8 @@ class DetailServiceView(tabs.TabView): context = super(DetailServiceView, self).get_context_data(**kwargs) context["service"] = self.get_data() context["service_name"] = self.service.name - context["environment_name"] = \ - api.get_environment_name(self.request, self.environment_id) + env = api.environment_get(self.request, self.environment_id) + context["environment_name"] = env.name return context def get_data(self): @@ -321,11 +319,8 @@ class DeploymentsView(tables.DataTableView): context = super(DeploymentsView, self).get_context_data(**kwargs) try: - environment_name = api.get_environment_name( - - self.request, - self.environment_id) - context['environment_name'] = environment_name + env = api.environment_get(self.request, self.environment_id) + context['environment_name'] = env.name except: msg = _("Sorry, this environment doesn't exist anymore") redirect = reverse("horizon:murano:environments:index") @@ -359,8 +354,8 @@ class DeploymentDetailsView(tabs.TabbedTableView): def get_context_data(self, **kwargs): context = super(DeploymentDetailsView, self).get_context_data(**kwargs) context["environment_id"] = self.environment_id - context["environment_name"] = \ - api.get_environment_name(self.request, self.environment_id) + env = api.environment_get(self.request, self.environment_id) + context["environment_name"] = env.name context["deployment_start_time"] = \ api.get_deployment_start(self.request, self.environment_id, diff --git a/muranodashboard/local/local_settings.py.example b/muranodashboard/local/local_settings.py.example index d841e6a2b..a1a84742a 100644 --- a/muranodashboard/local/local_settings.py.example +++ b/muranodashboard/local/local_settings.py.example @@ -2,9 +2,18 @@ import os from django.utils.translation import ugettext_lazy as _ +from openstack_dashboard import exceptions + DEBUG = True TEMPLATE_DEBUG = DEBUG +# Required for Django 1.5. +# If horizon is running in production (DEBUG is False), set this +# with the list of host/domain names that the application can serve. +# For more information see: +# https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts +#ALLOWED_HOSTS = ['horizon.example.com', ] + # Set SSL proxy settings: # For Django 1.4+ pass this header from the proxy after terminating the SSL, # and don't forget to strip it from the client's request. @@ -12,15 +21,62 @@ TEMPLATE_DEBUG = DEBUG # https://docs.djangoproject.com/en/1.4/ref/settings/#secure-proxy-ssl-header # SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https') -# Specify a regular expression to validate user passwords. -# HORIZON_CONFIG = { -# "password_validator": { -# "regex": '.*', -# "help_text": _("Your password does not meet the requirements.") -# }, -# 'help_url': "http://docs.openstack.org" +# If Horizon is being served through SSL, then uncomment the following two +# settings to better secure the cookies from security exploits +#CSRF_COOKIE_SECURE = True +#SESSION_COOKIE_SECURE = True + +# Overrides for OpenStack API versions. Use this setting to force the +# OpenStack dashboard to use a specfic API version for a given service API. +# NOTE: The version should be formatted as it appears in the URL for the +# service API. For example, The identity service APIs have inconsistent +# use of the decimal point, so valid options would be "2.0" or "3". +# OPENSTACK_API_VERSIONS = { +# "identity": 3 # } +# Set this to True if running on multi-domain model. When this is enabled, it +# will require user to enter the Domain name in addition to username for login. +# OPENSTACK_KEYSTONE_MULTIDOMAIN_SUPPORT = False + +# Overrides the default domain used when running on single-domain model +# with Keystone V3. All entities will be created in the default domain. +# OPENSTACK_KEYSTONE_DEFAULT_DOMAIN = 'Default' + +# Set Console type: +# valid options would be "AUTO", "VNC" or "SPICE" +# CONSOLE_TYPE = "AUTO" + +# Default OpenStack Dashboard configuration. +HORIZON_CONFIG = { + 'dashboards': ('project', 'admin', 'settings',), + 'default_dashboard': 'project', + 'user_home': 'openstack_dashboard.views.get_user_home', + 'ajax_queue_limit': 10, + 'auto_fade_alerts': { + 'delay': 3000, + 'fade_duration': 1500, + 'types': ['alert-success', 'alert-info'] + }, + 'help_url': "http://docs.openstack.org", + 'exceptions': {'recoverable': exceptions.RECOVERABLE, + 'not_found': exceptions.NOT_FOUND, + 'unauthorized': exceptions.UNAUTHORIZED}, +} + +# Specify a regular expression to validate user passwords. +# HORIZON_CONFIG["password_validator"] = { +# "regex": '.*', +# "help_text": _("Your password does not meet the requirements.") +# } + +# Disable simplified floating IP address management for deployments with +# multiple floating IP pools or complex network requirements. +# HORIZON_CONFIG["simple_ip_management"] = False + +# Turn off browser autocompletion for the login form if so desired. +# HORIZON_CONFIG["password_autocomplete"] = "off" + LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) # Set custom secret key: @@ -32,13 +88,24 @@ LOCAL_PATH = os.path.dirname(os.path.abspath(__file__)) # behind a load-balancer). Either you have to make sure that a session gets all # requests routed to the same dashboard instance or you set the same SECRET_KEY # for all of them. -# from horizon.utils import secret_key -# SECRET_KEY = secret_key.generate_or_read_from_file(os.path.join(LOCAL_PATH, '.secret_key_store')) +from horizon.utils import secret_key +SECRET_KEY = secret_key.generate_or_read_from_file(os.path.join(LOCAL_PATH, '.secret_key_store')) # We recommend you use memcached for development; otherwise after every reload # of the django development server, you will have to login again. To use -# memcached set CACHE_BACKED to something like 'memcached://127.0.0.1:11211/' -CACHE_BACKEND = 'locmem://' +# memcached set CACHES to something like +# CACHES = { +# 'default': { +# 'BACKEND' : 'django.core.cache.backends.memcached.MemcachedCache', +# 'LOCATION' : '127.0.0.1:11211', +# } +#} + +CACHES = { + 'default': { + 'BACKEND' : 'django.core.cache.backends.locmem.LocMemCache' + } +} # Send email to the console by default EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' @@ -64,6 +131,9 @@ OPENSTACK_KEYSTONE_DEFAULT_ROLE = "Member" # Disable SSL certificate checks (useful for self-signed certificates): # OPENSTACK_SSL_NO_VERIFY = True +# The CA certificate to use to verify SSL connections +# OPENSTACK_SSL_CACERT = '/path/to/cacert.pem' + # The OPENSTACK_KEYSTONE_BACKEND settings can be used to identify the # capabilities of the auth backend for Keystone. # If Keystone has been configured to use LDAP as the auth backend then set @@ -72,18 +142,62 @@ OPENSTACK_KEYSTONE_DEFAULT_ROLE = "Member" # TODO(tres): Remove these once Keystone has an API to identify auth backend. OPENSTACK_KEYSTONE_BACKEND = { 'name': 'native', - 'can_edit_user': True + 'can_edit_user': True, + 'can_edit_group': True, + 'can_edit_project': True, + 'can_edit_domain': True, + 'can_edit_role': True } OPENSTACK_HYPERVISOR_FEATURES = { - 'can_set_mount_point': True + 'can_set_mount_point': True, } +# The OPENSTACK_NEUTRON_NETWORK settings can be used to enable optional +# services provided by neutron. Options currenly available are load +# balancer service, security groups, quotas, VPN service. +OPENSTACK_NEUTRON_NETWORK = { + 'enable_lb': False, + 'enable_firewall': False, + 'enable_quotas': True, + 'enable_vpn': False, + # The profile_support option is used to detect if an external router can be + # configured via the dashboard. When using specific plugins the + # profile_support can be turned on if needed. + 'profile_support': None, + #'profile_support': 'cisco', +} + +# The OPENSTACK_IMAGE_BACKEND settings can be used to customize features +# in the OpenStack Dashboard related to the Image service, such as the list +# of supported image formats. +# OPENSTACK_IMAGE_BACKEND = { +# 'image_formats': [ +# ('', ''), +# ('aki', _('AKI - Amazon Kernel Image')), +# ('ami', _('AMI - Amazon Machine Image')), +# ('ari', _('ARI - Amazon Ramdisk Image')), +# ('iso', _('ISO - Optical Disk Image')), +# ('qcow2', _('QCOW2 - QEMU Emulator')), +# ('raw', _('Raw')), +# ('vdi', _('VDI')), +# ('vhd', _('VHD')), +# ('vmdk', _('VMDK')) +# ] +# } + # OPENSTACK_ENDPOINT_TYPE specifies the endpoint type to use for the endpoints # in the Keystone service catalog. Use this setting when Horizon is running -# external to the OpenStack environment. The default is 'internalURL'. +# external to the OpenStack environment. The default is 'publicURL'. #OPENSTACK_ENDPOINT_TYPE = "publicURL" +# SECONDARY_ENDPOINT_TYPE specifies the fallback endpoint type to use in the +# case that OPENSTACK_ENDPOINT_TYPE is not present in the endpoints +# in the Keystone service catalog. Use this setting when Horizon is running +# external to the OpenStack environment. The default is None. This +# value should differ from OPENSTACK_ENDPOINT_TYPE if used. +#SECONDARY_ENDPOINT_TYPE = "publicURL" + # The number of objects (Swift containers/objects or images) to display # on a single page before providing a paging element (a "more" link) # to paginate results. @@ -94,54 +208,243 @@ API_RESULT_PAGE_SIZE = 20 # of your entire OpenStack installation, and hopefully be in UTC. TIME_ZONE = "UTC" +# When launching an instance, the menu of available flavors is +# sorted by RAM usage, ascending. Provide a callback method here +# (and/or a flag for reverse sort) for the sorted() method if you'd +# like a different behaviour. For more info, see +# http://docs.python.org/2/library/functions.html#sorted +# CREATE_INSTANCE_FLAVOR_SORT = { +# 'key': my_awesome_callback_method, +# 'reverse': False, +# } + +# The Horizon Policy Enforcement engine uses these values to load per service +# policy rule files. The content of these files should match the files the +# OpenStack services are using to determine role based access control in the +# target installation. + +# Path to directory containing policy.json files +#POLICY_FILES_PATH = os.path.join(ROOT_PATH, "conf") +# Map of local copy of service policy files +#POLICY_FILES = { +# 'identity': 'keystone_policy.json', +# 'compute': 'nova_policy.json' +#} + +# Trove user and database extension support. By default support for +# creating users and databases on database instances is turned on. +# To disable these extensions set the permission here to something +# unusable such as ["!"]. +# TROVE_ADD_USER_PERMS = [] +# TROVE_ADD_DATABASE_PERMS = [] + LOGGING = { - 'version': 1, - # When set to True this will disable all logging except - # for loggers specified in this configuration dictionary. Note that - # if nothing is specified here and disable_existing_loggers is True, - # django.db.backends will still log unless it is disabled explicitly. - 'disable_existing_loggers': False, - 'handlers': { - 'null': { - 'level': 'DEBUG', - 'class': 'django.utils.log.NullHandler', - }, - 'console': { - # Set the level to "DEBUG" for verbose output logging. - 'level': 'INFO', - 'class': 'logging.StreamHandler', - }, - }, - 'loggers': { - # Logging from django.db.backends is VERY verbose, send to null - # by default. - 'django.db.backends': { - 'handlers': ['null'], - 'propagate': False, - }, - 'horizon': { - 'handlers': ['console'], - 'propagate': False, - }, - 'openstack_dashboard': { - 'handlers': ['console'], - 'propagate': False, - }, - 'novaclient': { - 'handlers': ['console'], - 'propagate': False, - }, - 'keystoneclient': { - 'handlers': ['console'], - 'propagate': False, - }, - 'glanceclient': { - 'handlers': ['console'], - 'propagate': False, - }, - 'nose.plugins.manager': { - 'handlers': ['console'], - 'propagate': False, - } - } + 'version': 1, + # When set to True this will disable all logging except + # for loggers specified in this configuration dictionary. Note that + # if nothing is specified here and disable_existing_loggers is True, + # django.db.backends will still log unless it is disabled explicitly. + 'disable_existing_loggers': False, + 'handlers': { + 'null': { + 'level': 'DEBUG', + 'class': 'django.utils.log.NullHandler', + }, + 'console': { + # Set the level to "DEBUG" for verbose output logging. + 'level': 'INFO', + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + # Logging from django.db.backends is VERY verbose, send to null + # by default. + 'django.db.backends': { + 'handlers': ['null'], + 'propagate': False, + }, + 'requests': { + 'handlers': ['null'], + 'propagate': False, + }, + 'horizon': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'openstack_dashboard': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'novaclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'cinderclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'keystoneclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'glanceclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'neutronclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'heatclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'ceilometerclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'troveclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'swiftclient': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'openstack_auth': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'nose.plugins.manager': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'django': { + 'handlers': ['console'], + 'level': 'DEBUG', + 'propagate': False, + }, + 'iso8601': { + 'handlers': ['null'], + 'propagate': False, + }, + } +} + +SECURITY_GROUP_RULES = { + 'all_tcp': { + 'name': 'ALL TCP', + 'ip_protocol': 'tcp', + 'from_port': '1', + 'to_port': '65535', + }, + 'all_udp': { + 'name': 'ALL UDP', + 'ip_protocol': 'udp', + 'from_port': '1', + 'to_port': '65535', + }, + 'all_icmp': { + 'name': 'ALL ICMP', + 'ip_protocol': 'icmp', + 'from_port': '-1', + 'to_port': '-1', + }, + 'ssh': { + 'name': 'SSH', + 'ip_protocol': 'tcp', + 'from_port': '22', + 'to_port': '22', + }, + 'smtp': { + 'name': 'SMTP', + 'ip_protocol': 'tcp', + 'from_port': '25', + 'to_port': '25', + }, + 'dns': { + 'name': 'DNS', + 'ip_protocol': 'tcp', + 'from_port': '53', + 'to_port': '53', + }, + 'http': { + 'name': 'HTTP', + 'ip_protocol': 'tcp', + 'from_port': '80', + 'to_port': '80', + }, + 'pop3': { + 'name': 'POP3', + 'ip_protocol': 'tcp', + 'from_port': '110', + 'to_port': '110', + }, + 'imap': { + 'name': 'IMAP', + 'ip_protocol': 'tcp', + 'from_port': '143', + 'to_port': '143', + }, + 'ldap': { + 'name': 'LDAP', + 'ip_protocol': 'tcp', + 'from_port': '389', + 'to_port': '389', + }, + 'https': { + 'name': 'HTTPS', + 'ip_protocol': 'tcp', + 'from_port': '443', + 'to_port': '443', + }, + 'smtps': { + 'name': 'SMTPS', + 'ip_protocol': 'tcp', + 'from_port': '465', + 'to_port': '465', + }, + 'imaps': { + 'name': 'IMAPS', + 'ip_protocol': 'tcp', + 'from_port': '993', + 'to_port': '993', + }, + 'pop3s': { + 'name': 'POP3S', + 'ip_protocol': 'tcp', + 'from_port': '995', + 'to_port': '995', + }, + 'ms_sql': { + 'name': 'MS SQL', + 'ip_protocol': 'tcp', + 'from_port': '1433', + 'to_port': '1433', + }, + 'mysql': { + 'name': 'MYSQL', + 'ip_protocol': 'tcp', + 'from_port': '3306', + 'to_port': '3306', + }, + 'rdp': { + 'name': 'RDP', + 'ip_protocol': 'tcp', + 'from_port': '3389', + 'to_port': '3389', + }, } diff --git a/muranodashboard/service_catalog/forms.py b/muranodashboard/service_catalog/forms.py index 550b7a7b3..abcaa8e76 100644 --- a/muranodashboard/service_catalog/forms.py +++ b/muranodashboard/service_catalog/forms.py @@ -36,7 +36,7 @@ class UploadServiceForm(SelfHandlingForm): messages.success(request, _('Service uploaded.')) return result except HTTPException as e: - log.exception(e) + log.exception(_('Uploading service failed')) redirect = reverse('horizon:murano:service_catalog:index') exceptions.handle(request, _('Unable to upload service. ' @@ -60,8 +60,8 @@ class UploadFileToService(SelfHandlingForm): def handle(self, request, data): filename = data['file'].name - log.debug('Uploading file to metadata repository {0} and assiging' - ' it to {1} service'.format(filename, self.service_id)) + log.debug(_('Uploading file to metadata repository {0} and assigning' + ' it to {1} service'.format(filename, self.service_id))) try: result = metadataclient(request).metadata_admin.\ upload_file_to_service(self.data_type, @@ -75,7 +75,7 @@ class UploadFileToService(SelfHandlingForm): except HTTPException as e: redirect = reverse('horizon:murano:service_catalog:manage_service', args=(self.service_id,)) - log.exception(e) + log.exception(_('Uploading file failed')) msg = _("Unable to upload {0} file of '{1}' type." " Error code: {2}".format(filename, self.data_type, @@ -116,7 +116,8 @@ class UploadFileForm(UploadFileToService): return result except HTTPException as e: redirect = reverse('horizon:murano:service_catalog:manage_files') - log.exception(e) + log.exception(_('Uploading file or ' + 'modifying service manifest file failed')) msg = _("Unable to upload {0} file of '{1}' type." " Error code: {2}".format(filename, self.data_type, diff --git a/muranodashboard/service_catalog/tables.py b/muranodashboard/service_catalog/tables.py index ec864e8ac..4a35e3100 100644 --- a/muranodashboard/service_catalog/tables.py +++ b/muranodashboard/service_catalog/tables.py @@ -66,8 +66,8 @@ class DownloadService(tables.Action): response['Content-Disposition'] = 'filename={name}.tar.gz'.format( name=service_id) return response - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Something went wrong during service downloading')) redirect = reverse('horizon:murano:service_catalog:index') exceptions.handle(request, _('Unable to download service.'), @@ -85,8 +85,9 @@ class ToggleEnabled(tables.BatchAction): for obj_id in obj_ids: try: metadataclient(request).metadata_admin.toggle_enabled(obj_id) - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Toggling service state in manifest file in ' + 'metadata repository failed')) exceptions.handle(request, _('Unable to toggle service state.')) else: @@ -94,7 +95,7 @@ class ToggleEnabled(tables.BatchAction): obj.enabled = not obj.enabled messages.success( request, - _("Service '{service} successfully toggled".format( + _("Service '{service}' successfully toggled".format( service=obj_id))) @@ -105,8 +106,9 @@ class DeleteService(tables.DeleteAction): def delete(self, request, obj_id): try: metadataclient(request).metadata_admin.delete_service(obj_id) - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Unable to delete service' + ' in Murano Metadata server')) exceptions.handle(request, _('Unable to remove service.'), redirect='horizon:murano:service_catalog:index') @@ -159,29 +161,6 @@ class ServiceCatalogTable(tables.DataTable): DeleteService) -class ToggleEnabled(tables.BatchAction): - name = 'toggle_enabled' - data_type_singular = _('Active') - data_type_plural = _('Active') - action_present = _('Toggle') - action_past = _('Toggled') - - def handle(self, table, request, obj_ids): - for obj_id in obj_ids: - try: - metadataclient(request).metadata_admin.toggle_enabled(obj_id) - except HTTPException as e: - LOG.exception(e) - exceptions.handle(request, - _('Unable to toggle service state.')) - else: - obj = table.get_object_by_id(obj_id) - obj.enabled = not obj.enabled - messages.success(request, - _("Service '{service} successfully " - "toggled".format(service=obj_id))) - - class DeleteFile(tables.DeleteAction): name = 'delete_file' data_type_singular = _('File') @@ -190,8 +169,8 @@ class DeleteFile(tables.DeleteAction): try: data_type, obj_id = obj_id.split('##') metadataclient(request).metadata_admin.delete(data_type, obj_id) - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Deleting file on Metadata')) redirect = reverse('horizon:murano:service_catalog:manage_files') exceptions.handle( request, @@ -209,8 +188,9 @@ class DeleteFileFromService(tables.DeleteAction): data_type, filename = obj_id.split('##') metadataclient(request).metadata_admin.delete_from_service( data_type, filename, service_id) - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Deleting manifest or file connected ' + 'with it failed in Metadata Service')) redirect = reverse('horizon:murano:service_catalog:manage_service', args=(service_id,)) exceptions.handle( @@ -246,8 +226,9 @@ class DownloadFile(tables.Action): response['Content-Disposition'] = 'filename={name}'.format(name= obj_id) return response - except HTTPException as e: - LOG.exception(e) + except HTTPException: + LOG.exception(_('Error during downloading file ' + 'from Murano Metadata Repository')) redirect = reverse('horizon:murano:service_catalog:manage_files') exceptions.handle( request, diff --git a/muranodashboard/service_catalog/utils.py b/muranodashboard/service_catalog/utils.py index 58c9c820a..44030b628 100644 --- a/muranodashboard/service_catalog/utils.py +++ b/muranodashboard/service_catalog/utils.py @@ -45,6 +45,11 @@ def define_tables(table_name, step_verbose_name): url = None classes = ('ajax-modal', 'btn-create') + def allowed(self, request, service): + if self.table.name == 'ui' and self.table.data: + return False + return True + class ObjectsTable(tables.DataTable): file_name = tables.Column('filename', verbose_name=_('File Name')) path = tables.Column('path', verbose_name=_('Nested Path')) diff --git a/muranodashboard/service_catalog/views.py b/muranodashboard/service_catalog/views.py index 4457e3f62..4aa5e0a2d 100644 --- a/muranodashboard/service_catalog/views.py +++ b/muranodashboard/service_catalog/views.py @@ -104,11 +104,11 @@ class ComposeServiceView(WorkflowView): raise RuntimeError('Not found service id') else: return files_dict - except (HTTPInternalServerError, NotFound) as e: - LOG.exception(e) - msg = _('Error with Murano Metadata Repository') + except (HTTPInternalServerError, NotFound): + err_msg = _('Error with Murano Metadata Repository') + LOG.exception(err_msg) redirect = reverse_lazy('horizon:murano:service_catalog:index') - exceptions.handle(self.request, msg, redirect) + exceptions.handle(self.request, err_msg, redirect) class UploadFileView2(ModalFormView): diff --git a/muranodashboard/settings.py b/muranodashboard/settings.py index 2c46d617f..51e12b818 100644 --- a/muranodashboard/settings.py +++ b/muranodashboard/settings.py @@ -1,5 +1,6 @@ import logging import os +import tempfile import sys from openstack_dashboard import exceptions @@ -13,10 +14,13 @@ if ROOT_PATH not in sys.path: DEBUG = False TEMPLATE_DEBUG = DEBUG +METADATA_CACHE_DIR = os.path.join(tempfile.gettempdir(), + 'muranodashboard-cache') + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.abspath(os.path.join(ROOT_PATH, 'dashboard.sqlite')) + 'NAME': os.path.join(METADATA_CACHE_DIR, 'openstack-dashboard.sqlite') } } @@ -126,7 +130,7 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' AUTHENTICATION_BACKENDS = ('openstack_auth.backend.KeystoneBackend',) MESSAGE_STORAGE = 'django.contrib.messages.storage.cookie.CookieStorage' -SESSION_ENGINE = 'django.contrib.sessions.backends.signed_cookies' +SESSION_ENGINE = 'django.contrib.sessions.backends.db' SESSION_COOKIE_HTTPONLY = True SESSION_EXPIRE_AT_BROWSER_CLOSE = True SESSION_COOKIE_SECURE = False diff --git a/setup.sh b/setup.sh index 046f04ae3..371ddcd5b 100755 --- a/setup.sh +++ b/setup.sh @@ -51,7 +51,7 @@ function install_prerequisites() if [ $? -eq 1 ]; then retval=1 return $retval - fi + fi find /var/lib/apt/lists/ -name "*cloud.archive*" | grep -q "havana_main" if [ $? -ne 0 ]; then add-apt-repository -y cloud-archive:havana >> $LOGFILE 2>&1 @@ -98,11 +98,11 @@ function make_tarball() chmod +x $setuppy rm -rf $RUN_DIR/*.egg-info cd $RUN_DIR && python $setuppy egg_info > /dev/null 2>&1 - if [ $? -ne 0 ];then + if [ $? -ne 0 ];then log "...\"$setuppy\" egg info creation fails, exiting!!!" retval=1 exit 1 - fi + fi rm -rf $RUN_DIR/dist/* log "...\"setup.py sdist\" output will be recorded in \"$LOGFILE\"" cd $RUN_DIR && $setuppy sdist >> $LOGFILE 2>&1 @@ -110,7 +110,7 @@ function make_tarball() log "...\"$setuppy\" tarball creation fails, exiting!!!" retval=1 exit 1 - fi + fi #TRBL_FILE=$(basename $(ls $RUN_DIR/dist/*.tar.gz | head -n 1)) TRBL_FILE=$(ls $RUN_DIR/dist/*.tar.gz | head -n 1) if [ ! -e "$TRBL_FILE" ]; then @@ -137,33 +137,33 @@ function run_pip_install() log "...pip install fails, exiting!" retval=1 exit 1 - fi + fi return $retval } modify_horizon_config() { INFILE=$1 - REMOVE=$2 + REMOVE=$2 PATTERN='from openstack_dashboard import policy' TMPFILE="./tmpfile" retval=0 if [ -f $INFILE ]; then lines=$(sed -ne '/^#START_MURANO_DASHBOARD/,/^#END_MURANO_DASHBOARD/ =' $INFILE) - if [ -n "$lines" ]; then - if [ ! -z $REMOVE ]; then + if [ -n "$lines" ]; then + if [ ! -z $REMOVE ]; then log "Removing $APPLICATION_NAME data from \"$INFILE\"..." sed -e '/^#START_MURANO_DASHBOARD/,/^#END_MURANO_DASHBOARD/ d' -i $INFILE - if [ $? -ne 0 ];then + if [ $? -ne 0 ];then log "...can't modify \"$INFILE\", check permissions or something else, exiting!!!" retval=1 return $retval else log "...success" - fi - else + fi + else log "\"$INFILE\" already has $APPLICATION_NAME data, you can change it manually and restart apache2/httpd service" - fi - else + fi + else if [ -z "$REMOVE" ]; then log "Adding $APPLICATION_NAME data to \"$INFILE\"..." rm -f $TMPFILE @@ -197,38 +197,38 @@ NETWORK_TOPOLOGY = 'routed' #END_MURANO_DASHBOARD EOF sed -ne "/$PATTERN/r $TMPFILE" -e 1x -e '2,${x;p}' -e '${x;p}' -i $INFILE - if [ $? -ne 0 ];then + if [ $? -ne 0 ];then log "Can't modify \"$INFILE\", check permissions or something else, exiting!!!" else rm -f $TMPFILE log "...success" - fi - fi - fi - else - echo "File \"$1\" not found, exiting!!!" + fi + fi + fi + else + echo "File \"$1\" not found, exiting!!!" retval=1 - fi + fi return $retval } function find_horizon_config() { retval=0 for cfg_file in $(echo $HORIZON_CONFIGS | sed 's/,/ /') - do + do if [ -e "$cfg_file" ]; then - log "Horizon config found at \"$cfg_file\"" - modify_horizon_config $cfg_file $1 + log "Horizon config found at \"$cfg_file\"" + modify_horizon_config $cfg_file $1 retval=0 break else retval=1 - fi - done + fi + done if [ $retval -eq 1 ]; then - log "Horizon config not found or openstack-dashboard does not installed, to override this set proper \"HORIZON_CONFIGS\" variable, exiting!!!" + log "Horizon config not found or openstack-dashboard does not installed, to override this set proper \"HORIZON_CONFIGS\" variable, exiting!!!" #exit 1 - fi + fi return $retval } @@ -244,7 +244,7 @@ function prepare_db() retval=1 else log "..success" - fi + fi return $retval } function rebuildstatic() @@ -264,15 +264,15 @@ function rebuildstatic() log "...openstack-dashboard manage.py not found, exiting!!!" retval=1 return $retval - fi + fi _old_murano_static="$(dirname $horizon_manage)/openstack_dashboard/static/muranodashboard" if [ -d "$_old_murano_static" ];then log "...$APPLICATION_NAME static for \"muranodashboard\" found under \"HORIZON\" STATIC, deleting \"$_old_murano_static\"..." rm -rf $_old_murano_static if [ $? -ne 0 ]; then log "...can't delete \"$_old_murano_static\, WARNING!!!" - fi - fi + fi + fi log "Rebuilding STATIC output will be recorded in \"$LOGFILE\"" #python $horizon_manage collectstatic --noinput >> $LOGFILE 2>&1 chmod a+rw $LOGFILE @@ -280,9 +280,9 @@ function rebuildstatic() if [ $? -ne 0 ]; then log "...\"$horizon_manage\" collectstatic failed, exiting!!!" retval=1 - else + else log "...success" - fi + fi prepare_db "$horizon_manage" || retval=$? return $retval } @@ -291,15 +291,15 @@ function run_pip_uninstall() find_pip retval=0 pack_to_del=$(is_py_package_installed "$APPLICATION_NAME") - if [ $? -eq 0 ]; then + if [ $? -eq 0 ]; then log "Running \"$PIPCMD uninstall $PIPARGS $APPLICATION_NAME\" output will be recorded in \"$LOGFILE\"" $PIPCMD uninstall $pack_to_del --yes >> $LOGFILE 2>&1 if [ $? -ne 0 ]; then log "...can't uninstall $APPLICATION_NAME with $PIPCMD" retval=1 - else + else log "...success" - fi + fi else log "Python package for \"$APPLICATION_NAME\" not found" fi @@ -337,13 +337,13 @@ function install_application() mk_dir "$APPLICATION_LOG_DIR" "$WEB_SERVICE_USER" "$WEB_SERVICE_GROUP" || exit $? mk_dir "$APPLICATION_CACHE_DIR" "$WEB_SERVICE_USER" "$WEB_SERVICE_GROUP" || exit $? horizon_etc_cfg=$(find /etc/openstack-dashboard -name "local_setting*" | head -n 1) - if [ $? -ne 0 ]; then + if [ $? -ne 0 ]; then log "Can't find horizon config under \"/etc/openstack-dashboard...\"" retval=1 else iniset '' 'ALLOWED_HOSTS' "'*'" $horizon_etc_cfg iniset '' 'DEBUG' 'True' $horizon_etc_cfg - fi + fi return $retval } function uninstall_application() @@ -355,7 +355,7 @@ function uninstall_application() function postinst() { rebuildstatic || exit $? - sleep 2 + sleep 2 chown -R $WEB_SERVICE_USER:$WEB_SERVICE_GROUP /var/lib/openstack-dashboard service $WEB_SERVICE_SYSNAME restart } @@ -381,17 +381,17 @@ case $COMMAND in install_application || exit $? postinst || exit $? log "...success" - ;; + ;; - uninstall ) + uninstall ) log "Uninstalling \"$APPLICATION_NAME\" from system..." uninstall_application || exit $? postuninst log "Software uninstalled, application logs located at \"$APPLICATION_LOG_DIR\", cache files - at \"$APPLICATION_CACHE_DIR'\" ." - ;; + ;; - * ) + * ) echo -e "Usage: $(basename "$0") [command] \nCommands:\n\tinstall - Install \"$APPLICATION_NAME\" software\n\tuninstall - Uninstall \"$APPLICATION_NAME\" software" - exit 1 - ;; + exit 1 + ;; esac \ No newline at end of file