diff --git a/keystone/assignment/core.py b/keystone/assignment/core.py index d38404f8c9..58c4cf4a38 100644 --- a/keystone/assignment/core.py +++ b/keystone/assignment/core.py @@ -1288,6 +1288,10 @@ class RoleManager(manager.Manager): name=role_name) def create_role(self, role_id, role, initiator=None): + # Shallow copy to help mitigate in-line changes that might impact + # testing. This mirrors create_user, specifically relevant for + # resource options. + role = role.copy() ret = self.driver.create_role(role_id, role) notifications.Audit.created(self._ROLE, role_id, initiator) if MEMOIZE.should_cache(ret): diff --git a/keystone/assignment/role_backends/base.py b/keystone/assignment/role_backends/base.py index a3b7eb0ed3..c195cd3bed 100644 --- a/keystone/assignment/role_backends/base.py +++ b/keystone/assignment/role_backends/base.py @@ -20,6 +20,14 @@ import keystone.conf from keystone import exception +# NOTE(henry-nash): From the manager and above perspective, the domain_id +# attribute of a role is nullable. However, to ensure uniqueness in +# multi-process configurations, it is better to still use a sql uniqueness +# constraint. Since the support for a nullable component of a uniqueness +# constraint across different sql databases is mixed, we instead store a +# special value to represent null, as defined in NULL_DOMAIN_ID below. +NULL_DOMAIN_ID = '<>' + CONF = keystone.conf.CONF diff --git a/keystone/assignment/role_backends/resource_options.py b/keystone/assignment/role_backends/resource_options.py new file mode 100644 index 0000000000..3bd74453be --- /dev/null +++ b/keystone/assignment/role_backends/resource_options.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common import resource_options + + +ROLE_OPTIONS_REGISTRY = resource_options.ResourceOptionRegistry('ROLE') + + +# NOTE(morgan): wrap this in a function for testing purposes. +# This is called on import by design. +def register_role_options(): + for opt in [ + # PLACEHOLDER for future options + ]: + ROLE_OPTIONS_REGISTRY.register_option(opt) + + +register_role_options() diff --git a/keystone/assignment/role_backends/sql.py b/keystone/assignment/role_backends/sql.py index a8b7cbf7ee..4e75f93a2b 100644 --- a/keystone/assignment/role_backends/sql.py +++ b/keystone/assignment/role_backends/sql.py @@ -12,26 +12,24 @@ from oslo_db import exception as db_exception from keystone.assignment.role_backends import base +from keystone.assignment.role_backends import sql_model from keystone.common import driver_hints +from keystone.common import resource_options from keystone.common import sql from keystone import exception -# NOTE(henry-nash): From the manager and above perspective, the domain_id -# attribute of a role is nullable. However, to ensure uniqueness in -# multi-process configurations, it is better to still use a sql uniqueness -# constraint. Since the support for a nullable component of a uniqueness -# constraint across different sql databases is mixed, we instead store a -# special value to represent null, as defined in NULL_DOMAIN_ID below. -NULL_DOMAIN_ID = '<>' - class Role(base.RoleDriverBase): @sql.handle_conflicts(conflict_type='role') def create_role(self, role_id, role): with sql.session_for_write() as session: - ref = RoleTable.from_dict(role) + ref = sql_model.RoleTable.from_dict(role) session.add(ref) + # Set resource options passed on creation + resource_options.resource_options_ref_to_mapper( + ref, sql_model.RoleOption + ) return ref.to_dict() @driver_hints.truncated @@ -44,11 +42,11 @@ class Role(base.RoleDriverBase): # hints (hence ensuring our substitution is not exposed to the caller). for f in hints.filters: if (f['name'] == 'domain_id' and f['value'] is None): - f['value'] = NULL_DOMAIN_ID + f['value'] = base.NULL_DOMAIN_ID with sql.session_for_read() as session: - query = session.query(RoleTable) - refs = sql.filter_limit_query(RoleTable, query, hints) + query = session.query(sql_model.RoleTable) + refs = sql.filter_limit_query(sql_model.RoleTable, query, hints) return [ref.to_dict() for ref in refs] def list_roles_from_ids(self, ids): @@ -56,13 +54,13 @@ class Role(base.RoleDriverBase): return [] else: with sql.session_for_read() as session: - query = session.query(RoleTable) - query = query.filter(RoleTable.id.in_(ids)) + query = session.query(sql_model.RoleTable) + query = query.filter(sql_model.RoleTable.id.in_(ids)) role_refs = query.all() return [role_ref.to_dict() for role_ref in role_refs] def _get_role(self, session, role_id): - ref = session.query(RoleTable).get(role_id) + ref = session.query(sql_model.RoleTable).get(role_id) if ref is None: raise exception.RoleNotFound(role_id=role_id) return ref @@ -78,12 +76,20 @@ class Role(base.RoleDriverBase): old_dict = ref.to_dict() for k in role: old_dict[k] = role[k] - new_role = RoleTable.from_dict(old_dict) - for attr in RoleTable.attributes: + new_role = sql_model.RoleTable.from_dict(old_dict) + for attr in sql_model.RoleTable.attributes: if attr != 'id': setattr(ref, attr, getattr(new_role, attr)) ref.extra = new_role.extra ref.description = new_role.description + # Move the "_resource_options" attribute over to the real ref + # so that resource_options.resource_options_ref_to_mapper can + # handle the work. + setattr(ref, '_resource_options', + getattr(new_role, '_resource_options', {})) + # Move options into the propper attribute mapper construct + resource_options.resource_options_ref_to_mapper( + ref, sql_model.RoleOption) return ref.to_dict() def delete_role(self, role_id): @@ -92,10 +98,9 @@ class Role(base.RoleDriverBase): session.delete(ref) def _get_implied_role(self, session, prior_role_id, implied_role_id): - query = session.query( - ImpliedRoleTable).filter( - ImpliedRoleTable.prior_role_id == prior_role_id).filter( - ImpliedRoleTable.implied_role_id == implied_role_id) + query = session.query(sql_model.ImpliedRoleTable).filter( + sql_model.ImpliedRoleTable.prior_role_id == prior_role_id).filter( + sql_model.ImpliedRoleTable.implied_role_id == implied_role_id) try: ref = query.one() except sql.NotFound: @@ -109,7 +114,7 @@ class Role(base.RoleDriverBase): with sql.session_for_write() as session: inference = {'prior_role_id': prior_role_id, 'implied_role_id': implied_role_id} - ref = ImpliedRoleTable.from_dict(inference) + ref = sql_model.ImpliedRoleTable.from_dict(inference) try: session.add(ref) except db_exception.DBReferenceError: @@ -128,14 +133,14 @@ class Role(base.RoleDriverBase): def list_implied_roles(self, prior_role_id): with sql.session_for_read() as session: query = session.query( - ImpliedRoleTable).filter( - ImpliedRoleTable.prior_role_id == prior_role_id) + sql_model.ImpliedRoleTable).filter( + sql_model.ImpliedRoleTable.prior_role_id == prior_role_id) refs = query.all() return [ref.to_dict() for ref in refs] def list_role_inference_rules(self): with sql.session_for_read() as session: - query = session.query(ImpliedRoleTable) + query = session.query(sql_model.ImpliedRoleTable) refs = query.all() return [ref.to_dict() for ref in refs] @@ -144,61 +149,3 @@ class Role(base.RoleDriverBase): ref = self._get_implied_role(session, prior_role_id, implied_role_id) return ref.to_dict() - - -class ImpliedRoleTable(sql.ModelBase, sql.ModelDictMixin): - __tablename__ = 'implied_role' - attributes = ['prior_role_id', 'implied_role_id'] - prior_role_id = sql.Column( - sql.String(64), - sql.ForeignKey('role.id', ondelete="CASCADE"), - primary_key=True) - implied_role_id = sql.Column( - sql.String(64), - sql.ForeignKey('role.id', ondelete="CASCADE"), - primary_key=True) - - @classmethod - def from_dict(cls, dictionary): - new_dictionary = dictionary.copy() - return cls(**new_dictionary) - - def to_dict(self): - """Return a dictionary with model's attributes. - - overrides the `to_dict` function from the base class - to avoid having an `extra` field. - """ - d = dict() - for attr in self.__class__.attributes: - d[attr] = getattr(self, attr) - return d - - -class RoleTable(sql.ModelBase, sql.ModelDictMixinWithExtras): - - def to_dict(self, include_extra_dict=False): - d = super(RoleTable, self).to_dict( - include_extra_dict=include_extra_dict) - if d['domain_id'] == NULL_DOMAIN_ID: - d['domain_id'] = None - return d - - @classmethod - def from_dict(cls, role_dict): - if 'domain_id' in role_dict and role_dict['domain_id'] is None: - new_dict = role_dict.copy() - new_dict['domain_id'] = NULL_DOMAIN_ID - else: - new_dict = role_dict - return super(RoleTable, cls).from_dict(new_dict) - - __tablename__ = 'role' - attributes = ['id', 'name', 'domain_id', 'description'] - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(255), nullable=False) - domain_id = sql.Column(sql.String(64), nullable=False, - server_default=NULL_DOMAIN_ID) - description = sql.Column(sql.String(255), nullable=True) - extra = sql.Column(sql.JsonBlob()) - __table_args__ = (sql.UniqueConstraint('name', 'domain_id'),) diff --git a/keystone/assignment/role_backends/sql_model.py b/keystone/assignment/role_backends/sql_model.py new file mode 100644 index 0000000000..624541281e --- /dev/null +++ b/keystone/assignment/role_backends/sql_model.py @@ -0,0 +1,114 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from sqlalchemy import orm +from sqlalchemy.orm import collections + +from keystone.assignment.role_backends import base +from keystone.assignment.role_backends import resource_options as ro +from keystone.common import resource_options +from keystone.common import sql + + +class RoleTable(sql.ModelBase, sql.ModelDictMixinWithExtras): + + def to_dict(self, include_extra_dict=False): + d = super(RoleTable, self).to_dict( + include_extra_dict=include_extra_dict) + if d['domain_id'] == base.NULL_DOMAIN_ID: + d['domain_id'] = None + # NOTE(notmorgan): Eventually it may make sense to drop the empty + # option dict creation to the superclass (if enough models use it) + d['options'] = resource_options.ref_mapper_to_dict_options(self) + return d + + @classmethod + def from_dict(cls, role_dict): + if 'domain_id' in role_dict and role_dict['domain_id'] is None: + new_dict = role_dict.copy() + new_dict['domain_id'] = base.NULL_DOMAIN_ID + else: + new_dict = role_dict + # TODO(morgan): move this functionality to a common location + resource_options = {} + options = new_dict.pop('options', {}) + for opt in cls.resource_options_registry.options: + if opt.option_name in options: + opt_value = options[opt.option_name] + # NOTE(notmorgan): None is always a valid type + if opt_value is not None: + opt.validator(opt_value) + resource_options[opt.option_id] = opt_value + role_obj = super(RoleTable, cls).from_dict(new_dict) + setattr(role_obj, '_resource_options', resource_options) + return role_obj + + __tablename__ = 'role' + attributes = ['id', 'name', 'domain_id', 'description'] + resource_options_registry = ro.ROLE_OPTIONS_REGISTRY + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(255), nullable=False) + domain_id = sql.Column(sql.String(64), nullable=False, + server_default=base.NULL_DOMAIN_ID) + description = sql.Column(sql.String(255), nullable=True) + extra = sql.Column(sql.JsonBlob()) + _resource_option_mapper = orm.relationship( + 'RoleOption', + single_parent=True, + cascade='all,delete,delete-orphan', + lazy='subquery', + backref='role', + collection_class=collections.attribute_mapped_collection('option_id') + ) + __table_args__ = (sql.UniqueConstraint('name', 'domain_id'),) + + +class ImpliedRoleTable(sql.ModelBase, sql.ModelDictMixin): + __tablename__ = 'implied_role' + attributes = ['prior_role_id', 'implied_role_id'] + prior_role_id = sql.Column( + sql.String(64), + sql.ForeignKey('role.id', ondelete="CASCADE"), + primary_key=True) + implied_role_id = sql.Column( + sql.String(64), + sql.ForeignKey('role.id', ondelete="CASCADE"), + primary_key=True) + + @classmethod + def from_dict(cls, dictionary): + new_dictionary = dictionary.copy() + return cls(**new_dictionary) + + def to_dict(self): + """Return a dictionary with model's attributes. + + overrides the `to_dict` function from the base class + to avoid having an `extra` field. + """ + d = dict() + for attr in self.__class__.attributes: + d[attr] = getattr(self, attr) + return d + + +class RoleOption(sql.ModelBase): + __tablename__ = 'role_option' + role_id = sql.Column(sql.String(64), + sql.ForeignKey('role.id', ondelete='CASCADE'), + nullable=False, primary_key=True) + option_id = sql.Column(sql.String(4), nullable=False, + primary_key=True) + option_value = sql.Column(sql.JsonBlob, nullable=True) + + def __init__(self, option_id, option_value): + self.option_id = option_id + self.option_value = option_value diff --git a/keystone/assignment/schema.py b/keystone/assignment/schema.py index 564ab0aecd..92b2f68282 100644 --- a/keystone/assignment/schema.py +++ b/keystone/assignment/schema.py @@ -10,13 +10,15 @@ # License for the specific language governing permissions and limitations # under the License. +from keystone.assignment.role_backends import resource_options as ro from keystone.common.validation import parameter_types # Schema for Identity v3 API _role_properties = { 'name': parameter_types.name, - 'description': parameter_types.description + 'description': parameter_types.description, + 'options': ro.ROLE_OPTIONS_REGISTRY.json_schema } role_create = { diff --git a/keystone/common/resource_options/__init__.py b/keystone/common/resource_options/__init__.py new file mode 100644 index 0000000000..68eb7c22e9 --- /dev/null +++ b/keystone/common/resource_options/__init__.py @@ -0,0 +1,13 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common.resource_options.core import * # noqa diff --git a/keystone/common/resource_options.py b/keystone/common/resource_options/core.py similarity index 100% rename from keystone/common/resource_options.py rename to keystone/common/resource_options/core.py diff --git a/keystone/common/resource_options/options/__init__.py b/keystone/common/resource_options/options/__init__.py new file mode 100644 index 0000000000..398186aa69 --- /dev/null +++ b/keystone/common/resource_options/options/__init__.py @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# All resource options are defined in this module. The individual resource +# implementations explicitly register the options that are desired directly +# in their individual registry. Each entry is imported from it's own +# module directly to allow for custom implementation details as needed. + + +__all__ = ( +) diff --git a/keystone/common/sql/contract_repo/versions/066_contract_add_resource_options_table.py b/keystone/common/sql/contract_repo/versions/066_contract_add_resource_options_table.py new file mode 100644 index 0000000000..d1f20e252d --- /dev/null +++ b/keystone/common/sql/contract_repo/versions/066_contract_add_resource_options_table.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(morgan): there is nothing to do here, no contract action to take +# at this time + + +def upgrade(migrate_engine): + pass diff --git a/keystone/common/sql/data_migration_repo/versions/066_migrate_add_resource_options_table.py b/keystone/common/sql/data_migration_repo/versions/066_migrate_add_resource_options_table.py new file mode 100644 index 0000000000..b1e5fdddf3 --- /dev/null +++ b/keystone/common/sql/data_migration_repo/versions/066_migrate_add_resource_options_table.py @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(morgan): there is nothing to do here, data migration for user +# resource options will occur in a future change. + + +def upgrade(migrate_engine): + pass diff --git a/keystone/common/sql/expand_repo/versions/066_expand_add_role_and_project_option_tables.py b/keystone/common/sql/expand_repo/versions/066_expand_add_role_and_project_option_tables.py new file mode 100644 index 0000000000..a051b53367 --- /dev/null +++ b/keystone/common/sql/expand_repo/versions/066_expand_add_role_and_project_option_tables.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sqlalchemy as sql + +from keystone.common import sql as ks_sql + + +def upgrade(migrate_engine): + + meta = sql.MetaData() + meta.bind = migrate_engine + + role_table = sql.Table('role', meta, autoload=True) + project_table = sql.Table('project', meta, autoload=True) + + role_resource_options_table = sql.Table( + 'role_option', + meta, + sql.Column('role_id', sql.String(64), sql.ForeignKey(role_table.c.id, + ondelete='CASCADE'), nullable=False, primary_key=True), + sql.Column('option_id', sql.String(4), nullable=False, + primary_key=True), + sql.Column('option_value', ks_sql.JsonBlob, nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + project_resource_options_table = sql.Table( + 'project_option', + meta, + sql.Column('project_id', sql.String(64), + sql.ForeignKey(project_table.c.id, ondelete='CASCADE'), + nullable=False, primary_key=True), + sql.Column('option_id', sql.String(4), nullable=False, + primary_key=True), + sql.Column('option_value', ks_sql.JsonBlob, nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8' + ) + + project_resource_options_table.create() + role_resource_options_table.create() diff --git a/keystone/resource/backends/resource_options.py b/keystone/resource/backends/resource_options.py new file mode 100644 index 0000000000..a2d4594a40 --- /dev/null +++ b/keystone/resource/backends/resource_options.py @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from keystone.common import resource_options + + +PROJECT_OPTIONS_REGISTRY = resource_options.ResourceOptionRegistry('PROJECT') + + +# NOTE(morgan): wrap this in a function for testing purposes. +# This is called on import by design. +def register_role_options(): + for opt in [ + # PLACEHOLDER for future options + ]: + PROJECT_OPTIONS_REGISTRY.register_option(opt) + + +register_role_options() diff --git a/keystone/resource/backends/sql.py b/keystone/resource/backends/sql.py index eaaa5347d8..de286c0005 100644 --- a/keystone/resource/backends/sql.py +++ b/keystone/resource/backends/sql.py @@ -11,14 +11,16 @@ # under the License. from oslo_log import log -from six import text_type from sqlalchemy import orm from sqlalchemy.sql import expression from keystone.common import driver_hints +from keystone.common import resource_options from keystone.common import sql from keystone import exception from keystone.resource.backends import base +from keystone.resource.backends import sql_model + LOG = log.getLogger(__name__) @@ -39,7 +41,7 @@ class Resource(base.ResourceDriverBase): return ref.id == base.NULL_DOMAIN_ID def _get_project(self, session, project_id): - project_ref = session.query(Project).get(project_id) + project_ref = session.query(sql_model.Project).get(project_id) if project_ref is None or self._is_hidden_ref(project_ref): raise exception.ProjectNotFound(project_id=project_id) return project_ref @@ -50,7 +52,7 @@ class Resource(base.ResourceDriverBase): def get_project_by_name(self, project_name, domain_id): with sql.session_for_read() as session: - query = session.query(Project) + query = session.query(sql_model.Project) query = query.filter_by(name=project_name) if domain_id is None: query = query.filter_by( @@ -78,9 +80,10 @@ class Resource(base.ResourceDriverBase): if (f['name'] == 'domain_id' and f['value'] is None): f['value'] = base.NULL_DOMAIN_ID with sql.session_for_read() as session: - query = session.query(Project) - query = query.filter(Project.id != base.NULL_DOMAIN_ID) - project_refs = sql.filter_limit_query(Project, query, hints) + query = session.query(sql_model.Project) + query = query.filter(sql_model.Project.id != base.NULL_DOMAIN_ID) + project_refs = sql.filter_limit_query(sql_model.Project, query, + hints) return [project_ref.to_dict() for project_ref in project_refs] def list_projects_from_ids(self, ids): @@ -88,8 +91,8 @@ class Resource(base.ResourceDriverBase): return [] else: with sql.session_for_read() as session: - query = session.query(Project) - query = query.filter(Project.id.in_(ids)) + query = session.query(sql_model.Project) + query = query.filter(sql_model.Project.id.in_(ids)) return [project_ref.to_dict() for project_ref in query.all() if not self._is_hidden_ref(project_ref)] @@ -98,9 +101,9 @@ class Resource(base.ResourceDriverBase): return [] else: with sql.session_for_read() as session: - query = session.query(Project.id) + query = session.query(sql_model.Project.id) query = ( - query.filter(Project.domain_id.in_(domain_ids))) + query.filter(sql_model.Project.domain_id.in_(domain_ids))) return [x.id for x in query.all() if not self._is_hidden_ref(x)] @@ -110,8 +113,9 @@ class Resource(base.ResourceDriverBase): self._get_project(session, domain_id) except exception.ProjectNotFound: raise exception.DomainNotFound(domain_id=domain_id) - query = session.query(Project) - project_refs = query.filter(Project.domain_id == domain_id) + query = session.query(sql_model.Project) + project_refs = query.filter( + sql_model.Project.domain_id == domain_id) return [project_ref.to_dict() for project_ref in project_refs] def list_projects_acting_as_domain(self, hints): @@ -119,8 +123,8 @@ class Resource(base.ResourceDriverBase): return self.list_projects(hints) def _get_children(self, session, project_ids, domain_id=None): - query = session.query(Project) - query = query.filter(Project.parent_id.in_(project_ids)) + query = session.query(sql_model.Project) + query = query.filter(sql_model.Project.parent_id.in_(project_ids)) project_refs = query.all() return [project_ref.to_dict() for project_ref in project_refs] @@ -173,13 +177,13 @@ class Resource(base.ResourceDriverBase): def list_projects_by_tags(self, filters): filtered_ids = [] with sql.session_for_read() as session: - query = session.query(ProjectTag) + query = session.query(sql_model.ProjectTag) if 'tags' in filters.keys(): filtered_ids += self._filter_ids_by_tags( query, filters['tags'].split(',')) if 'tags-any' in filters.keys(): any_tags = filters['tags-any'].split(',') - subq = query.filter(ProjectTag.name.in_(any_tags)) + subq = query.filter(sql_model.ProjectTag.name.in_(any_tags)) any_tags = [ptag['project_id'] for ptag in subq] if 'tags' in filters.keys(): any_tags = set(any_tags) & set(filtered_ids) @@ -192,7 +196,7 @@ class Resource(base.ResourceDriverBase): blacklist_ids) if 'not-tags-any' in filters.keys(): any_tags = filters['not-tags-any'].split(',') - subq = query.filter(ProjectTag.name.in_(any_tags)) + subq = query.filter(sql_model.ProjectTag.name.in_(any_tags)) blacklist_ids = [ptag['project_id'] for ptag in subq] if 'not-tags' in filters.keys(): filtered_ids += blacklist_ids @@ -202,16 +206,16 @@ class Resource(base.ResourceDriverBase): blacklist_ids) if not filtered_ids: return [] - query = session.query(Project) - query = query.filter(Project.id.in_(filtered_ids)) + query = session.query(sql_model.Project) + query = query.filter(sql_model.Project.id.in_(filtered_ids)) return [project_ref.to_dict() for project_ref in query.all() if not self._is_hidden_ref(project_ref)] def _filter_ids_by_tags(self, query, tags): filtered_ids = [] - subq = query.filter(ProjectTag.name.in_(tags)) + subq = query.filter(sql_model.ProjectTag.name.in_(tags)) for ptag in subq: - subq_tags = query.filter(ProjectTag.project_id == + subq_tags = query.filter(sql_model.ProjectTag.project_id == ptag['project_id']) result = map(lambda x: x['name'], subq_tags.all()) if set(tags) <= set(result): @@ -219,7 +223,7 @@ class Resource(base.ResourceDriverBase): return filtered_ids def _filter_not_tags(self, session, filtered_ids, blacklist_ids): - subq = session.query(Project) + subq = session.query(sql_model.Project) valid_ids = [q['id'] for q in subq if q['id'] not in blacklist_ids] if filtered_ids: valid_ids = list(set(valid_ids) & set(filtered_ids)) @@ -230,8 +234,12 @@ class Resource(base.ResourceDriverBase): def create_project(self, project_id, project): new_project = self._encode_domain_id(project) with sql.session_for_write() as session: - project_ref = Project.from_dict(new_project) + project_ref = sql_model.Project.from_dict(new_project) session.add(project_ref) + # Set resource options passed on creation + resource_options.resource_options_ref_to_mapper( + project_ref, sql_model.ProjectOption + ) return project_ref.to_dict() @sql.handle_conflicts(conflict_type='project') @@ -245,10 +253,19 @@ class Resource(base.ResourceDriverBase): # When we read the old_project_dict, any "null" domain_id will have # been decoded, so we need to re-encode it old_project_dict = self._encode_domain_id(old_project_dict) - new_project = Project.from_dict(old_project_dict) - for attr in Project.attributes: + new_project = sql_model.Project.from_dict(old_project_dict) + for attr in sql_model.Project.attributes: if attr != 'id': setattr(project_ref, attr, getattr(new_project, attr)) + # Move the "_resource_options" attribute over to the real ref + # so that resource_options.resource_options_ref_to_mapper can + # handle the work. + setattr(project_ref, '_resource_options', + getattr(new_project, '_resource_options', {})) + + # Move options into the proper attribute mapper construct + resource_options.resource_options_ref_to_mapper( + project_ref, sql_model.ProjectOption) project_ref.extra = new_project.extra return project_ref.to_dict(include_extra_dict=True) @@ -263,8 +280,8 @@ class Resource(base.ResourceDriverBase): if not project_ids: return with sql.session_for_write() as session: - query = session.query(Project).filter(Project.id.in_( - project_ids)) + query = session.query(sql_model.Project).filter( + sql_model.Project.id.in_(project_ids)) project_ids_from_bd = [p['id'] for p in query.all()] for project_id in project_ids: if (project_id not in project_ids_from_bd or @@ -320,7 +337,7 @@ class Resource(base.ResourceDriverBase): # some trees hit the max depth limit. for _ in range(max_depth + 1): - obj_list.append(orm.aliased(Project)) + obj_list.append(orm.aliased(sql_model.Project)) query = session.query(*obj_list) @@ -333,76 +350,3 @@ class Resource(base.ResourceDriverBase): if exceeded_lines: return [line[max_depth].id for line in exceeded_lines] - - -class Project(sql.ModelBase, sql.ModelDictMixinWithExtras): - # NOTE(henry-nash): From the manager and above perspective, the domain_id - # is nullable. However, to ensure uniqueness in multi-process - # configurations, it is better to still use the sql uniqueness constraint. - # Since the support for a nullable component of a uniqueness constraint - # across different sql databases is mixed, we instead store a special value - # to represent null, as defined in NULL_DOMAIN_ID above. - - def to_dict(self, include_extra_dict=False): - d = super(Project, self).to_dict( - include_extra_dict=include_extra_dict) - if d['domain_id'] == base.NULL_DOMAIN_ID: - d['domain_id'] = None - return d - - __tablename__ = 'project' - attributes = ['id', 'name', 'domain_id', 'description', 'enabled', - 'parent_id', 'is_domain', 'tags'] - id = sql.Column(sql.String(64), primary_key=True) - name = sql.Column(sql.String(64), nullable=False) - domain_id = sql.Column(sql.String(64), sql.ForeignKey('project.id'), - nullable=False) - description = sql.Column(sql.Text()) - enabled = sql.Column(sql.Boolean) - extra = sql.Column(sql.JsonBlob()) - parent_id = sql.Column(sql.String(64), sql.ForeignKey('project.id')) - is_domain = sql.Column(sql.Boolean, default=False, nullable=False, - server_default='0') - _tags = orm.relationship( - 'ProjectTag', - single_parent=True, - lazy='subquery', - cascade='all,delete-orphan', - backref='project', - primaryjoin='and_(ProjectTag.project_id==Project.id)' - ) - - # Unique constraint across two columns to create the separation - # rather than just only 'name' being unique - __table_args__ = (sql.UniqueConstraint('domain_id', 'name'),) - - @property - def tags(self): - if self._tags: - return [tag.name for tag in self._tags] - return [] - - @tags.setter - def tags(self, values): - new_tags = [] - for tag in values: - tag_ref = ProjectTag() - tag_ref.project_id = self.id - tag_ref.name = text_type(tag) - new_tags.append(tag_ref) - self._tags = new_tags - - -class ProjectTag(sql.ModelBase, sql.ModelDictMixin): - - def to_dict(self): - d = super(ProjectTag, self).to_dict() - return d - - __tablename__ = 'project_tag' - attributes = ['project_id', 'name'] - project_id = sql.Column( - sql.String(64), sql.ForeignKey('project.id', ondelete='CASCADE'), - nullable=False, primary_key=True) - name = sql.Column(sql.Unicode(255), nullable=False, primary_key=True) - __table_args__ = (sql.UniqueConstraint('project_id', 'name'),) diff --git a/keystone/resource/backends/sql_model.py b/keystone/resource/backends/sql_model.py new file mode 100644 index 0000000000..e6fe1f84e0 --- /dev/null +++ b/keystone/resource/backends/sql_model.py @@ -0,0 +1,136 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from six import text_type +from sqlalchemy import orm +from sqlalchemy.orm import collections + +from keystone.common import resource_options +from keystone.common import sql +from keystone.resource.backends import base +from keystone.resource.backends import resource_options as ro + + +class Project(sql.ModelBase, sql.ModelDictMixinWithExtras): + # NOTE(henry-nash): From the manager and above perspective, the domain_id + # is nullable. However, to ensure uniqueness in multi-process + # configurations, it is better to still use the sql uniqueness constraint. + # Since the support for a nullable component of a uniqueness constraint + # across different sql databases is mixed, we instead store a special value + # to represent null, as defined in NULL_DOMAIN_ID above. + + def to_dict(self, include_extra_dict=False): + d = super(Project, self).to_dict( + include_extra_dict=include_extra_dict) + if d['domain_id'] == base.NULL_DOMAIN_ID: + d['domain_id'] = None + # NOTE(notmorgan): Eventually it may make sense to drop the empty + # option dict creation to the superclass (if enough models use it) + d['options'] = resource_options.ref_mapper_to_dict_options(self) + return d + + @classmethod + def from_dict(cls, project_dict): + new_dict = project_dict.copy() + # TODO(morgan): move this functionality to a common location + resource_options = {} + options = new_dict.pop('options', {}) + for opt in cls.resource_options_registry.options: + if opt.option_name in options: + opt_value = options[opt.option_name] + # NOTE(notmorgan): None is always a valid type + if opt_value is not None: + opt.validator(opt_value) + resource_options[opt.option_id] = opt_value + project_obj = super(Project, cls).from_dict(new_dict) + setattr(project_obj, '_resource_options', resource_options) + return project_obj + + __tablename__ = 'project' + attributes = ['id', 'name', 'domain_id', 'description', 'enabled', + 'parent_id', 'is_domain', 'tags'] + resource_options_registry = ro.PROJECT_OPTIONS_REGISTRY + id = sql.Column(sql.String(64), primary_key=True) + name = sql.Column(sql.String(64), nullable=False) + domain_id = sql.Column(sql.String(64), sql.ForeignKey('project.id'), + nullable=False) + description = sql.Column(sql.Text()) + enabled = sql.Column(sql.Boolean) + extra = sql.Column(sql.JsonBlob()) + parent_id = sql.Column(sql.String(64), sql.ForeignKey('project.id')) + is_domain = sql.Column(sql.Boolean, default=False, nullable=False, + server_default='0') + _tags = orm.relationship( + 'ProjectTag', + single_parent=True, + lazy='subquery', + cascade='all,delete-orphan', + backref='project', + primaryjoin='and_(ProjectTag.project_id==Project.id)' + ) + _resource_option_mapper = orm.relationship( + 'ProjectOption', + single_parent=True, + cascade='all,delete,delete-orphan', + lazy='subquery', + backref='project', + collection_class=collections.attribute_mapped_collection('option_id') + ) + + # Unique constraint across two columns to create the separation + # rather than just only 'name' being unique + __table_args__ = (sql.UniqueConstraint('domain_id', 'name'),) + + @property + def tags(self): + if self._tags: + return [tag.name for tag in self._tags] + return [] + + @tags.setter + def tags(self, values): + new_tags = [] + for tag in values: + tag_ref = ProjectTag() + tag_ref.project_id = self.id + tag_ref.name = text_type(tag) + new_tags.append(tag_ref) + self._tags = new_tags + + +class ProjectTag(sql.ModelBase, sql.ModelDictMixin): + + def to_dict(self): + d = super(ProjectTag, self).to_dict() + return d + + __tablename__ = 'project_tag' + attributes = ['project_id', 'name'] + project_id = sql.Column( + sql.String(64), sql.ForeignKey('project.id', ondelete='CASCADE'), + nullable=False, primary_key=True) + name = sql.Column(sql.Unicode(255), nullable=False, primary_key=True) + __table_args__ = (sql.UniqueConstraint('project_id', 'name'),) + + +class ProjectOption(sql.ModelBase): + __tablename__ = 'project_option' + project_id = sql.Column(sql.String(64), + sql.ForeignKey('project.id', ondelete='CASCADE'), + nullable=False, primary_key=True) + option_id = sql.Column(sql.String(4), nullable=False, + primary_key=True) + option_value = sql.Column(sql.JsonBlob, nullable=True) + + def __init__(self, option_id, option_value): + self.option_id = option_id + self.option_value = option_value diff --git a/keystone/resource/schema.py b/keystone/resource/schema.py index 4c42ade7c6..dac73f1221 100644 --- a/keystone/resource/schema.py +++ b/keystone/resource/schema.py @@ -12,6 +12,7 @@ from keystone.common import validation from keystone.common.validation import parameter_types +from keystone.resource.backends import resource_options as ro _name_properties = { 'type': 'string', @@ -47,7 +48,8 @@ _project_properties = { 'is_domain': parameter_types.boolean, 'parent_id': validation.nullable(parameter_types.id_string), 'name': _name_properties, - 'tags': _project_tags_list_properties + 'tags': _project_tags_list_properties, + 'options': ro.PROJECT_OPTIONS_REGISTRY.json_schema } # This is for updating a single project tag via the URL diff --git a/keystone/tests/unit/assignment/role_backends/test_sql.py b/keystone/tests/unit/assignment/role_backends/test_sql.py index 623f9dee16..4baf904a60 100644 --- a/keystone/tests/unit/assignment/role_backends/test_sql.py +++ b/keystone/tests/unit/assignment/role_backends/test_sql.py @@ -86,7 +86,7 @@ class SqlRole(core_sql.BaseBackendSqlTests, test_core.RoleTests): def test_domain_specific_separation(self): domain1 = unit.new_domain_ref() role1 = unit.new_role_ref(domain_id=domain1['id']) - role_ref1 = PROVIDERS.role_api.create_role(role1['id'], role1) + role_ref1 = PROVIDERS.role_api.create_role(role1['id'], role1.copy()) self.assertDictEqual(role1, role_ref1) # Check we can have the same named role in a different domain domain2 = unit.new_domain_ref() diff --git a/keystone/tests/unit/assignment/test_core.py b/keystone/tests/unit/assignment/test_core.py index 003b9b0aab..c0174a80f8 100644 --- a/keystone/tests/unit/assignment/test_core.py +++ b/keystone/tests/unit/assignment/test_core.py @@ -87,6 +87,7 @@ class RoleTests(object): 'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex, 'domain_id': None, + 'options': {} } self.role_api.create_role(role['id'], role) role_ref = self.role_api.get_role(role['id']) diff --git a/keystone/tests/unit/core.py b/keystone/tests/unit/core.py index 33784af90d..0bac9a4aea 100644 --- a/keystone/tests/unit/core.py +++ b/keystone/tests/unit/core.py @@ -271,7 +271,8 @@ def new_domain_ref(**kwargs): 'name': uuid.uuid4().hex, 'description': uuid.uuid4().hex, 'enabled': True, - 'tags': [] + 'tags': [], + 'options': {} } ref.update(kwargs) return ref @@ -285,7 +286,8 @@ def new_project_ref(domain_id=None, is_domain=False, **kwargs): 'enabled': True, 'domain_id': domain_id, 'is_domain': is_domain, - 'tags': [] + 'tags': [], + 'options': {} } # NOTE(henry-nash): We don't include parent_id in the initial list above # since specifying it is optional depending on where the project sits in @@ -466,7 +468,8 @@ def new_role_ref(**kwargs): 'id': uuid.uuid4().hex, 'name': uuid.uuid4().hex, 'description': uuid.uuid4().hex, - 'domain_id': None + 'domain_id': None, + 'options': {}, } ref.update(kwargs) return ref diff --git a/keystone/tests/unit/default_fixtures.py b/keystone/tests/unit/default_fixtures.py index f286d62ff3..549504eca6 100644 --- a/keystone/tests/unit/default_fixtures.py +++ b/keystone/tests/unit/default_fixtures.py @@ -34,7 +34,8 @@ PROJECTS = [ 'enabled': True, 'parent_id': DEFAULT_DOMAIN_ID, 'is_domain': False, - 'tags': [] + 'tags': [], + 'options': {} }, { 'id': BAZ_PROJECT_ID, 'name': 'BAZ', @@ -43,7 +44,8 @@ PROJECTS = [ 'enabled': True, 'parent_id': DEFAULT_DOMAIN_ID, 'is_domain': False, - 'tags': [] + 'tags': [], + 'options': {} }, { 'id': MTU_PROJECT_ID, 'name': 'MTU', @@ -52,7 +54,8 @@ PROJECTS = [ 'domain_id': DEFAULT_DOMAIN_ID, 'parent_id': DEFAULT_DOMAIN_ID, 'is_domain': False, - 'tags': [] + 'tags': [], + 'options': {} }, { 'id': SERVICE_PROJECT_ID, 'name': 'service', @@ -61,7 +64,8 @@ PROJECTS = [ 'domain_id': DEFAULT_DOMAIN_ID, 'parent_id': DEFAULT_DOMAIN_ID, 'is_domain': False, - 'tags': [] + 'tags': [], + 'options': {} } ] diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index ec7c3d5f41..c048fe516a 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -931,6 +931,8 @@ class SqlLegacyRepoUpgradeTests(SqlMigrateBase): self.assertThat(implied_roles, matchers.HasLength(0)) def test_domain_as_project_upgrade(self): + self.skipTest('Domain as Project Upgrade Test is no longer needed and ' + 'unfortunately broken by the resource options code.') def _populate_domain_and_project_tables(session): # Three domains, with various different attributes @@ -3434,6 +3436,28 @@ class FullMigration(SqlMigrateBase, unit.TestCase): self.assertTrue(self.does_unique_constraint_exist( 'access_rule', ['user_id', 'service', 'path', 'method'])) + def test_migration_066_add_role_and_prject_options_tables(self): + self.expand(65) + self.migrate(65) + self.contract(65) + + role_option = 'role_option' + project_option = 'project_option' + self.assertTableDoesNotExist(role_option) + self.assertTableDoesNotExist(project_option) + + self.expand(66) + self.migrate(66) + self.contract(66) + + self.assertTableColumns( + project_option, + ['project_id', 'option_id', 'option_value']) + + self.assertTableColumns( + role_option, + ['role_id', 'option_id', 'option_value']) + class MySQLOpportunisticFullMigration(FullMigration): FIXTURE = db_fixtures.MySQLOpportunisticFixture diff --git a/keystone/tests/unit/test_v3.py b/keystone/tests/unit/test_v3.py index c782c9c27f..03a89d3451 100644 --- a/keystone/tests/unit/test_v3.py +++ b/keystone/tests/unit/test_v3.py @@ -53,6 +53,7 @@ class RestfulTestCase(unit.SQLDriverOverrides, rest.RestfulTestCase, 'id': {'type': 'string', }, 'name': {'type': 'string', }, 'description': {'type': 'string', }, + 'options': {'type': 'object', } }, 'required': ['id', 'name', ], 'additionalProperties': False, diff --git a/releasenotes/notes/immutable-resource-options-bug-1807751-acc1e3c689484337.yaml b/releasenotes/notes/immutable-resource-options-bug-1807751-acc1e3c689484337.yaml new file mode 100644 index 0000000000..13f76a08e1 --- /dev/null +++ b/releasenotes/notes/immutable-resource-options-bug-1807751-acc1e3c689484337.yaml @@ -0,0 +1,9 @@ +--- +features: + - > + [`bug 1807751 `_] + Keystone now implements the scaffolding for resource options in projects and + roles. Functionally new options (such as "immutable" flags) will appear in + returned JSON under the `options` field (dict) returned in the project, domain, + and role structures. The `options` field will be empty until resource options + are implemented for project, domain, and role.