From 6a89dd2feb46d85889e90a0c83f2fbebcf67e9de Mon Sep 17 00:00:00 2001 From: Boden R Date: Tue, 1 May 2018 15:29:16 -0600 Subject: [PATCH] use db utils from lib The APIs our consumers are using from neutron.db_utils were rehomed into neutron-lib with https://review.openstack.org/#/c/540161/ This patch consumes them by removing the rehomed APIs and using lib's implementation where applicable. NeutronLibImpact Change-Id: I7ee53bce917feae8e37bf278eb3121a5af47131c --- neutron/db/_model_query.py | 7 +-- neutron/db/_utils.py | 60 +------------------ neutron/db/address_scope_db.py | 2 +- neutron/db/agents_db.py | 2 +- neutron/db/allowedaddresspairs_db.py | 2 +- neutron/db/common_db_mixin.py | 9 +-- neutron/db/db_base_plugin_common.py | 2 +- neutron/db/db_base_plugin_v2.py | 2 +- neutron/db/external_net_db.py | 2 +- neutron/db/flavors_db.py | 2 +- neutron/db/ipam_backend_mixin.py | 2 +- neutron/db/l3_db.py | 9 +-- neutron/db/metering/metering_db.py | 2 +- neutron/db/portsecurity_db_common.py | 2 +- neutron/db/rbac_db_mixin.py | 2 +- neutron/db/securitygroups_db.py | 2 +- neutron/plugins/ml2/plugin.py | 2 +- neutron/services/auto_allocate/db.py | 2 +- .../network_ip_availability/plugin.py | 2 +- neutron/services/provider_configuration.py | 2 +- neutron/services/segments/db.py | 2 +- .../unit/extensions/test_portsecurity.py | 2 +- 22 files changed, 33 insertions(+), 88 deletions(-) diff --git a/neutron/db/_model_query.py b/neutron/db/_model_query.py index e621ed871e6..b71288dadbf 100644 --- a/neutron/db/_model_query.py +++ b/neutron/db/_model_query.py @@ -24,7 +24,6 @@ from oslo_db.sqlalchemy import utils as sa_utils from sqlalchemy import sql, or_, and_ from sqlalchemy.ext import associationproxy -from neutron.db import _utils as ndb_utils # Classes implementing extensions will register hooks into this dictionary # for "augmenting" the "core way" of building a query for retrieving objects @@ -137,7 +136,7 @@ def query_with_hooks(context, model): query = context.session.query(model) # define basic filter condition for model query query_filter = None - if ndb_utils.model_query_scope_is_project(context, model): + if db_utils.model_query_scope_is_project(context, model): if hasattr(model, 'rbac_entries'): query = query.outerjoin(model.rbac_entries) rbac_model = model.rbac_entries.property.mapper.class_ @@ -237,8 +236,8 @@ def apply_filters(query, model, filters, context=None): query.session.query(rbac.object_id).filter(is_shared) ) elif (not context or - not ndb_utils.model_query_scope_is_project(context, - model)): + not db_utils.model_query_scope_is_project( + context, model)): # we only want to join if we aren't using the subquery # and if we aren't already joined because this is a # scoped query diff --git a/neutron/db/_utils.py b/neutron/db/_utils.py index b50480e43be..1a5d449cc3a 100644 --- a/neutron/db/_utils.py +++ b/neutron/db/_utils.py @@ -17,10 +17,9 @@ NOTE: This module shall not be used by external projects. It will be moved import contextlib -from neutron_lib.api import attributes +from neutron_lib.db import utils as db_utils from oslo_log import log as logging from oslo_utils import excutils -from sqlalchemy.ext import associationproxy LOG = logging.getLogger(__name__) @@ -72,68 +71,13 @@ def safe_creation(context, create_fn, delete_fn, create_bindings, return obj, value -def model_query_scope_is_project(context, model): - # Unless a context has 'admin' or 'advanced-service' rights the - # query will be scoped to a single project_id - return ((not context.is_admin and hasattr(model, 'project_id')) and - (not context.is_advsvc and hasattr(model, 'project_id'))) - - def model_query(context, model): query = context.session.query(model) # define basic filter condition for model query query_filter = None - if model_query_scope_is_project(context, model): + if db_utils.model_query_scope_is_project(context, model): query_filter = (model.tenant_id == context.tenant_id) if query_filter is not None: query = query.filter(query_filter) return query - - -# NOTE: This used to be CommonDbMixin._fields() -def resource_fields(resource, fields): - """Return only the resource items that are in fields. - - :param resource: a resource dictionary - :type resource: dict - :param fields: a list of fields to select from the resource - :type fields: list - - """ - if fields: - resource = {key: item for key, item in resource.items() - if key in fields} - return attributes.populate_project_info(resource) - - -# NOTE: This used to be CommonDbMixin._filter_non_model_columns -def filter_non_model_columns(data, model): - """Return the attributes from data which are model columns. - - Return a new dict with items from data that whose keys are columns in - the model or are association proxies of the model. - """ - columns = [c.name for c in model.__table__.columns] - return dict((k, v) for (k, v) in - data.items() if k in columns or - isinstance(getattr(model, k, None), - associationproxy.AssociationProxy)) - - -# NOTE: This used to be CommonDbMixin._get_marker_obj -def get_marker_obj(plugin, context, resource, limit, marker): - """Retrieve a resource marker object. - - This function is used to invoke: - plugin._get_(context, marker) - It is used for pagination. - - :param plugin: The plugin processing the request. - :param context: The request context. - :param resource: The resource name. - :param limit: Indicates if pagination is in effect. - :param marker: The id of the marker object. - """ - if limit and marker: - return getattr(plugin, '_get_%s' % resource)(context, marker) diff --git a/neutron/db/address_scope_db.py b/neutron/db/address_scope_db.py index 541e0e75ff6..9a25f35db21 100644 --- a/neutron/db/address_scope_db.py +++ b/neutron/db/address_scope_db.py @@ -15,12 +15,12 @@ from neutron_lib.api.definitions import address_scope as apidef from neutron_lib.api.definitions import network as net_def from neutron_lib import constants +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import address_scope as api_err from oslo_utils import uuidutils from neutron._i18n import _ from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.extensions import address_scope as ext_address_scope from neutron.objects import address_scope as obj_addr_scope diff --git a/neutron/db/agents_db.py b/neutron/db/agents_db.py index 3a079a4efbc..a1d3e61b949 100644 --- a/neutron/db/agents_db.py +++ b/neutron/db/agents_db.py @@ -24,6 +24,7 @@ from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants from neutron_lib import context +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import agent as agent_exc from neutron_lib.exceptions import availability_zone as az_exc from neutron_lib.plugins import directory @@ -39,7 +40,6 @@ from neutron.api.rpc.callbacks import version_manager from neutron.common import constants as n_const from neutron.conf.agent.database import agents_db from neutron.db import _model_query as model_query -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db.models import agent as agent_model from neutron.extensions import agent as ext_agent diff --git a/neutron/db/allowedaddresspairs_db.py b/neutron/db/allowedaddresspairs_db.py index 6b184904470..374907d3fe8 100644 --- a/neutron/db/allowedaddresspairs_db.py +++ b/neutron/db/allowedaddresspairs_db.py @@ -16,12 +16,12 @@ from neutron_lib.api.definitions import allowedaddresspairs as addr_apidef from neutron_lib.api.definitions import port as port_def from neutron_lib.api import validators +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import allowedaddresspairs as addr_exc from neutron_lib.objects import exceptions from neutron.common import utils from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.objects.port.extensions import (allowedaddresspairs as obj_addr_pair) diff --git a/neutron/db/common_db_mixin.py b/neutron/db/common_db_mixin.py index c1c652e9ccd..3934b911e36 100644 --- a/neutron/db/common_db_mixin.py +++ b/neutron/db/common_db_mixin.py @@ -13,9 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib.db import utils as db_utils + from neutron.db import _model_query from neutron.db import _resource_extend -from neutron.db import _utils as ndb_utils # TODO(HenryG): Deprecate and schedule for removal @@ -39,7 +40,7 @@ class CommonDbMixin(object): @staticmethod def _fields(resource, fields): - return ndb_utils.resource_fields(resource, fields) + return db_utils.resource_fields(resource, fields) @staticmethod def _get_by_id(context, model, id): @@ -79,8 +80,8 @@ class CommonDbMixin(object): # TODO(HenryG): Remove this when available in neutron-lib def _get_marker_obj(self, context, resource, limit, marker): - return ndb_utils.get_marker_obj(self, context, resource, limit, marker) + return db_utils.get_marker_obj(self, context, resource, limit, marker) @staticmethod def _filter_non_model_columns(data, model): - return ndb_utils.filter_non_model_columns(data, model) + return db_utils.filter_non_model_columns(data, model) diff --git a/neutron/db/db_base_plugin_common.py b/neutron/db/db_base_plugin_common.py index b34f3e20c98..ea41afc6ce3 100644 --- a/neutron/db/db_base_plugin_common.py +++ b/neutron/db/db_base_plugin_common.py @@ -21,6 +21,7 @@ from neutron_lib.api.definitions import subnet as subnet_def from neutron_lib.api.definitions import subnetpool as subnetpool_def from neutron_lib.api import validators from neutron_lib import constants +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from neutron_lib.utils import net from oslo_config import cfg @@ -31,7 +32,6 @@ from neutron.common import constants as n_const from neutron.common import exceptions from neutron.db import _model_query as model_query from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import models_v2 diff --git a/neutron/db/db_base_plugin_v2.py b/neutron/db/db_base_plugin_v2.py index 34573a77030..c74c978f9eb 100644 --- a/neutron/db/db_base_plugin_v2.py +++ b/neutron/db/db_base_plugin_v2.py @@ -27,6 +27,7 @@ from neutron_lib.callbacks import resources from neutron_lib import constants from neutron_lib import context as ctx from neutron_lib.db import api as lib_db_api +from neutron_lib.db import utils as ndb_utils from neutron_lib import exceptions as exc from neutron_lib.exceptions import l3 as l3_exc from neutron_lib.plugins import constants as plugin_constants @@ -47,7 +48,6 @@ from neutron.common import ipv6_utils from neutron.common import utils from neutron.db import _model_query as model_query from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as ndb_utils from neutron.db import api as db_api from neutron.db import db_base_plugin_common from neutron.db import ipam_pluggable_backend diff --git a/neutron/db/external_net_db.py b/neutron/db/external_net_db.py index d55b81b6104..fe986435ad1 100644 --- a/neutron/db/external_net_db.py +++ b/neutron/db/external_net_db.py @@ -20,6 +20,7 @@ from neutron_lib.callbacks import events from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from neutron_lib.exceptions import external_net as extnet_exc from neutron_lib.plugins import constants as plugin_constants @@ -29,7 +30,6 @@ from sqlalchemy.sql import expression as expr from neutron._i18n import _ from neutron.db import _model_query as model_query from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import models_v2 from neutron.db import rbac_db_models as rbac_db from neutron.extensions import rbac as rbac_ext diff --git a/neutron/db/flavors_db.py b/neutron/db/flavors_db.py index 544f1dab1e2..6bd1aac6137 100644 --- a/neutron/db/flavors_db.py +++ b/neutron/db/flavors_db.py @@ -12,11 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import flavors as flav_exc from oslo_db import exception as db_exc from oslo_log import log as logging -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import servicetype_db as sdb diff --git a/neutron/db/ipam_backend_mixin.py b/neutron/db/ipam_backend_mixin.py index d4c835b806e..b7ccc074c2a 100644 --- a/neutron/db/ipam_backend_mixin.py +++ b/neutron/db/ipam_backend_mixin.py @@ -22,6 +22,7 @@ from neutron_lib.api.definitions import ip_allocation as ipalloc_apidef from neutron_lib.api.definitions import portbindings from neutron_lib.api import validators from neutron_lib import constants as const +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as exc from oslo_config import cfg from oslo_log import log as logging @@ -33,7 +34,6 @@ from neutron.common import exceptions as n_exc from neutron.common import ipv6_utils from neutron.common import utils as common_utils from neutron.db import _model_query as model_query -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import db_base_plugin_common from neutron.db.models import segment as segment_model diff --git a/neutron/db/l3_db.py b/neutron/db/l3_db.py index d363f712602..66227468aa9 100644 --- a/neutron/db/l3_db.py +++ b/neutron/db/l3_db.py @@ -26,6 +26,7 @@ from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants from neutron_lib import context as n_ctx +from neutron_lib.db import utils as lib_db_utils from neutron_lib import exceptions as n_exc from neutron_lib.exceptions import l3 as l3_exc from neutron_lib.plugins import constants as plugin_constants @@ -209,7 +210,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase, # plugins. if process_extensions: resource_extend.apply_funcs(l3_apidef.ROUTERS, res, router) - return db_utils.resource_fields(res, fields) + return lib_db_utils.resource_fields(res, fields) def _create_router_db(self, context, router, tenant_id): """Create the DB object.""" @@ -591,8 +592,8 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase, def get_routers(self, context, filters=None, fields=None, sorts=None, limit=None, marker=None, page_reverse=False): - marker_obj = db_utils.get_marker_obj(self, context, 'router', - limit, marker) + marker_obj = lib_db_utils.get_marker_obj( + self, context, 'router', limit, marker) return model_query.get_collection(context, l3_models.Router, self._make_router_dict, filters=filters, fields=fields, @@ -1072,7 +1073,7 @@ class L3_NAT_dbonly_mixin(l3.RouterPluginBase, if process_extensions: resource_extend.apply_funcs( l3_apidef.FLOATINGIPS, res, floatingip.db_obj) - return db_utils.resource_fields(res, fields) + return lib_db_utils.resource_fields(res, fields) def _get_router_for_floatingip(self, context, internal_port, internal_subnet_id, diff --git a/neutron/db/metering/metering_db.py b/neutron/db/metering/metering_db.py index 85c7988a6dd..bc5d550db3f 100644 --- a/neutron/db/metering/metering_db.py +++ b/neutron/db/metering/metering_db.py @@ -13,13 +13,13 @@ # under the License. import netaddr +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import metering as metering_exc from oslo_db import exception as db_exc from oslo_utils import uuidutils from neutron.api.rpc.agentnotifiers import metering_rpc_agent_api from neutron.common import constants -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin as base_db from neutron.db import l3_dvr_db diff --git a/neutron/db/portsecurity_db_common.py b/neutron/db/portsecurity_db_common.py index e5717f6b93a..a44f27dc55a 100644 --- a/neutron/db/portsecurity_db_common.py +++ b/neutron/db/portsecurity_db_common.py @@ -13,8 +13,8 @@ # under the License. from neutron_lib.api.definitions import port_security as psec +from neutron_lib.db import utils as db_utils -from neutron.db import _utils as db_utils from neutron.objects import network from neutron.objects.port.extensions import port_security as p_ps diff --git a/neutron/db/rbac_db_mixin.py b/neutron/db/rbac_db_mixin.py index 6d3a670f6a3..92ccacf75c8 100644 --- a/neutron/db/rbac_db_mixin.py +++ b/neutron/db/rbac_db_mixin.py @@ -17,12 +17,12 @@ from neutron_lib.callbacks import events from neutron_lib.callbacks import exceptions as c_exc from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from oslo_db import exception as db_exc from sqlalchemy.orm import exc from neutron.db import _model_query as model_query -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import rbac_db_models as models diff --git a/neutron/db/securitygroups_db.py b/neutron/db/securitygroups_db.py index d83a7d249b7..2833d94acee 100644 --- a/neutron/db/securitygroups_db.py +++ b/neutron/db/securitygroups_db.py @@ -20,6 +20,7 @@ from neutron_lib.callbacks import exceptions from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from neutron_lib.utils import helpers from neutron_lib.utils import net @@ -32,7 +33,6 @@ from neutron.common import constants as n_const from neutron.common import utils from neutron.db import _model_query as model_query from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db.models import securitygroup as sg_models from neutron.extensions import securitygroup as ext_sg diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py index 9de2780c35f..7ef9cf35f52 100644 --- a/neutron/plugins/ml2/plugin.py +++ b/neutron/plugins/ml2/plugin.py @@ -33,6 +33,7 @@ from neutron_lib.callbacks import exceptions from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants as const +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as exc from neutron_lib.exceptions import allowedaddresspairs as addr_exc from neutron_lib.exceptions import port_security as psec_exc @@ -65,7 +66,6 @@ from neutron.common import rpc as n_rpc from neutron.common import utils from neutron.db import _model_query as model_query from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import address_scope_db from neutron.db import agents_db from neutron.db import agentschedulers_db diff --git a/neutron/services/auto_allocate/db.py b/neutron/services/auto_allocate/db.py index 4ce1b463325..c63585eaacd 100644 --- a/neutron/services/auto_allocate/db.py +++ b/neutron/services/auto_allocate/db.py @@ -20,6 +20,7 @@ from neutron_lib.api.definitions import network as net_def from neutron_lib.callbacks import events from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from neutron_lib.objects import exceptions as obj_exc from neutron_lib.plugins import constants @@ -29,7 +30,6 @@ from oslo_log import log as logging from neutron._i18n import _ from neutron.common import exceptions as c_exc from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.objects import auto_allocate as auto_allocate_obj diff --git a/neutron/services/network_ip_availability/plugin.py b/neutron/services/network_ip_availability/plugin.py index b3ccb5258c3..5391763a90c 100644 --- a/neutron/services/network_ip_availability/plugin.py +++ b/neutron/services/network_ip_availability/plugin.py @@ -13,9 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions -from neutron.db import _utils as db_utils import neutron.db.db_base_plugin_v2 as db_base_plugin_v2 import neutron.db.network_ip_availability_db as ip_availability_db diff --git a/neutron/services/provider_configuration.py b/neutron/services/provider_configuration.py index 97b42006eca..416f720ec23 100644 --- a/neutron/services/provider_configuration.py +++ b/neutron/services/provider_configuration.py @@ -18,6 +18,7 @@ import itertools import os from neutron_lib.db import constants as db_const +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from oslo_config import cfg from oslo_log import log as logging @@ -26,7 +27,6 @@ import stevedore from neutron._i18n import _ from neutron.conf.services import provider_configuration as prov_config -from neutron.db import _utils as db_utils LOG = logging.getLogger(__name__) diff --git a/neutron/services/segments/db.py b/neutron/services/segments/db.py index 2eed58da287..9d203643396 100644 --- a/neutron/services/segments/db.py +++ b/neutron/services/segments/db.py @@ -18,6 +18,7 @@ from neutron_lib.callbacks import events from neutron_lib.callbacks import registry from neutron_lib.callbacks import resources from neutron_lib import constants +from neutron_lib.db import utils as db_utils from neutron_lib import exceptions as n_exc from neutron_lib.plugins import directory from oslo_concurrency import lockutils @@ -26,7 +27,6 @@ from oslo_log import helpers as log_helpers from oslo_utils import uuidutils from neutron.db import _resource_extend as resource_extend -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import common_db_mixin from neutron.db import segments_db as db diff --git a/neutron/tests/unit/extensions/test_portsecurity.py b/neutron/tests/unit/extensions/test_portsecurity.py index 15c24c43df9..80a3fafc5f6 100644 --- a/neutron/tests/unit/extensions/test_portsecurity.py +++ b/neutron/tests/unit/extensions/test_portsecurity.py @@ -18,11 +18,11 @@ import copy from neutron_lib.api.definitions import port_security as psec from neutron_lib.api import validators from neutron_lib import context +from neutron_lib.db import utils as db_utils from neutron_lib.exceptions import port_security as psec_exc from neutron_lib.plugins import directory from webob import exc -from neutron.db import _utils as db_utils from neutron.db import api as db_api from neutron.db import db_base_plugin_v2 from neutron.db import portsecurity_db