From 589ac355f3ddf505983925d80b81c20107ceace0 Mon Sep 17 00:00:00 2001 From: Matthew Oliver Date: Thu, 3 Feb 2022 16:18:39 +1100 Subject: [PATCH] Move *_swift_info functions into a new registry module The *_swift_info functions use in module global dicts to provide a registry mechanism for registering and getting swift info. This is an abnormal pattern and doesn't quite fit into utils. Further we looking at following this pattern for sensitive info to trim in the future. So this patch does some house cleaning and moves this registry to a new module swift.common.registry. And updates all the references to it. For backwards compat we still import the *_swift_info methods into utils for any 3rd party tools or middleware. Change-Id: I71fd7f50d1aafc001d6905438f42de4e58af8421 --- swift/common/middleware/account_quotas.py | 2 +- swift/common/middleware/bulk.py | 4 +- swift/common/middleware/cname_lookup.py | 3 +- swift/common/middleware/container_quotas.py | 2 +- swift/common/middleware/container_sync.py | 3 +- swift/common/middleware/crossdomain.py | 2 +- swift/common/middleware/crypto/__init__.py | 3 +- swift/common/middleware/domain_remap.py | 4 +- swift/common/middleware/etag_quoter.py | 3 +- swift/common/middleware/formpost.py | 7 +- swift/common/middleware/name_check.py | 3 +- swift/common/middleware/ratelimit.py | 3 +- swift/common/middleware/read_only.py | 4 +- .../middleware/s3api/controllers/bucket.py | 4 +- .../s3api/controllers/multi_delete.py | 3 +- .../middleware/s3api/controllers/obj.py | 3 +- .../s3api/controllers/versioning.py | 3 +- swift/common/middleware/s3api/s3api.py | 6 +- swift/common/middleware/s3api/s3request.py | 4 +- swift/common/middleware/slo.py | 7 +- swift/common/middleware/staticweb.py | 3 +- swift/common/middleware/symlink.py | 3 +- swift/common/middleware/tempauth.py | 3 +- swift/common/middleware/tempurl.py | 6 +- .../middleware/versioned_writes/__init__.py | 4 +- swift/common/registry.py | 86 +++++++ swift/common/utils.py | 75 +----- swift/proxy/controllers/info.py | 4 +- swift/proxy/server.py | 3 +- .../common/middleware/crypto/test_crypto.py | 34 +-- .../common/middleware/s3api/test_s3api.py | 10 +- test/unit/common/middleware/test_bulk.py | 8 +- .../common/middleware/test_cname_lookup.py | 10 +- .../common/middleware/test_domain_remap.py | 10 +- .../unit/common/middleware/test_name_check.py | 10 +- test/unit/common/middleware/test_ratelimit.py | 8 +- test/unit/common/middleware/test_slo.py | 10 +- test/unit/common/middleware/test_symlink.py | 3 +- test/unit/common/middleware/test_tempurl.py | 10 +- .../middleware/test_versioned_writes.py | 10 +- test/unit/common/test_registry.py | 213 ++++++++++++++++++ test/unit/common/test_utils.py | 185 --------------- test/unit/proxy/controllers/test_info.py | 58 ++--- test/unit/proxy/test_server.py | 8 +- 44 files changed, 455 insertions(+), 392 deletions(-) create mode 100644 swift/common/registry.py create mode 100644 test/unit/common/test_registry.py diff --git a/swift/common/middleware/account_quotas.py b/swift/common/middleware/account_quotas.py index 5b7731b81f..cd41e34dae 100644 --- a/swift/common/middleware/account_quotas.py +++ b/swift/common/middleware/account_quotas.py @@ -54,7 +54,7 @@ account size has been updated. from swift.common.swob import HTTPForbidden, HTTPBadRequest, \ HTTPRequestEntityTooLarge, wsgify -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info diff --git a/swift/common/middleware/bulk.py b/swift/common/middleware/bulk.py index 86798d755f..df0c881a74 100644 --- a/swift/common/middleware/bulk.py +++ b/swift/common/middleware/bulk.py @@ -206,8 +206,8 @@ from swift.common.swob import Request, HTTPBadGateway, \ HTTPPreconditionFailed, HTTPRequestEntityTooLarge, HTTPNotAcceptable, \ HTTPLengthRequired, HTTPException, HTTPServerError, wsgify, \ bytes_to_wsgi, str_to_wsgi, wsgi_unquote, wsgi_quote, wsgi_to_str -from swift.common.utils import get_logger, register_swift_info, \ - StreamingPile +from swift.common.utils import get_logger, StreamingPile +from swift.common.registry import register_swift_info from swift.common import constraints from swift.common.http import HTTP_UNAUTHORIZED, HTTP_NOT_FOUND, HTTP_CONFLICT from swift.common.request_helpers import is_user_meta diff --git a/swift/common/middleware/cname_lookup.py b/swift/common/middleware/cname_lookup.py index a1a3b4d765..2578c08fcf 100644 --- a/swift/common/middleware/cname_lookup.py +++ b/swift/common/middleware/cname_lookup.py @@ -44,7 +44,8 @@ from swift.common.middleware import RewriteContext from swift.common.swob import Request, HTTPBadRequest, \ str_to_wsgi, wsgi_to_str from swift.common.utils import cache_from_env, get_logger, is_valid_ip, \ - list_from_csv, parse_socket_string, register_swift_info + list_from_csv, parse_socket_string +from swift.common.registry import register_swift_info def lookup_cname(domain, resolver): # pragma: no cover diff --git a/swift/common/middleware/container_quotas.py b/swift/common/middleware/container_quotas.py index a78876aca5..2a10c8d2ce 100644 --- a/swift/common/middleware/container_quotas.py +++ b/swift/common/middleware/container_quotas.py @@ -54,7 +54,7 @@ For example:: from swift.common.http import is_success from swift.common.swob import HTTPRequestEntityTooLarge, HTTPBadRequest, \ wsgify -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_container_info diff --git a/swift/common/middleware/container_sync.py b/swift/common/middleware/container_sync.py index 74c63afa47..cc6b16aeea 100644 --- a/swift/common/middleware/container_sync.py +++ b/swift/common/middleware/container_sync.py @@ -19,8 +19,9 @@ from swift.common.constraints import valid_api_version from swift.common.container_sync_realms import ContainerSyncRealms from swift.common.swob import HTTPBadRequest, HTTPUnauthorized, wsgify from swift.common.utils import ( - config_true_value, get_logger, register_swift_info, streq_const_time) + config_true_value, get_logger, streq_const_time) from swift.proxy.controllers.base import get_container_info +from swift.common.registry import register_swift_info class ContainerSync(object): diff --git a/swift/common/middleware/crossdomain.py b/swift/common/middleware/crossdomain.py index 13a45a316b..ffe73d43fc 100644 --- a/swift/common/middleware/crossdomain.py +++ b/swift/common/middleware/crossdomain.py @@ -14,7 +14,7 @@ # limitations under the License. from swift.common.swob import Request, Response -from swift.common.utils import register_swift_info +from swift.common.registry import register_swift_info class CrossDomainMiddleware(object): diff --git a/swift/common/middleware/crypto/__init__.py b/swift/common/middleware/crypto/__init__.py index b526fcbaa7..a70e7b8609 100644 --- a/swift/common/middleware/crypto/__init__.py +++ b/swift/common/middleware/crypto/__init__.py @@ -20,7 +20,8 @@ instance of an :class:`~swift.common.middleware.crypto.encrypter.Encrypter`. from swift.common.middleware.crypto.decrypter import Decrypter from swift.common.middleware.crypto.encrypter import Encrypter -from swift.common.utils import config_true_value, register_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info def filter_factory(global_conf, **local_conf): diff --git a/swift/common/middleware/domain_remap.py b/swift/common/middleware/domain_remap.py index f42e65b88d..aec8e90e93 100644 --- a/swift/common/middleware/domain_remap.py +++ b/swift/common/middleware/domain_remap.py @@ -100,8 +100,8 @@ storage end points as sync destinations. from swift.common.middleware import RewriteContext from swift.common.swob import Request, HTTPBadRequest, wsgi_quote -from swift.common.utils import config_true_value, list_from_csv, \ - register_swift_info +from swift.common.utils import config_true_value, list_from_csv +from swift.common.registry import register_swift_info class _DomainRemapContext(RewriteContext): diff --git a/swift/common/middleware/etag_quoter.py b/swift/common/middleware/etag_quoter.py index ea3cbb30d6..d67c1b48b7 100644 --- a/swift/common/middleware/etag_quoter.py +++ b/swift/common/middleware/etag_quoter.py @@ -42,7 +42,8 @@ useful if some subset of applications expect Etags to be bare MD5s. from swift.common.constraints import valid_api_version from swift.common.http import is_success from swift.common.swob import Request -from swift.common.utils import config_true_value, register_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info, get_container_info diff --git a/swift/common/middleware/formpost.py b/swift/common/middleware/formpost.py index af5a5cf5d7..84a8ee09b7 100644 --- a/swift/common/middleware/formpost.py +++ b/swift/common/middleware/formpost.py @@ -132,9 +132,10 @@ from six.moves.urllib.parse import quote from swift.common.constraints import valid_api_version from swift.common.exceptions import MimeInvalid from swift.common.middleware.tempurl import get_tempurl_keys_from_metadata -from swift.common.utils import streq_const_time, register_swift_info, \ - parse_content_disposition, parse_mime_headers, \ - iter_multipart_mime_documents, reiterate, close_if_possible +from swift.common.utils import streq_const_time, parse_content_disposition, \ + parse_mime_headers, iter_multipart_mime_documents, reiterate, \ + close_if_possible +from swift.common.registry import register_swift_info from swift.common.wsgi import make_pre_authed_env from swift.common.swob import HTTPUnauthorized, wsgi_to_str, str_to_wsgi from swift.proxy.controllers.base import get_account_info, get_container_info diff --git a/swift/common/middleware/name_check.py b/swift/common/middleware/name_check.py index 9fea316492..b13c5a76bd 100644 --- a/swift/common/middleware/name_check.py +++ b/swift/common/middleware/name_check.py @@ -41,7 +41,8 @@ The filter returns HTTPBadRequest if path is invalid. ''' import re -from swift.common.utils import get_logger, register_swift_info +from swift.common.utils import get_logger +from swift.common.registry import register_swift_info from swift.common.swob import Request, HTTPBadRequest diff --git a/swift/common/middleware/ratelimit.py b/swift/common/middleware/ratelimit.py index 96ab036948..801cb7a842 100644 --- a/swift/common/middleware/ratelimit.py +++ b/swift/common/middleware/ratelimit.py @@ -17,7 +17,8 @@ from swift import gettext_ as _ import eventlet -from swift.common.utils import cache_from_env, get_logger, register_swift_info +from swift.common.utils import cache_from_env, get_logger +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_account_info, get_container_info from swift.common.constraints import valid_api_version from swift.common.memcached import MemcacheConnectionError diff --git a/swift/common/middleware/read_only.py b/swift/common/middleware/read_only.py index c62959c8eb..3d54fdc5b4 100644 --- a/swift/common/middleware/read_only.py +++ b/swift/common/middleware/read_only.py @@ -14,8 +14,8 @@ # limitations under the License. from swift.common.constraints import check_account_format from swift.common.swob import HTTPMethodNotAllowed, Request -from swift.common.utils import get_logger, config_true_value, \ - register_swift_info +from swift.common.utils import get_logger, config_true_value +from swift.common.registry import register_swift_info from swift.proxy.controllers.base import get_info """ diff --git a/swift/common/middleware/s3api/controllers/bucket.py b/swift/common/middleware/s3api/controllers/bucket.py index 30c9ec28ee..bd71bf058c 100644 --- a/swift/common/middleware/s3api/controllers/bucket.py +++ b/swift/common/middleware/s3api/controllers/bucket.py @@ -23,8 +23,8 @@ from swift.common import swob from swift.common.http import HTTP_OK from swift.common.middleware.versioned_writes.object_versioning import \ DELETE_MARKER_CONTENT_TYPE -from swift.common.utils import json, public, config_true_value, Timestamp, \ - get_swift_info +from swift.common.utils import json, public, config_true_value, Timestamp +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller from swift.common.middleware.s3api.etree import Element, SubElement, \ diff --git a/swift/common/middleware/s3api/controllers/multi_delete.py b/swift/common/middleware/s3api/controllers/multi_delete.py index 6d1c522b56..49a51f663c 100644 --- a/swift/common/middleware/s3api/controllers/multi_delete.py +++ b/swift/common/middleware/s3api/controllers/multi_delete.py @@ -18,7 +18,8 @@ import json from swift.common.constraints import MAX_OBJECT_NAME_LENGTH from swift.common.http import HTTP_NO_CONTENT -from swift.common.utils import public, StreamingPile, get_swift_info +from swift.common.utils import public, StreamingPile +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation diff --git a/swift/common/middleware/s3api/controllers/obj.py b/swift/common/middleware/s3api/controllers/obj.py index cae97b5ae2..b1f310e6e6 100644 --- a/swift/common/middleware/s3api/controllers/obj.py +++ b/swift/common/middleware/s3api/controllers/obj.py @@ -20,7 +20,8 @@ from swift.common.http import HTTP_OK, HTTP_PARTIAL_CONTENT, HTTP_NO_CONTENT from swift.common.request_helpers import update_etag_is_at_header from swift.common.swob import Range, content_range_header_value, \ normalize_etag -from swift.common.utils import public, list_from_csv, get_swift_info +from swift.common.utils import public, list_from_csv +from swift.common.registry import get_swift_info from swift.common.middleware.versioned_writes.object_versioning import \ DELETE_MARKER_CONTENT_TYPE diff --git a/swift/common/middleware/s3api/controllers/versioning.py b/swift/common/middleware/s3api/controllers/versioning.py index 9c835f3c29..2d31d2af50 100644 --- a/swift/common/middleware/s3api/controllers/versioning.py +++ b/swift/common/middleware/s3api/controllers/versioning.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from swift.common.utils import public, get_swift_info, config_true_value +from swift.common.utils import public, config_true_value +from swift.common.registry import get_swift_info from swift.common.middleware.s3api.controllers.base import Controller, \ bucket_operation diff --git a/swift/common/middleware/s3api/s3api.py b/swift/common/middleware/s3api/s3api.py index 9c225c53b9..56b6e997f1 100644 --- a/swift/common/middleware/s3api/s3api.py +++ b/swift/common/middleware/s3api/s3api.py @@ -156,11 +156,11 @@ from swift.common.middleware.s3api.exception import NotS3Request, \ from swift.common.middleware.s3api.s3request import get_request_class from swift.common.middleware.s3api.s3response import ErrorResponse, \ InternalError, MethodNotAllowed, S3ResponseBase, S3NotImplemented -from swift.common.utils import get_logger, register_swift_info, \ - config_true_value, config_positive_int_value, split_path, \ - closing_if_possible, list_from_csv +from swift.common.utils import get_logger, config_true_value, \ + config_positive_int_value, split_path, closing_if_possible, list_from_csv from swift.common.middleware.s3api.utils import Config from swift.common.middleware.s3api.acl_handlers import get_acl_handler +from swift.common.registry import register_swift_info class ListingEtagMiddleware(object): diff --git a/swift/common/middleware/s3api/s3request.py b/swift/common/middleware/s3api/s3request.py index d4c33512de..1522853400 100644 --- a/swift/common/middleware/s3api/s3request.py +++ b/swift/common/middleware/s3api/s3request.py @@ -25,8 +25,8 @@ import six from six.moves.urllib.parse import quote, unquote, parse_qsl import string -from swift.common.utils import split_path, json, get_swift_info, \ - close_if_possible, md5 +from swift.common.utils import split_path, json, close_if_possible, md5 +from swift.common.registry import get_swift_info from swift.common import swob from swift.common.http import HTTP_OK, HTTP_CREATED, HTTP_ACCEPTED, \ HTTP_NO_CONTENT, HTTP_UNAUTHORIZED, HTTP_FORBIDDEN, HTTP_NOT_FOUND, \ diff --git a/swift/common/middleware/slo.py b/swift/common/middleware/slo.py index 88554e1cb4..857a52f224 100644 --- a/swift/common/middleware/slo.py +++ b/swift/common/middleware/slo.py @@ -345,9 +345,10 @@ from swift.common.swob import Request, HTTPBadRequest, HTTPServerError, \ RESPONSE_REASONS, str_to_wsgi, bytes_to_wsgi, wsgi_to_str, wsgi_quote from swift.common.utils import get_logger, config_true_value, \ get_valid_utf8_str, override_bytes_from_content_type, split_path, \ - register_swift_info, RateLimitedIterator, quote, close_if_possible, \ - closing_if_possible, LRUCache, StreamingPile, strict_b64decode, \ - Timestamp, drain_and_close, get_expirer_container, md5 + RateLimitedIterator, quote, close_if_possible, closing_if_possible, \ + LRUCache, StreamingPile, strict_b64decode, Timestamp, drain_and_close, \ + get_expirer_container, md5 +from swift.common.registry import register_swift_info from swift.common.request_helpers import SegmentedIterable, \ get_sys_meta_prefix, update_etag_is_at_header, resolve_etag_is_at_header, \ get_container_update_override_key, update_ignore_range_header diff --git a/swift/common/middleware/staticweb.py b/swift/common/middleware/staticweb.py index 7e25e61675..23377dfae3 100644 --- a/swift/common/middleware/staticweb.py +++ b/swift/common/middleware/staticweb.py @@ -131,7 +131,8 @@ from six.moves.urllib.parse import urlparse from swift.common.request_helpers import html_escape from swift.common.utils import human_readable, split_path, config_true_value, \ - quote, register_swift_info, get_logger + quote, get_logger +from swift.common.registry import register_swift_info from swift.common.wsgi import make_env, WSGIContext from swift.common.http import is_success, is_redirection, HTTP_NOT_FOUND from swift.common.swob import Response, HTTPMovedPermanently, HTTPNotFound, \ diff --git a/swift/common/middleware/symlink.py b/swift/common/middleware/symlink.py index 2aac5d1ebc..d951577d91 100644 --- a/swift/common/middleware/symlink.py +++ b/swift/common/middleware/symlink.py @@ -201,9 +201,10 @@ import json import os from cgi import parse_header -from swift.common.utils import get_logger, register_swift_info, split_path, \ +from swift.common.utils import get_logger, split_path, \ MD5_OF_EMPTY_STRING, close_if_possible, closing_if_possible, \ config_true_value, drain_and_close +from swift.common.registry import register_swift_info from swift.common.constraints import check_account_format from swift.common.wsgi import WSGIContext, make_subrequest, \ make_pre_authed_request diff --git a/swift/common/middleware/tempauth.py b/swift/common/middleware/tempauth.py index be1ff0b6cb..c7242faa55 100644 --- a/swift/common/middleware/tempauth.py +++ b/swift/common/middleware/tempauth.py @@ -191,7 +191,8 @@ from swift.common.request_helpers import get_sys_meta_prefix from swift.common.middleware.acl import ( clean_acl, parse_acl, referrer_allowed, acls_from_account_info) from swift.common.utils import cache_from_env, get_logger, \ - split_path, config_true_value, register_swift_info + split_path, config_true_value +from swift.common.registry import register_swift_info from swift.common.utils import config_read_reseller_options, quote from swift.proxy.controllers.base import get_account_info diff --git a/swift/common/middleware/tempurl.py b/swift/common/middleware/tempurl.py index 1fcd5bcdd0..c7169eba0f 100644 --- a/swift/common/middleware/tempurl.py +++ b/swift/common/middleware/tempurl.py @@ -314,8 +314,8 @@ from swift.common.header_key_dict import HeaderKeyDict from swift.common.swob import header_to_environ_key, HTTPUnauthorized, \ HTTPBadRequest, wsgi_to_str from swift.common.utils import split_path, get_valid_utf8_str, \ - register_swift_info, get_hmac, streq_const_time, quote, get_logger, \ - strict_b64decode + get_hmac, streq_const_time, quote, get_logger, strict_b64decode +from swift.common.registry import register_swift_info DISALLOWED_INCOMING_HEADERS = 'x-object-manifest x-symlink-target' @@ -498,7 +498,7 @@ class TempURL(object): if env['REQUEST_METHOD'] == 'OPTIONS': return self.app(env, start_response) info = self._get_temp_url_info(env) - temp_url_sig, temp_url_expires, temp_url_prefix, filename,\ + temp_url_sig, temp_url_expires, temp_url_prefix, filename, \ inline_disposition, temp_url_ip_range = info if temp_url_sig is None and temp_url_expires is None: return self.app(env, start_response) diff --git a/swift/common/middleware/versioned_writes/__init__.py b/swift/common/middleware/versioned_writes/__init__.py index 1bc41c3e20..368cb9b716 100644 --- a/swift/common/middleware/versioned_writes/__init__.py +++ b/swift/common/middleware/versioned_writes/__init__.py @@ -25,8 +25,8 @@ from swift.common.middleware.versioned_writes. \ from swift.common.middleware.versioned_writes. \ object_versioning import ObjectVersioningMiddleware -from swift.common.utils import config_true_value, register_swift_info, \ - get_swift_info +from swift.common.utils import config_true_value +from swift.common.registry import register_swift_info, get_swift_info def filter_factory(global_conf, **local_conf): diff --git a/swift/common/registry.py b/swift/common/registry.py new file mode 100644 index 0000000000..9e515ae237 --- /dev/null +++ b/swift/common/registry.py @@ -0,0 +1,86 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Used by get_swift_info and register_swift_info to store information about +# the swift cluster. +from copy import deepcopy + +_swift_info = {} +_swift_admin_info = {} + + +def get_swift_info(admin=False, disallowed_sections=None): + """ + Returns information about the swift cluster that has been previously + registered with the register_swift_info call. + + :param admin: boolean value, if True will additionally return an 'admin' + section with information previously registered as admin + info. + :param disallowed_sections: list of section names to be withheld from the + information returned. + :returns: dictionary of information about the swift cluster. + """ + disallowed_sections = disallowed_sections or [] + info = deepcopy(_swift_info) + for section in disallowed_sections: + key_to_pop = None + sub_section_dict = info + for sub_section in section.split('.'): + if key_to_pop: + sub_section_dict = sub_section_dict.get(key_to_pop, {}) + if not isinstance(sub_section_dict, dict): + sub_section_dict = {} + break + key_to_pop = sub_section + sub_section_dict.pop(key_to_pop, None) + + if admin: + info['admin'] = dict(_swift_admin_info) + info['admin']['disallowed_sections'] = list(disallowed_sections) + return info + + +def register_swift_info(name='swift', admin=False, **kwargs): + """ + Registers information about the swift cluster to be retrieved with calls + to get_swift_info. + + NOTE: Do not use "." in the param: name or any keys in kwargs. "." is used + in the disallowed_sections to remove unwanted keys from /info. + + :param name: string, the section name to place the information under. + :param admin: boolean, if True, information will be registered to an + admin section which can optionally be withheld when + requesting the information. + :param kwargs: key value arguments representing the information to be + added. + :raises ValueError: if name or any of the keys in kwargs has "." in it + """ + if name == 'admin' or name == 'disallowed_sections': + raise ValueError('\'{0}\' is reserved name.'.format(name)) + + if admin: + dict_to_use = _swift_admin_info + else: + dict_to_use = _swift_info + if name not in dict_to_use: + if "." in name: + raise ValueError('Cannot use "." in a swift_info key: %s' % name) + dict_to_use[name] = {} + for key, val in kwargs.items(): + if "." in key: + raise ValueError('Cannot use "." in a swift_info key: %s' % key) + dict_to_use[name][key] = val diff --git a/swift/common/utils.py b/swift/common/utils.py index 624e299633..2981e7f7dd 100644 --- a/swift/common/utils.py +++ b/swift/common/utils.py @@ -45,7 +45,6 @@ from random import random, shuffle from contextlib import contextmanager, closing import ctypes import ctypes.util -from copy import deepcopy from optparse import OptionParser import traceback import warnings @@ -90,6 +89,9 @@ from swift.common.http import is_server_error from swift.common.header_key_dict import HeaderKeyDict from swift.common.linkat import linkat +# For backwards compatability with 3rd party middlewares +from swift.common.registry import register_swift_info, get_swift_info # noqa + # logging doesn't import patched as cleanly as one would like from logging.handlers import SysLogHandler import logging @@ -318,77 +320,6 @@ def get_hmac(request_method, path, expires, key, digest=sha1, return hmac.new(key, message, digest).hexdigest() -# Used by get_swift_info and register_swift_info to store information about -# the swift cluster. -_swift_info = {} -_swift_admin_info = {} - - -def get_swift_info(admin=False, disallowed_sections=None): - """ - Returns information about the swift cluster that has been previously - registered with the register_swift_info call. - - :param admin: boolean value, if True will additionally return an 'admin' - section with information previously registered as admin - info. - :param disallowed_sections: list of section names to be withheld from the - information returned. - :returns: dictionary of information about the swift cluster. - """ - disallowed_sections = disallowed_sections or [] - info = deepcopy(_swift_info) - for section in disallowed_sections: - key_to_pop = None - sub_section_dict = info - for sub_section in section.split('.'): - if key_to_pop: - sub_section_dict = sub_section_dict.get(key_to_pop, {}) - if not isinstance(sub_section_dict, dict): - sub_section_dict = {} - break - key_to_pop = sub_section - sub_section_dict.pop(key_to_pop, None) - - if admin: - info['admin'] = dict(_swift_admin_info) - info['admin']['disallowed_sections'] = list(disallowed_sections) - return info - - -def register_swift_info(name='swift', admin=False, **kwargs): - """ - Registers information about the swift cluster to be retrieved with calls - to get_swift_info. - - NOTE: Do not use "." in the param: name or any keys in kwargs. "." is used - in the disallowed_sections to remove unwanted keys from /info. - - :param name: string, the section name to place the information under. - :param admin: boolean, if True, information will be registered to an - admin section which can optionally be withheld when - requesting the information. - :param kwargs: key value arguments representing the information to be - added. - :raises ValueError: if name or any of the keys in kwargs has "." in it - """ - if name == 'admin' or name == 'disallowed_sections': - raise ValueError('\'{0}\' is reserved name.'.format(name)) - - if admin: - dict_to_use = _swift_admin_info - else: - dict_to_use = _swift_info - if name not in dict_to_use: - if "." in name: - raise ValueError('Cannot use "." in a swift_info key: %s' % name) - dict_to_use[name] = {} - for key, val in kwargs.items(): - if "." in key: - raise ValueError('Cannot use "." in a swift_info key: %s' % key) - dict_to_use[name][key] = val - - def backward(f, blocksize=4096): """ A generator returning lines from a file starting with the last line, diff --git a/swift/proxy/controllers/info.py b/swift/proxy/controllers/info.py index 1d148e9988..05bc1a998b 100644 --- a/swift/proxy/controllers/info.py +++ b/swift/proxy/controllers/info.py @@ -16,8 +16,8 @@ import json from time import time -from swift.common.utils import public, get_hmac, get_swift_info, \ - streq_const_time +from swift.common.utils import public, get_hmac, streq_const_time +from swift.common.registry import get_swift_info from swift.proxy.controllers.base import Controller, delay_denial from swift.common.swob import HTTPOk, HTTPForbidden, HTTPUnauthorized diff --git a/swift/proxy/server.py b/swift/proxy/server.py index 1e1388c419..5474102131 100644 --- a/swift/proxy/server.py +++ b/swift/proxy/server.py @@ -34,8 +34,9 @@ from swift.common.ring import Ring from swift.common.utils import Watchdog, get_logger, \ get_remote_client, split_path, config_true_value, generate_trans_id, \ affinity_key_function, affinity_locality_predicate, list_from_csv, \ - register_swift_info, parse_prefixed_conf, config_auto_int_value, \ + parse_prefixed_conf, config_auto_int_value, \ config_request_node_count_value, config_percent_value +from swift.common.registry import register_swift_info from swift.common.constraints import check_utf8, valid_api_version from swift.proxy.controllers import AccountController, ContainerController, \ ObjectControllerRouter, InfoController diff --git a/test/unit/common/middleware/crypto/test_crypto.py b/test/unit/common/middleware/crypto/test_crypto.py index 882e959de9..2daef42bdb 100644 --- a/test/unit/common/middleware/crypto/test_crypto.py +++ b/test/unit/common/middleware/crypto/test_crypto.py @@ -15,7 +15,7 @@ import unittest import mock -from swift.common import utils +from swift.common import registry from swift.common.middleware import crypto @@ -24,36 +24,38 @@ class TestCrypto(unittest.TestCase): def do_test(conf, expect_enabled): fake_app = object() - with mock.patch.dict('swift.common.utils._swift_admin_info', + with mock.patch.dict('swift.common.registry._swift_admin_info', clear=True): # we're not expecting utils._swift_info to be modified but mock # it anyway just in case it is - with mock.patch.dict('swift.common.utils._swift_info', + with mock.patch.dict('swift.common.registry._swift_info', clear=True): # Sanity checks... - self.assertNotIn('encryption', utils._swift_admin_info) + self.assertNotIn('encryption', registry._swift_admin_info) self.assertNotIn('encryption', - utils.get_swift_info(admin=True)) - self.assertNotIn('encryption', - utils.get_swift_info(admin=True)['admin']) + registry.get_swift_info(admin=True)) + self.assertNotIn( + 'encryption', + registry.get_swift_info(admin=True)['admin']) factory = crypto.filter_factory(conf) self.assertTrue(callable(factory)) filtered_app = factory(fake_app) - self.assertNotIn('encryption', utils._swift_info) - self.assertNotIn('encryption', utils.get_swift_info()) + self.assertNotIn('encryption', registry._swift_info) + self.assertNotIn('encryption', registry.get_swift_info()) self.assertNotIn('encryption', - utils.get_swift_info(admin=True)) + registry.get_swift_info(admin=True)) - self.assertIn('encryption', utils._swift_admin_info) - self.assertDictEqual({'enabled': expect_enabled}, - utils._swift_admin_info['encryption']) - self.assertIn('encryption', - utils.get_swift_info(admin=True)['admin']) + self.assertIn('encryption', registry._swift_admin_info) self.assertDictEqual( {'enabled': expect_enabled}, - utils.get_swift_info( + registry._swift_admin_info['encryption']) + self.assertIn('encryption', + registry.get_swift_info(admin=True)['admin']) + self.assertDictEqual( + {'enabled': expect_enabled}, + registry.get_swift_info( admin=True)['admin']['encryption']) self.assertIsInstance(filtered_app, crypto.decrypter.Decrypter) diff --git a/test/unit/common/middleware/s3api/test_s3api.py b/test/unit/common/middleware/s3api/test_s3api.py index 57a2cde6e6..a0f37f24eb 100644 --- a/test/unit/common/middleware/s3api/test_s3api.py +++ b/test/unit/common/middleware/s3api/test_s3api.py @@ -28,7 +28,7 @@ from six.moves.urllib.parse import unquote, quote import swift.common.middleware.s3api from swift.common.middleware.s3api.utils import Config from swift.common.middleware.keystoneauth import KeystoneAuth -from swift.common import swob, utils +from swift.common import swob, registry from swift.common.swob import Request from swift.common.utils import md5 @@ -759,15 +759,15 @@ class TestS3ApiMiddleware(S3ApiTestCase): def test_mfa(self): self._test_unsupported_header('x-amz-mfa') - @mock.patch.object(utils, '_swift_admin_info', new_callable=dict) + @mock.patch.object(registry, '_swift_admin_info', new_callable=dict) def test_server_side_encryption(self, mock_info): sse_header = 'x-amz-server-side-encryption' self._test_unsupported_header(sse_header, 'AES256') self._test_unsupported_header(sse_header, 'aws:kms') - utils.register_swift_info('encryption', admin=True, enabled=False) + registry.register_swift_info('encryption', admin=True, enabled=False) self._test_unsupported_header(sse_header, 'AES256') self._test_unsupported_header(sse_header, 'aws:kms') - utils.register_swift_info('encryption', admin=True, enabled=True) + registry.register_swift_info('encryption', admin=True, enabled=True) # AES256 now works self.swift.register('PUT', '/v1/AUTH_X/bucket/object', swob.HTTPCreated, {}, None) @@ -870,7 +870,7 @@ class TestS3ApiMiddleware(S3ApiTestCase): def test_registered_defaults(self): conf_from_file = {k: str(v) for k, v in self.conf.items()} filter_factory(conf_from_file) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('s3api' in swift_info) self.assertEqual(swift_info['s3api'].get('max_bucket_listing'), self.conf['max_bucket_listing']) diff --git a/test/unit/common/middleware/test_bulk.py b/test/unit/common/middleware/test_bulk.py index 166abe8efa..5b3a324baf 100644 --- a/test/unit/common/middleware/test_bulk.py +++ b/test/unit/common/middleware/test_bulk.py @@ -29,7 +29,7 @@ from eventlet import sleep from mock import patch, call from test.debug_logger import debug_logger from test.unit.common.middleware.helpers import FakeSwift -from swift.common import utils, constraints +from swift.common import utils, constraints, registry from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import bulk from swift.common.swob import Request, Response, HTTPException, \ @@ -1035,12 +1035,12 @@ class TestConfig(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): bulk.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('bulk_upload' in swift_info) self.assertTrue(isinstance( swift_info['bulk_upload'].get('max_containers_per_extraction'), diff --git a/test/unit/common/middleware/test_cname_lookup.py b/test/unit/common/middleware/test_cname_lookup.py index 2dbea66540..600228e555 100644 --- a/test/unit/common/middleware/test_cname_lookup.py +++ b/test/unit/common/middleware/test_cname_lookup.py @@ -24,7 +24,7 @@ except ImportError: skip = True else: # executed if the try has no errors skip = False -from swift.common import utils +from swift.common import registry from swift.common.middleware import cname_lookup from swift.common.swob import Request, HTTPMovedPermanently @@ -437,17 +437,17 @@ class TestCNAMELookup(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): cname_lookup.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('cname_lookup', swift_info) self.assertEqual(swift_info['cname_lookup'].get('lookup_depth'), 1) def test_registered_nondefaults(self): cname_lookup.filter_factory({'lookup_depth': '2'}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('cname_lookup', swift_info) self.assertEqual(swift_info['cname_lookup'].get('lookup_depth'), 2) diff --git a/test/unit/common/middleware/test_domain_remap.py b/test/unit/common/middleware/test_domain_remap.py index e89db9a73e..317a20a440 100644 --- a/test/unit/common/middleware/test_domain_remap.py +++ b/test/unit/common/middleware/test_domain_remap.py @@ -18,7 +18,7 @@ import unittest from swift.common.swob import Request, HTTPMovedPermanently from swift.common.middleware import domain_remap -from swift.common import utils +from swift.common import registry class FakeApp(object): @@ -321,12 +321,12 @@ class TestDomainRemapClientMangling(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): domain_remap.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('domain_remap', swift_info) self.assertEqual(swift_info['domain_remap'], { 'default_reseller_prefix': None}) @@ -334,7 +334,7 @@ class TestSwiftInfo(unittest.TestCase): def test_registered_nondefaults(self): domain_remap.filter_factory({'default_reseller_prefix': 'cupcake', 'mangle_client_paths': 'yes'}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('domain_remap', swift_info) self.assertEqual(swift_info['domain_remap'], { 'default_reseller_prefix': 'cupcake'}) diff --git a/test/unit/common/middleware/test_name_check.py b/test/unit/common/middleware/test_name_check.py index a9d56f117d..09c97f39ed 100644 --- a/test/unit/common/middleware/test_name_check.py +++ b/test/unit/common/middleware/test_name_check.py @@ -26,7 +26,7 @@ import unittest from swift.common.swob import Request, Response from swift.common.middleware import name_check -from swift.common import utils +from swift.common import registry MAX_LENGTH = 255 FORBIDDEN_CHARS = '\'\"<>`' @@ -117,12 +117,12 @@ class TestNameCheckMiddleware(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): name_check.filter_factory({})(FakeApp()) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('name_check' in swift_info) self.assertTrue(isinstance( swift_info['name_check'].get('maximum_length'), @@ -139,7 +139,7 @@ class TestSwiftInfo(unittest.TestCase): 'forbidden_chars': '\'\"`', 'forbidden_regexp': r"/\./|/\.\./|/\.$"} name_check.filter_factory(conf)(FakeApp()) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('name_check' in swift_info) self.assertEqual(swift_info['name_check'].get('maximum_length'), 512) self.assertEqual(set(swift_info['name_check'].get('forbidden_chars')), diff --git a/test/unit/common/middleware/test_ratelimit.py b/test/unit/common/middleware/test_ratelimit.py index 8af95edaa5..7b137f9ba4 100644 --- a/test/unit/common/middleware/test_ratelimit.py +++ b/test/unit/common/middleware/test_ratelimit.py @@ -24,7 +24,7 @@ from swift.common.middleware import ratelimit from swift.proxy.controllers.base import get_cache_key, \ headers_to_container_info from swift.common.swob import Request -from swift.common import utils +from swift.common import registry threading = eventlet.patcher.original('threading') @@ -546,8 +546,8 @@ class TestRateLimit(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): @@ -568,7 +568,7 @@ class TestSwiftInfo(unittest.TestCase): 'container_listing_ratelimit_50': 50} ratelimit.filter_factory(test_limits)('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('ratelimit', swift_info) self.assertEqual(swift_info['ratelimit'] ['account_ratelimit'], 1.0) diff --git a/test/unit/common/middleware/test_slo.py b/test/unit/common/middleware/test_slo.py index f4c45423f6..045778a167 100644 --- a/test/unit/common/middleware/test_slo.py +++ b/test/unit/common/middleware/test_slo.py @@ -24,7 +24,7 @@ from mock import patch import six from io import BytesIO -from swift.common import swob, utils +from swift.common import swob, registry from swift.common.header_key_dict import HeaderKeyDict from swift.common.middleware import slo from swift.common.swob import Request, HTTPException, str_to_wsgi, \ @@ -4534,12 +4534,12 @@ class TestSloBulkDeleter(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): mware = slo.filter_factory({})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), mware.max_manifest_segments) @@ -4564,7 +4564,7 @@ class TestSwiftInfo(unittest.TestCase): rate_limit_segments_per_sec=2, yield_frequency=5, concurrency=1, delete_concurrency=3, allow_async_delete='y') mware = slo.filter_factory(conf)('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertTrue('slo' in swift_info) self.assertEqual(swift_info['slo'].get('max_manifest_segments'), 500) self.assertEqual(swift_info['slo'].get('min_segment_size'), 1) diff --git a/test/unit/common/middleware/test_symlink.py b/test/unit/common/middleware/test_symlink.py index 3687305bf7..9d88382846 100644 --- a/test/unit/common/middleware/test_symlink.py +++ b/test/unit/common/middleware/test_symlink.py @@ -25,7 +25,8 @@ from swift.common.middleware import symlink, copy, versioned_writes, \ listing_formats from swift.common.swob import Request from swift.common.request_helpers import get_reserved_name -from swift.common.utils import MD5_OF_EMPTY_STRING, get_swift_info +from swift.common.utils import MD5_OF_EMPTY_STRING +from swift.common.registry import get_swift_info from test.unit.common.middleware.helpers import FakeSwift from test.unit.common.middleware.test_versioned_writes import FakeCache diff --git a/test/unit/common/middleware/test_tempurl.py b/test/unit/common/middleware/test_tempurl.py index 0ec82615ca..00546d7c3f 100644 --- a/test/unit/common/middleware/test_tempurl.py +++ b/test/unit/common/middleware/test_tempurl.py @@ -41,7 +41,7 @@ from time import time, strftime, gmtime from swift.common.middleware import tempauth, tempurl from swift.common.header_key_dict import HeaderKeyDict from swift.common.swob import Request, Response -from swift.common import utils +from swift.common import utils, registry class FakeApp(object): @@ -1571,12 +1571,12 @@ class TestTempURL(unittest.TestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): tempurl.filter_factory({}) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('tempurl', swift_info) info = swift_info['tempurl'] self.assertEqual(set(info['methods']), @@ -1599,7 +1599,7 @@ class TestSwiftInfo(unittest.TestCase): 'outgoing_allow_headers': 'x-object-meta-* content-type', 'allowed_digests': 'sha512 md5 not-a-valid-digest', }) - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('tempurl', swift_info) info = swift_info['tempurl'] self.assertEqual(set(info['methods']), diff --git a/test/unit/common/middleware/test_versioned_writes.py b/test/unit/common/middleware/test_versioned_writes.py index 2f997940a4..ff6ec59da9 100644 --- a/test/unit/common/middleware/test_versioned_writes.py +++ b/test/unit/common/middleware/test_versioned_writes.py @@ -19,7 +19,7 @@ import os import time import mock import unittest -from swift.common import swob, utils +from swift.common import swob, utils, registry from swift.common.middleware import versioned_writes, copy from swift.common.swob import Request from test.unit.common.middleware import helpers @@ -1515,19 +1515,19 @@ class VersionedWritesCopyingTestCase(VersionedWritesBaseTestCase): class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): versioned_writes.filter_factory({})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() # in default, versioned_writes is not in swift_info self.assertNotIn('versioned_writes', swift_info) def test_registered_explicitly_set(self): versioned_writes.filter_factory( {'allow_versioned_writes': 'true'})('have to pass in an app') - swift_info = utils.get_swift_info() + swift_info = registry.get_swift_info() self.assertIn('versioned_writes', swift_info) self.assertEqual( swift_info['versioned_writes'].get('allowed_flags'), diff --git a/test/unit/common/test_registry.py b/test/unit/common/test_registry.py new file mode 100644 index 0000000000..6fc17b3ba5 --- /dev/null +++ b/test/unit/common/test_registry.py @@ -0,0 +1,213 @@ +# Copyright (c) 2022 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from swift.common import registry, utils + +import unittest + + +class TestSwiftInfo(unittest.TestCase): + + def tearDown(self): + registry._swift_info = {} + registry._swift_admin_info = {} + + def test_register_swift_info(self): + registry.register_swift_info(foo='bar') + registry.register_swift_info(lorem='ipsum') + registry.register_swift_info('cap1', cap1_foo='cap1_bar') + registry.register_swift_info('cap1', cap1_lorem='cap1_ipsum') + + self.assertTrue('swift' in registry._swift_info) + self.assertTrue('foo' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + self.assertTrue('lorem' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['lorem'], 'ipsum') + + self.assertTrue('cap1' in registry._swift_info) + self.assertTrue('cap1_foo' in registry._swift_info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + self.assertTrue('cap1_lorem' in registry._swift_info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_lorem'], + 'cap1_ipsum') + + self.assertRaises(ValueError, + registry.register_swift_info, 'admin', foo='bar') + + self.assertRaises(ValueError, + registry.register_swift_info, 'disallowed_sections', + disallowed_sections=None) + + registry.register_swift_info('goodkey', foo='5.6') + self.assertRaises(ValueError, + registry.register_swift_info, 'bad.key', foo='5.6') + data = {'bad.key': '5.6'} + self.assertRaises(ValueError, + registry.register_swift_info, 'goodkey', **data) + + def test_get_swift_info(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info() + + self.assertNotIn('admin', info) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + + self.assertIn('cap1', info) + self.assertIn('cap1_foo', info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + + def test_get_swift_info_with_disallowed_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap3_foo': 'cap3_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info(disallowed_sections=['cap1', 'cap3']) + + self.assertNotIn('admin', info) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(info['swift']['foo'], 'bar') + + self.assertNotIn('cap1', info) + + self.assertIn('cap2', info) + self.assertIn('cap2_foo', info['cap2']) + self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') + + self.assertNotIn('cap3', info) + + def test_register_swift_admin_info(self): + registry.register_swift_info(admin=True, admin_foo='admin_bar') + registry.register_swift_info(admin=True, admin_lorem='admin_ipsum') + registry.register_swift_info('cap1', admin=True, ac1_foo='ac1_bar') + registry.register_swift_info('cap1', admin=True, ac1_lorem='ac1_ipsum') + + self.assertIn('swift', registry._swift_admin_info) + self.assertIn('admin_foo', registry._swift_admin_info['swift']) + self.assertEqual( + registry._swift_admin_info['swift']['admin_foo'], 'admin_bar') + self.assertIn('admin_lorem', registry._swift_admin_info['swift']) + self.assertEqual( + registry._swift_admin_info['swift']['admin_lorem'], 'admin_ipsum') + + self.assertIn('cap1', registry._swift_admin_info) + self.assertIn('ac1_foo', registry._swift_admin_info['cap1']) + self.assertEqual( + registry._swift_admin_info['cap1']['ac1_foo'], 'ac1_bar') + self.assertIn('ac1_lorem', registry._swift_admin_info['cap1']) + self.assertEqual( + registry._swift_admin_info['cap1']['ac1_lorem'], 'ac1_ipsum') + + self.assertNotIn('swift', registry._swift_info) + self.assertNotIn('cap1', registry._swift_info) + + def test_get_swift_admin_info(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info(admin=True) + + self.assertIn('admin', info) + self.assertIn('admin_cap1', info['admin']) + self.assertIn('ac1_foo', info['admin']['admin_cap1']) + self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + + self.assertIn('cap1', info) + self.assertIn('cap1_foo', info['cap1']) + self.assertEqual(registry._swift_info['cap1']['cap1_foo'], 'cap1_bar') + + def test_get_swift_admin_info_with_disallowed_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap3_foo': 'cap3_bar'}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info( + admin=True, disallowed_sections=['cap1', 'cap3']) + + self.assertIn('admin', info) + self.assertIn('admin_cap1', info['admin']) + self.assertIn('ac1_foo', info['admin']['admin_cap1']) + self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') + self.assertIn('disallowed_sections', info['admin']) + self.assertIn('cap1', info['admin']['disallowed_sections']) + self.assertNotIn('cap2', info['admin']['disallowed_sections']) + self.assertIn('cap3', info['admin']['disallowed_sections']) + + self.assertIn('swift', info) + self.assertIn('foo', info['swift']) + self.assertEqual(info['swift']['foo'], 'bar') + + self.assertNotIn('cap1', info) + + self.assertIn('cap2', info) + self.assertIn('cap2_foo', info['cap2']) + self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') + + self.assertNotIn('cap3', info) + + def test_get_swift_admin_info_with_disallowed_sub_sections(self): + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': {'cap1_foo': 'cap1_bar', + 'cap1_moo': 'cap1_baa'}, + 'cap2': {'cap2_foo': 'cap2_bar'}, + 'cap3': {'cap2_foo': 'cap2_bar'}, + 'cap4': {'a': {'b': {'c': 'c'}, + 'b.c': 'b.c'}}} + registry._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} + + info = registry.get_swift_info( + admin=True, disallowed_sections=['cap1.cap1_foo', 'cap3', + 'cap4.a.b.c']) + self.assertNotIn('cap3', info) + self.assertEqual(info['cap1']['cap1_moo'], 'cap1_baa') + self.assertNotIn('cap1_foo', info['cap1']) + self.assertNotIn('c', info['cap4']['a']['b']) + self.assertEqual(info['cap4']['a']['b.c'], 'b.c') + + def test_get_swift_info_with_unmatched_disallowed_sections(self): + cap1 = {'cap1_foo': 'cap1_bar', + 'cap1_moo': 'cap1_baa'} + registry._swift_info = {'swift': {'foo': 'bar'}, + 'cap1': cap1} + # expect no exceptions + info = registry.get_swift_info( + disallowed_sections=['cap2.cap1_foo', 'cap1.no_match', + 'cap1.cap1_foo.no_match.no_match']) + self.assertEqual(info['cap1'], cap1) + + def test_register_swift_info_import_from_utils(self): + # verify that the functions are available to import from utils + utils.register_swift_info(foo='bar') + self.assertTrue('swift' in registry._swift_info) + self.assertTrue('foo' in registry._swift_info['swift']) + self.assertEqual(registry._swift_info['swift']['foo'], 'bar') + self.assertEqual(registry.get_swift_info(admin=True), + utils.get_swift_info(admin=True)) diff --git a/test/unit/common/test_utils.py b/test/unit/common/test_utils.py index a5272a8b85..ba2ff6df21 100644 --- a/test/unit/common/test_utils.py +++ b/test/unit/common/test_utils.py @@ -5152,191 +5152,6 @@ class TestUnlinkOlder(unittest.TestCase): utils.unlink_paths_older_than([path], next(self.ts)) -class TestSwiftInfo(unittest.TestCase): - - def tearDown(self): - utils._swift_info = {} - utils._swift_admin_info = {} - - def test_register_swift_info(self): - utils.register_swift_info(foo='bar') - utils.register_swift_info(lorem='ipsum') - utils.register_swift_info('cap1', cap1_foo='cap1_bar') - utils.register_swift_info('cap1', cap1_lorem='cap1_ipsum') - - self.assertTrue('swift' in utils._swift_info) - self.assertTrue('foo' in utils._swift_info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - self.assertTrue('lorem' in utils._swift_info['swift']) - self.assertEqual(utils._swift_info['swift']['lorem'], 'ipsum') - - self.assertTrue('cap1' in utils._swift_info) - self.assertTrue('cap1_foo' in utils._swift_info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - self.assertTrue('cap1_lorem' in utils._swift_info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_lorem'], 'cap1_ipsum') - - self.assertRaises(ValueError, - utils.register_swift_info, 'admin', foo='bar') - - self.assertRaises(ValueError, - utils.register_swift_info, 'disallowed_sections', - disallowed_sections=None) - - utils.register_swift_info('goodkey', foo='5.6') - self.assertRaises(ValueError, - utils.register_swift_info, 'bad.key', foo='5.6') - data = {'bad.key': '5.6'} - self.assertRaises(ValueError, - utils.register_swift_info, 'goodkey', **data) - - def test_get_swift_info(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info() - - self.assertNotIn('admin', info) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - - self.assertIn('cap1', info) - self.assertIn('cap1_foo', info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - - def test_get_swift_info_with_disallowed_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap3_foo': 'cap3_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info(disallowed_sections=['cap1', 'cap3']) - - self.assertNotIn('admin', info) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(info['swift']['foo'], 'bar') - - self.assertNotIn('cap1', info) - - self.assertIn('cap2', info) - self.assertIn('cap2_foo', info['cap2']) - self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') - - self.assertNotIn('cap3', info) - - def test_register_swift_admin_info(self): - utils.register_swift_info(admin=True, admin_foo='admin_bar') - utils.register_swift_info(admin=True, admin_lorem='admin_ipsum') - utils.register_swift_info('cap1', admin=True, ac1_foo='ac1_bar') - utils.register_swift_info('cap1', admin=True, ac1_lorem='ac1_ipsum') - - self.assertIn('swift', utils._swift_admin_info) - self.assertIn('admin_foo', utils._swift_admin_info['swift']) - self.assertEqual( - utils._swift_admin_info['swift']['admin_foo'], 'admin_bar') - self.assertIn('admin_lorem', utils._swift_admin_info['swift']) - self.assertEqual( - utils._swift_admin_info['swift']['admin_lorem'], 'admin_ipsum') - - self.assertIn('cap1', utils._swift_admin_info) - self.assertIn('ac1_foo', utils._swift_admin_info['cap1']) - self.assertEqual( - utils._swift_admin_info['cap1']['ac1_foo'], 'ac1_bar') - self.assertIn('ac1_lorem', utils._swift_admin_info['cap1']) - self.assertEqual( - utils._swift_admin_info['cap1']['ac1_lorem'], 'ac1_ipsum') - - self.assertNotIn('swift', utils._swift_info) - self.assertNotIn('cap1', utils._swift_info) - - def test_get_swift_admin_info(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info(admin=True) - - self.assertIn('admin', info) - self.assertIn('admin_cap1', info['admin']) - self.assertIn('ac1_foo', info['admin']['admin_cap1']) - self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(utils._swift_info['swift']['foo'], 'bar') - - self.assertIn('cap1', info) - self.assertIn('cap1_foo', info['cap1']) - self.assertEqual(utils._swift_info['cap1']['cap1_foo'], 'cap1_bar') - - def test_get_swift_admin_info_with_disallowed_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap3_foo': 'cap3_bar'}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info( - admin=True, disallowed_sections=['cap1', 'cap3']) - - self.assertIn('admin', info) - self.assertIn('admin_cap1', info['admin']) - self.assertIn('ac1_foo', info['admin']['admin_cap1']) - self.assertEqual(info['admin']['admin_cap1']['ac1_foo'], 'ac1_bar') - self.assertIn('disallowed_sections', info['admin']) - self.assertIn('cap1', info['admin']['disallowed_sections']) - self.assertNotIn('cap2', info['admin']['disallowed_sections']) - self.assertIn('cap3', info['admin']['disallowed_sections']) - - self.assertIn('swift', info) - self.assertIn('foo', info['swift']) - self.assertEqual(info['swift']['foo'], 'bar') - - self.assertNotIn('cap1', info) - - self.assertIn('cap2', info) - self.assertIn('cap2_foo', info['cap2']) - self.assertEqual(info['cap2']['cap2_foo'], 'cap2_bar') - - self.assertNotIn('cap3', info) - - def test_get_swift_admin_info_with_disallowed_sub_sections(self): - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': {'cap1_foo': 'cap1_bar', - 'cap1_moo': 'cap1_baa'}, - 'cap2': {'cap2_foo': 'cap2_bar'}, - 'cap3': {'cap2_foo': 'cap2_bar'}, - 'cap4': {'a': {'b': {'c': 'c'}, - 'b.c': 'b.c'}}} - utils._swift_admin_info = {'admin_cap1': {'ac1_foo': 'ac1_bar'}} - - info = utils.get_swift_info( - admin=True, disallowed_sections=['cap1.cap1_foo', 'cap3', - 'cap4.a.b.c']) - self.assertNotIn('cap3', info) - self.assertEqual(info['cap1']['cap1_moo'], 'cap1_baa') - self.assertNotIn('cap1_foo', info['cap1']) - self.assertNotIn('c', info['cap4']['a']['b']) - self.assertEqual(info['cap4']['a']['b.c'], 'b.c') - - def test_get_swift_info_with_unmatched_disallowed_sections(self): - cap1 = {'cap1_foo': 'cap1_bar', - 'cap1_moo': 'cap1_baa'} - utils._swift_info = {'swift': {'foo': 'bar'}, - 'cap1': cap1} - # expect no exceptions - info = utils.get_swift_info( - disallowed_sections=['cap2.cap1_foo', 'cap1.no_match', - 'cap1.cap1_foo.no_match.no_match']) - self.assertEqual(info['cap1'], cap1) - - class TestFileLikeIter(unittest.TestCase): def test_iter_file_iter(self): diff --git a/test/unit/proxy/controllers/test_info.py b/test/unit/proxy/controllers/test_info.py index 2317acfbe1..7109ffa8ae 100644 --- a/test/unit/proxy/controllers/test_info.py +++ b/test/unit/proxy/controllers/test_info.py @@ -20,15 +20,15 @@ from mock import Mock from swift.proxy.controllers import InfoController from swift.proxy.server import Application as ProxyApp -from swift.common import utils +from swift.common import utils, registry from swift.common.swob import Request, HTTPException class TestInfoController(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def get_controller(self, expose_info=None, disallowed_sections=None, admin_key=None): @@ -54,8 +54,8 @@ class TestInfoController(unittest.TestCase): def test_get_info(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) @@ -80,8 +80,8 @@ class TestInfoController(unittest.TestCase): def test_get_info_cors(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}, @@ -99,8 +99,8 @@ class TestInfoController(unittest.TestCase): def test_head_info(self): controller = self.get_controller(expose_info=True) - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'HEAD'}) @@ -111,9 +111,9 @@ class TestInfoController(unittest.TestCase): def test_disallow_info(self): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2']) - utils._swift_info = {'foo': {'bar': 'baz'}, - 'foo2': {'bar2': 'baz2'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}, + 'foo2': {'bar2': 'baz2'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} req = Request.blank( '/info', environ={'REQUEST_METHOD': 'GET'}) @@ -128,8 +128,8 @@ class TestInfoController(unittest.TestCase): def test_disabled_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, '') @@ -144,8 +144,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -165,8 +165,8 @@ class TestInfoController(unittest.TestCase): def test_head_admin_info(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -191,8 +191,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_method(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('HEAD', '/info', expires, 'secret-admin-key') @@ -207,8 +207,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_expires(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = 1 sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') @@ -233,8 +233,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_path(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'secret-admin-key') @@ -249,8 +249,8 @@ class TestInfoController(unittest.TestCase): def test_get_admin_info_invalid_key(self): controller = self.get_controller(expose_info=True, admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/foo', expires, 'invalid-admin-key') @@ -266,9 +266,9 @@ class TestInfoController(unittest.TestCase): controller = self.get_controller(expose_info=True, disallowed_sections=['foo2'], admin_key='secret-admin-key') - utils._swift_info = {'foo': {'bar': 'baz'}, - 'foo2': {'bar2': 'baz2'}} - utils._swift_admin_info = {'qux': {'quux': 'corge'}} + registry._swift_info = {'foo': {'bar': 'baz'}, + 'foo2': {'bar2': 'baz2'}} + registry._swift_admin_info = {'qux': {'quux': 'corge'}} expires = int(time.time() + 86400) sig = utils.get_hmac('GET', '/info', expires, 'secret-admin-key') diff --git a/test/unit/proxy/test_server.py b/test/unit/proxy/test_server.py index 1550ba55b7..7486e9faef 100644 --- a/test/unit/proxy/test_server.py +++ b/test/unit/proxy/test_server.py @@ -67,7 +67,7 @@ from swift.common.middleware import proxy_logging, versioned_writes, \ from swift.common.middleware.acl import parse_acl, format_acl from swift.common.exceptions import ChunkReadTimeout, DiskFileNotExist, \ APIVersionError, ChunkReadError -from swift.common import utils, constraints +from swift.common import utils, constraints, registry from swift.common.utils import hash_path, storage_directory, \ parse_content_type, parse_mime_headers, \ iter_multipart_mime_documents, public, mkdirs, NullLogger, md5 @@ -11274,8 +11274,8 @@ class TestProxyObjectPerformance(unittest.TestCase): StoragePolicy(3, 'bert', object_ring=FakeRing())]) class TestSwiftInfo(unittest.TestCase): def setUp(self): - utils._swift_info = {} - utils._swift_admin_info = {} + registry._swift_info = {} + registry._swift_admin_info = {} def test_registered_defaults(self): app = proxy_server.Application({}, @@ -11312,7 +11312,7 @@ class TestSwiftInfo(unittest.TestCase): # other items are added to swift info self.assertEqual(len(si), 17) - si = utils.get_swift_info()['swift'] + si = registry.get_swift_info()['swift'] # Tehse settings is by default excluded by disallowed_sections self.assertEqual(si['valid_api_versions'], constraints.VALID_API_VERSIONS)