From 157f358057c3e12158ee4ba354566c2ad5a9ecb5 Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Sun, 13 Dec 2020 12:36:17 -0600 Subject: [PATCH] [goal] Deprecate the JSON formatted policy file As per the community goal of migrating the policy file the format from JSON to YAML[1], we need to do two things: 1. Change the default value of '[oslo_policy] policy_file'' config option from 'policy.json' to 'policy.yaml' with upgrade checks. 2. Deprecate the JSON formatted policy file on the project side via warning in doc and releasenotes. Also replace policy.json to policy.yaml ref from doc and code. [1]https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html Change-Id: I1aa12bcd2638390f25d57ce8abeeec248121dc02 --- api-ref/source/v1/parameters.yaml | 2 +- doc/source/admin/stack-domain-users.rst | 4 ++-- doc/source/configuration/sample_policy.rst | 8 ++++++++ heat/api/cfn/v1/stacks.py | 2 +- heat/api/openstack/v1/actions.py | 2 +- heat/api/openstack/v1/build_info.py | 2 +- heat/api/openstack/v1/events.py | 2 +- heat/api/openstack/v1/resources.py | 2 +- heat/api/openstack/v1/services.py | 2 +- heat/api/openstack/v1/stacks.py | 2 +- heat/cmd/status.py | 10 +++------- heat/common/config.py | 5 +++++ heat/common/policy.py | 7 +++++++ heat/tests/cmd/test_status.py | 16 +++++++++++---- heat/tests/common.py | 5 ++++- heat/tests/convergence/framework/reality.py | 5 ++++- heat/tests/test_common_context.py | 2 +- heat/tests/test_common_policy.py | 3 ++- lower-constraints.txt | 12 +++++------ ...ormatted-policy-file-ac2f98b31ece0baf.yaml | 20 +++++++++++++++++++ requirements.txt | 8 ++++---- 21 files changed, 86 insertions(+), 35 deletions(-) create mode 100644 releasenotes/notes/deprecate-json-formatted-policy-file-ac2f98b31ece0baf.yaml diff --git a/api-ref/source/v1/parameters.yaml b/api-ref/source/v1/parameters.yaml index d61866794c..0a9db587c7 100644 --- a/api-ref/source/v1/parameters.yaml +++ b/api-ref/source/v1/parameters.yaml @@ -115,7 +115,7 @@ deployment_server_id_query: global_tenant: description: | Set to ``true`` to include stacks from all tenants (projects) in the stack - list. Specify policy requirements in the Orchestration ``policy.json`` + list. Specify policy requirements in the Orchestration ``policy.yaml`` file. in: query required: false diff --git a/doc/source/admin/stack-domain-users.rst b/doc/source/admin/stack-domain-users.rst index fa869d943f..4818554d2c 100644 --- a/doc/source/admin/stack-domain-users.rst +++ b/doc/source/admin/stack-domain-users.rst @@ -124,7 +124,7 @@ The following steps are run during stack creation: perspective) to the stack owners project. The users who are created in the stack domain are still assigned the ``heat_stack_user`` role, so the API surface they can access is limited through - the :file:`policy.json` file. + the :file:`policy.yaml` file. For more information, see :keystone-doc:`OpenStack Identity documentation <>`. @@ -133,7 +133,7 @@ The following steps are run during stack creation: retrieved. Details are retrieved from the database for both the stack owner's project (the default API path to the stack) and the stack domain project, subject to the - :file:`policy.json` restrictions. + :file:`policy.yaml` restrictions. This means there are now two paths that can result in the same data being retrieved through the Orchestration API. diff --git a/doc/source/configuration/sample_policy.rst b/doc/source/configuration/sample_policy.rst index 78814abe82..0b9ce0976e 100644 --- a/doc/source/configuration/sample_policy.rst +++ b/doc/source/configuration/sample_policy.rst @@ -2,6 +2,14 @@ Heat Sample Policy ================== +.. warning:: + + JSON formatted policy file is deprecated since Heat 17.0.0 (Xena). + This `oslopolicy-convert-json-to-yaml`__ tool will migrate your existing + JSON-formatted policy file to YAML in a backward-compatible way. + +.. __: https://docs.openstack.org/oslo.policy/latest/cli/oslopolicy-convert-json-to-yaml.html + The following is a sample heat policy file that has been auto-generated from default policy values in code. If you're using the default policies, then the maintenance of this file is not necessary, and it should not be copied into diff --git a/heat/api/cfn/v1/stacks.py b/heat/api/cfn/v1/stacks.py index 245c8d57d9..881c43d88b 100644 --- a/heat/api/cfn/v1/stacks.py +++ b/heat/api/cfn/v1/stacks.py @@ -49,7 +49,7 @@ class StackController(object): raise exception.HeatInvalidActionError() def _enforce(self, req, action): - """Authorize an action against the policy.json and policies in code.""" + """Authorize an action against the policy.yaml and policies in code.""" try: self.policy.enforce(req.context, action, is_registered_policy=True) except heat_exception.Forbidden: diff --git a/heat/api/openstack/v1/actions.py b/heat/api/openstack/v1/actions.py index 3a99a445e2..553c09a67f 100644 --- a/heat/api/openstack/v1/actions.py +++ b/heat/api/openstack/v1/actions.py @@ -25,7 +25,7 @@ class ActionController(object): Implements the API for stack actions """ - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'actions' diff --git a/heat/api/openstack/v1/build_info.py b/heat/api/openstack/v1/build_info.py index 2743f621b2..923d38225d 100644 --- a/heat/api/openstack/v1/build_info.py +++ b/heat/api/openstack/v1/build_info.py @@ -24,7 +24,7 @@ class BuildInfoController(object): Returns build information for current app. """ - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'build_info' diff --git a/heat/api/openstack/v1/events.py b/heat/api/openstack/v1/events.py index 87aa000a83..7c0527408d 100644 --- a/heat/api/openstack/v1/events.py +++ b/heat/api/openstack/v1/events.py @@ -83,7 +83,7 @@ class EventController(object): Implements the API actions. """ - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'events' diff --git a/heat/api/openstack/v1/resources.py b/heat/api/openstack/v1/resources.py index 199f778e40..f6d7cd8cba 100644 --- a/heat/api/openstack/v1/resources.py +++ b/heat/api/openstack/v1/resources.py @@ -74,7 +74,7 @@ class ResourceController(object): Implements the API actions. """ - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'resource' diff --git a/heat/api/openstack/v1/services.py b/heat/api/openstack/v1/services.py index c51167b7c6..4c61e2aed0 100644 --- a/heat/api/openstack/v1/services.py +++ b/heat/api/openstack/v1/services.py @@ -25,7 +25,7 @@ from heat.rpc import client as rpc_client class ServiceController(object): """WSGI controller for reporting the heat engine status in Heat v1 API.""" - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'service' diff --git a/heat/api/openstack/v1/stacks.py b/heat/api/openstack/v1/stacks.py index 1da8265099..80b8e90b59 100644 --- a/heat/api/openstack/v1/stacks.py +++ b/heat/api/openstack/v1/stacks.py @@ -183,7 +183,7 @@ class StackController(object): Implements the API actions. """ - # Define request scope (must match what is in policy.json or policies in + # Define request scope (must match what is in policy.yaml or policies in # code) REQUEST_SCOPE = 'stacks' diff --git a/heat/cmd/status.py b/heat/cmd/status.py index 3281176f1e..4bf48d017b 100644 --- a/heat/cmd/status.py +++ b/heat/cmd/status.py @@ -15,6 +15,7 @@ import sys from oslo_config import cfg +from oslo_upgradecheck import common_checks from oslo_upgradecheck import upgradecheck from heat.common.i18n import _ @@ -28,11 +29,6 @@ class Checks(upgradecheck.UpgradeCommands): and added to _upgrade_checks tuple. """ - def _check_placeholder(self): - # This is just a placeholder for upgrade checks, it should be - # removed when the actual checks are added - return upgradecheck.Result(upgradecheck.Code.SUCCESS) - # The format of the check functions is to return an # oslo_upgradecheck.upgradecheck.Result # object with the appropriate @@ -41,8 +37,8 @@ class Checks(upgradecheck.UpgradeCommands): # in the returned Result's "details" attribute. The # summary will be rolled up at the end of the check() method. _upgrade_checks = ( - # In the future there should be some real checks added here - (_('Placeholder'), _check_placeholder), + (_('Policy File JSON to YAML Migration'), + (common_checks.check_policy_json, {'conf': cfg.CONF})), ) diff --git a/heat/common/config.py b/heat/common/config.py index 1c03b6f1b8..e978fdb407 100644 --- a/heat/common/config.py +++ b/heat/common/config.py @@ -19,6 +19,7 @@ from oslo_config import cfg from oslo_db import options as oslo_db_ops from oslo_log import log as logging from oslo_middleware import cors +from oslo_policy import opts as policy_opts from osprofiler import opts as profiler from heat.common import exception @@ -596,3 +597,7 @@ def set_config_defaults(): 'DELETE', 'PATCH'] ) + # TODO(gmann): Remove setting the default value of config policy_file + # once oslo_policy change the default value to 'policy.yaml'. + # https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49 + policy_opts.set_defaults(cfg.CONF, 'policy.yaml') diff --git a/heat/common/policy.py b/heat/common/policy.py index 38971e0017..9b72e833a2 100644 --- a/heat/common/policy.py +++ b/heat/common/policy.py @@ -19,6 +19,7 @@ from oslo_config import cfg from oslo_log import log as logging +from oslo_policy import opts from oslo_policy import policy from oslo_utils import excutils @@ -33,6 +34,12 @@ LOG = logging.getLogger(__name__) DEFAULT_RULES = policy.Rules.from_dict({'default': '!'}) DEFAULT_RESOURCE_RULES = policy.Rules.from_dict({'default': '@'}) +# TODO(gmann): Remove setting the default value of config policy_file +# once oslo_policy change the default value to 'policy.yaml'. +# https://github.com/openstack/oslo.policy/blob/a626ad12fe5a3abd49d70e3e5b95589d279ab578/oslo_policy/opts.py#L49 +DEFAULT_POLICY_FILE = 'policy.yaml' +opts.set_defaults(CONF, DEFAULT_POLICY_FILE) + ENFORCER = None diff --git a/heat/tests/cmd/test_status.py b/heat/tests/cmd/test_status.py index ede3ba63b0..8f73a525da 100644 --- a/heat/tests/cmd/test_status.py +++ b/heat/tests/cmd/test_status.py @@ -12,6 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. +from unittest import mock + from oslo_upgradecheck.upgradecheck import Code from heat.cmd import status @@ -24,7 +26,13 @@ class TestUpgradeChecks(common.HeatTestCase): super(TestUpgradeChecks, self).setUp() self.cmd = status.Checks() - def test__check_placeholder(self): - check_result = self.cmd._check_placeholder() - self.assertEqual( - Code.SUCCESS, check_result.code) + @mock.patch('oslo_utils.fileutils.is_json') + def test_checks(self, mock_util): + mock_util.return_value = False + for name, func in self.cmd._upgrade_checks: + if isinstance(func, tuple): + func_name, kwargs = func + result = func_name(self, **kwargs) + else: + result = func(self) + self.assertEqual(Code.SUCCESS, result.code) diff --git a/heat/tests/common.py b/heat/tests/common.py index b3c1fdc995..baafe94be7 100644 --- a/heat/tests/common.py +++ b/heat/tests/common.py @@ -81,7 +81,7 @@ class HeatTestCase(testscenarios.WithScenarios, testtools.TestCase, FakeLogMixin): def setUp(self, mock_keystone=True, mock_resource_policy=True, - quieten_logging=True): + quieten_logging=True, mock_find_file=True): super(HeatTestCase, self).setUp() self.setup_logging(quieten=quieten_logging) self.warnings = self.useFixture(fixtures.WarningsCapture()) @@ -126,6 +126,9 @@ class HeatTestCase(testscenarios.WithScenarios, '/etc/heat/templates', templ_path) + if mock_find_file: + self.mock_find_file = self.patchobject( + cfg.ConfigOpts, 'find_file') if mock_keystone: self.stub_keystoneclient() if mock_resource_policy: diff --git a/heat/tests/convergence/framework/reality.py b/heat/tests/convergence/framework/reality.py index dcd7a79c7a..055782b389 100644 --- a/heat/tests/convergence/framework/reality.py +++ b/heat/tests/convergence/framework/reality.py @@ -11,6 +11,8 @@ # License for the specific language governing permissions and limitations # under the License. +from unittest import mock + from heat.common import exception from heat.db.sqlalchemy import api as db_api from heat.tests import utils @@ -49,4 +51,5 @@ class RealityStore(object): return res_data.value -reality = RealityStore() +with mock.patch("oslo_config.cfg.ConfigOpts.find_file"): + reality = RealityStore() diff --git a/heat/tests/test_common_context.py b/heat/tests/test_common_context.py index 01604c6652..31f5bb736f 100644 --- a/heat/tests/test_common_context.py +++ b/heat/tests/test_common_context.py @@ -391,7 +391,7 @@ class RequestContextMiddlewareTest(common.HeatTestCase): )] def setUp(self): - super(RequestContextMiddlewareTest, self).setUp() + super(RequestContextMiddlewareTest, self).setUp(mock_find_file=False) self.fixture = self.useFixture(config_fixture.Config()) self.fixture.conf(args=['--config-dir', policy_path]) policy_opts.set_defaults(cfg.CONF, 'check_admin.json') diff --git a/heat/tests/test_common_policy.py b/heat/tests/test_common_policy.py index eb2753c4ac..99c23c436d 100644 --- a/heat/tests/test_common_policy.py +++ b/heat/tests/test_common_policy.py @@ -33,7 +33,8 @@ policy_path = os.path.dirname(os.path.realpath(__file__)) + "/policy/" class TestPolicyEnforcer(common.HeatTestCase): def setUp(self): - super(TestPolicyEnforcer, self).setUp(mock_resource_policy=False) + super(TestPolicyEnforcer, self).setUp( + mock_resource_policy=False, mock_find_file=False) self.fixture = self.useFixture(config_fixture.Config()) self.fixture.conf(args=['--config-dir', policy_path]) diff --git a/lower-constraints.txt b/lower-constraints.txt index 47b9cac968..25c90e155d 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -66,7 +66,7 @@ os-service-types==1.2.0 osc-lib==1.10.0 oslo.cache==1.26.0 oslo.concurrency==3.26.0 -oslo.config==6.0.0 +oslo.config==6.8.0 oslo.context==2.22.0 oslo.db==6.0.0 oslo.i18n==3.20.0 @@ -77,12 +77,12 @@ oslo.policy==3.7.0 oslo.reports==1.18.0 oslo.serialization==2.25.0 oslo.service==1.24.0 -oslo.upgradecheck==0.1.0 -oslo.utils==3.40.0 +oslo.upgradecheck==1.3.0 +oslo.utils==4.5.0 oslo.versionedobjects==1.31.2 oslotest==3.2.0 osprofiler==1.4.0 -packaging==17.1 +packaging==20.4 paramiko==2.7.1 Paste==2.0.3 PasteDeploy==1.5.0 @@ -132,9 +132,9 @@ python-zunclient==3.4.0 pytz==2013.6 PyYAML==5.1 repoze.lru==0.7 -requests==2.14.2 +requests==2.23.0 requestsexceptions==1.4.0 -rfc3986==1.1.0 +rfc3986==1.2.0 Routes==2.3.1 simplejson==3.13.2 six==1.10.0 diff --git a/releasenotes/notes/deprecate-json-formatted-policy-file-ac2f98b31ece0baf.yaml b/releasenotes/notes/deprecate-json-formatted-policy-file-ac2f98b31ece0baf.yaml new file mode 100644 index 0000000000..00519ae7be --- /dev/null +++ b/releasenotes/notes/deprecate-json-formatted-policy-file-ac2f98b31ece0baf.yaml @@ -0,0 +1,20 @@ +--- +upgrade: + - | + The default value of ``[oslo_policy] policy_file`` config option has + been changed from ``policy.json`` to ``policy.yaml``. + Operators who are utilizing customized or previously generated + static policy JSON files (which are not needed by default), should + generate new policy files or convert them in YAML format. Use the + `oslopolicy-convert-json-to-yaml + `_ + tool to convert a JSON to YAML formatted policy file in + backward compatible way. +deprecations: + - | + Use of JSON policy files was deprecated by the ``oslo.policy`` library + during the Victoria development cycle. As a result, this deprecation is + being noted in the Xena cycle with an anticipated future removal of support + by ``oslo.policy``. As such operators will need to convert to YAML policy + files. Please see the upgrade notes for details on migration of any + custom policy files. diff --git a/requirements.txt b/requirements.txt index 46ef806465..2f54c37aaa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ netaddr>=0.7.18 # BSD neutron-lib>=1.14.0 # Apache-2.0 openstacksdk>=0.11.2 # Apache-2.0 oslo.cache>=1.26.0 # Apache-2.0 -oslo.config>=6.0.0 # Apache-2.0 +oslo.config>=6.8.0 # Apache-2.0 oslo.concurrency>=3.26.0 # Apache-2.0 oslo.context>=2.22.0 # Apache-2.0 oslo.db>=6.0.0 # Apache-2.0 @@ -28,8 +28,8 @@ oslo.policy>=3.7.0 # Apache-2.0 oslo.reports>=1.18.0 # Apache-2.0 oslo.serialization>=2.25.0 # Apache-2.0 oslo.service!=1.28.1,>=1.24.0 # Apache-2.0 -oslo.upgradecheck>=0.1.0 # Apache-2.0 -oslo.utils>=3.40.0 # Apache-2.0 +oslo.upgradecheck>=1.3.0 # Apache-2.0 +oslo.utils>=4.5.0 # Apache-2.0 osprofiler>=1.4.0 # Apache-2.0 oslo.versionedobjects>=1.31.2 # Apache-2.0 PasteDeploy>=1.5.0 # MIT @@ -59,7 +59,7 @@ python-zaqarclient>=1.3.0 # Apache-2.0 python-zunclient>=3.4.0 # Apache-2.0 pytz>=2013.6 # MIT PyYAML>=5.1 # MIT -requests>=2.14.2 # Apache-2.0 +requests>=2.23.0 # Apache-2.0 tenacity>=6.1.0 # Apache-2.0 Routes>=2.3.1 # MIT SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT