From c70506727800d99fbbe8133f0bf0ccb1e70d690a Mon Sep 17 00:00:00 2001 From: Ghanshyam Mann Date: Thu, 19 Nov 2020 18:27:32 -0600 Subject: [PATCH] Add common checks for json to yaml policy file migration As part of community goal[1] for migrating the policy file from JSON format to YAML we need to add upgrade check to detect any JSON formatted file. That check needs to be done in each service side which are deprecating the JSON format. So to avoid duplicate code, this commit add that check in oslo_upgradechecks and that can be enabled from service side during they migrate to YAML format. Service will enable it like below: _upgrade_checks = ( (_('Policy File JSON to YAML Migration'), (common_checks.check_policy_json, {'conf': CONF})), ) Also modify the check() method to accept the **kwargs for checks. [1] https://governance.openstack.org/tc/goals/selected/wallaby/migrate-policy-format-from-json-to-yaml.html Change-Id: Ifa71c20879c964f794a40b981744d95db4ab1602 --- lower-constraints.txt | 2 + oslo_upgradecheck/common_checks.py | 36 ++++++ oslo_upgradecheck/tests/test_common_checks.py | 103 ++++++++++++++++++ oslo_upgradecheck/upgradecheck.py | 6 +- requirements.txt | 1 + test-requirements.txt | 1 + 6 files changed, 148 insertions(+), 1 deletion(-) create mode 100644 oslo_upgradecheck/common_checks.py create mode 100644 oslo_upgradecheck/tests/test_common_checks.py diff --git a/lower-constraints.txt b/lower-constraints.txt index 6224498..5ded84f 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -4,3 +4,5 @@ enum34==1.0.4 PrettyTable==0.7.1 oslotest==1.5.1 stestr==2.0.0 +oslo.serialization==2.21.1 +oslo.utils==4.5.0 diff --git a/oslo_upgradecheck/common_checks.py b/oslo_upgradecheck/common_checks.py new file mode 100644 index 0000000..1881980 --- /dev/null +++ b/oslo_upgradecheck/common_checks.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_utils import fileutils + +from oslo_upgradecheck import upgradecheck + +""" +Common checks which can be used by multiple services. +""" + + +def check_policy_json(self, conf): + "Checks to see if policy file is JSON-formatted policy file." + msg = ("Your policy file is JSON-formatted which is " + "deprecated. You need to switch to YAML-formatted file. " + "Use the ``oslopolicy-convert-json-to-yaml`` " + "tool to convert the existing JSON-formatted files to " + "YAML in a backwards-compatible manner: " + "https://docs.openstack.org/oslo.policy/" + "latest/cli/oslopolicy-convert-json-to-yaml.html.") + status = upgradecheck.Result(upgradecheck.Code.SUCCESS) + # Check if policy file exist and is JSON-formatted. + policy_path = conf.find_file(conf.oslo_policy.policy_file) + if policy_path and fileutils.is_json(policy_path): + status = upgradecheck.Result(upgradecheck.Code.FAILURE, msg) + return status diff --git a/oslo_upgradecheck/tests/test_common_checks.py b/oslo_upgradecheck/tests/test_common_checks.py new file mode 100644 index 0000000..c5c54a2 --- /dev/null +++ b/oslo_upgradecheck/tests/test_common_checks.py @@ -0,0 +1,103 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import fixtures +import os.path +import tempfile +import yaml + +from oslo_config import cfg +from oslo_config import fixture as config +from oslo_serialization import jsonutils +from oslotest import base + +from oslo_upgradecheck import common_checks +from oslo_upgradecheck import upgradecheck + + +class TestUpgradeCheckPolicyJSON(base.BaseTestCase): + + def setUp(self): + super(TestUpgradeCheckPolicyJSON, self).setUp() + conf_fixture = self.useFixture(config.Config()) + conf_fixture.load_raw_values() + self.conf = conf_fixture.conf + policy_file_opt = cfg.StrOpt('policy_file', + default='policy.json', + help='policy file') + self.conf.register_opt(policy_file_opt, group="oslo_policy") + + self.cmd = upgradecheck.UpgradeCommands() + self.cmd._upgrade_checks = (('Policy File JSON to YAML Migration', + (common_checks.check_policy_json, + {'conf': self.conf})),) + + self.data = { + 'rule_admin': 'True', + 'rule_admin2': 'is_admin:True' + } + self.temp_dir = self.useFixture(fixtures.TempDir()) + fd, self.json_file = tempfile.mkstemp(dir=self.temp_dir.path) + fd, self.yaml_file = tempfile.mkstemp(dir=self.temp_dir.path) + with open(self.json_file, 'w') as fh: + jsonutils.dump(self.data, fh) + with open(self.yaml_file, 'w') as fh: + yaml.dump(self.data, fh) + + original_search_dirs = cfg._search_dirs + + def fake_search_dirs(dirs, name): + dirs.append(self.temp_dir.path) + return original_search_dirs(dirs, name) + + mock_search_dir = self.useFixture( + fixtures.MockPatch('oslo_config.cfg._search_dirs')).mock + mock_search_dir.side_effect = fake_search_dirs + + def test_policy_json_file_fail_upgrade(self): + # Test with policy json file full path set in config. + self.conf.set_override('policy_file', self.json_file, + group="oslo_policy") + self.assertEqual(upgradecheck.Code.FAILURE, + self.cmd.check()) + + def test_policy_yaml_file_pass_upgrade(self): + # Test with full policy yaml file path set in config. + self.conf.set_override('policy_file', self.yaml_file, + group="oslo_policy") + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_no_policy_file_pass_upgrade(self): + # Test with no policy file exist, means use policy from code. + self.conf.set_override('policy_file', 'non_exist_file', + group="oslo_policy") + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_default_policy_yaml_file_pass_upgrade(self): + self.conf.set_override('policy_file', 'policy.yaml', + group="oslo_policy") + tmpfilename = os.path.join(self.temp_dir.path, 'policy.yaml') + with open(tmpfilename, 'w') as fh: + yaml.dump(self.data, fh) + self.assertEqual(upgradecheck.Code.SUCCESS, + self.cmd.check()) + + def test_old_default_policy_json_file_fail_upgrade(self): + self.conf.set_override('policy_file', 'policy.json', + group="oslo_policy") + tmpfilename = os.path.join(self.temp_dir.path, 'policy.json') + with open(tmpfilename, 'w') as fh: + jsonutils.dump(self.data, fh) + self.assertEqual(upgradecheck.Code.FAILURE, + self.cmd.check()) diff --git a/oslo_upgradecheck/upgradecheck.py b/oslo_upgradecheck/upgradecheck.py index 744413d..62c9a68 100644 --- a/oslo_upgradecheck/upgradecheck.py +++ b/oslo_upgradecheck/upgradecheck.py @@ -99,7 +99,11 @@ class UpgradeCommands(object): # This is a list if 2-item tuples for the check name and it's results. check_results = [] for name, func in self._upgrade_checks: - result = func(self) + if isinstance(func, tuple): + func_name, kwargs = func + result = func_name(self, **kwargs) + else: + result = func(self) # store the result of the check for the summary table check_results.append((name, result)) # we want to end up with the highest level code of all checks diff --git a/requirements.txt b/requirements.txt index a843587..4b13697 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ oslo.config>=5.2.0 # Apache-2.0 oslo.i18n>=3.15.3 # Apache-2.0 PrettyTable<0.8,>=0.7.1 # BSD +oslo.utils>=4.5.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index fdc4461..ccb9645 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -6,3 +6,4 @@ hacking>=3.0,<3.1.0 # Apache-2.0 oslotest>=1.5.1 stestr>=2.0.0 pre-commit>=2.6.0 # MIT +oslo.serialization>=2.21.1 # Apache-2.0