diff --git a/collector/collector/api/app.py b/collector/collector/api/app.py index 61ac9cc..665d7c7 100644 --- a/collector/collector/api/app.py +++ b/collector/collector/api/app.py @@ -20,6 +20,8 @@ import flask_sqlalchemy import os from sqlalchemy.exc import IntegrityError +from collector.api.config import index_filtering_rules + app = Flask(__name__) @@ -28,6 +30,10 @@ app = Flask(__name__) app.config['JSONSCHEMA_DIR'] = os.path.join(app.root_path, 'schemas') flask_jsonschema.JsonSchema(app) +# We should rebuild packages based keys in the FILTERING_RULES. +# Sorted tuples built from packages lists are used as keys. +index_filtering_rules(app) + db = flask_sqlalchemy.SQLAlchemy(app) diff --git a/collector/collector/api/config.py b/collector/collector/api/config.py index 8078962..e45bef3 100644 --- a/collector/collector/api/config.py +++ b/collector/collector/api/config.py @@ -29,7 +29,17 @@ class Production(object): # Filtration is performed by release and build_id from installation # info fuel_release data. # - # Structure of FILTERING_RULES: {release: {build_id: from_dt}} + # Structure of FILTERING_RULES for releases < 8.0: + # {release: {build_id: from_dt}} + # Structure of FILTERING_RULES for releases >= 8.0: + # {release: {('fuel-nailgun-8.0.0-1.mos8212.noarch', + # 'fuel-library8.0-8.0.0-1.mos7718.noarch'): from_dt}} + # + # PAY ATTENTION: you must use tuples as indexes in the FILTERING_RULES + # + # If packages and build_id are set simultaneously both conditions + # will be checked. Installation info will be filtered if any of build_id + # or packages filtered. # # Example of FILTERING_RULES: # {'6.1': @@ -42,7 +52,9 @@ class Production(object): # '2015-04-13_06-18-10': None # }, # '6.1.1': {}, # All builds of 6.1.1 filtered - # '7.0': None # All builds of 7.0 not filtered + # '7.0': None, # All builds of 7.0 not filtered + # '8.0': {('fuel-nailgun-8.0.0-1.mos8212.noarch',): '2016-02-01T23:00:18', + # ('fuel-nailgun-8.0.0-2.mos9345.noarch',): '2016-02-10',} # } # # If you don't need any filtration, please set FILTERING_RULES = None @@ -62,3 +74,39 @@ class Testing(Production): SQLALCHEMY_DATABASE_URI = \ 'postgresql://collector:collector@localhost/collector' SQLALCHEMY_ECHO = True + + +def normalize_build_info(build_info): + """Prepare build info for searching in the filtering rules + + :param build_info: build_id or packages list + :return: build_id or ordered tuple of packages + """ + if isinstance(build_info, (list, tuple)): + return tuple(sorted(build_info)) + + return build_info + + +def index_filtering_rules(app): + """Rebuilds packages based keys in FILTERING_RULES. + + For accurate search we need to have sorted packages tuples as indexes + in the FILTERING_RULES. + + :param app: Flask application + """ + + filtering_rules = app.config.get('FILTERING_RULES') + if not filtering_rules: + return + + for rules in filtering_rules.itervalues(): + if not rules: + continue + + for build_info, from_dt in rules.iteritems(): + normalized_info = normalize_build_info(build_info) + if normalized_info not in rules: + rules[normalized_info] = from_dt + rules.pop(build_info) diff --git a/collector/collector/api/resources/installation_structure.py b/collector/collector/api/resources/installation_structure.py index ed7565d..59d185c 100644 --- a/collector/collector/api/resources/installation_structure.py +++ b/collector/collector/api/resources/installation_structure.py @@ -26,6 +26,7 @@ from collector.api.app import db from collector.api.common.util import db_transaction from collector.api.common.util import exec_time from collector.api.common.util import handle_response +from collector.api.config import normalize_build_info from collector.api.db.model import InstallationStructure @@ -57,6 +58,40 @@ def post(): return status_code, {'status': 'ok'} +def _is_filtered_by_build_info(build_info, filtering_rules): + """Calculates is build_info should be filtered or not. + + :param build_info: build_id or packages from the + installation info structure + :param filtering_rules: filtering rules for release + """ + + # We don't have 'build_id' in structure since release 8.0 + # and 'packages' before 8.0 + if build_info is None: + return False + + build_info = normalize_build_info(build_info) + + # build info not found + if build_info not in filtering_rules: + return True + + build_rules = filtering_rules.get(build_info) + + # No from_dt specified + if build_rules is None: + return False + + # from_dt in the past + from_dt = parser.parse(build_rules) + cur_dt = datetime.utcnow() + if from_dt <= cur_dt: + return False + + return True + + def _is_filtered(structure): """Checks is structure should be filtered or not. For filtering uses rules defined at app.config['FILTERING_RULES'] @@ -71,32 +106,23 @@ def _is_filtered(structure): # Extracting data from structure fuel_release = structure.get('fuel_release', {}) release = fuel_release.get('release') - build_id = structure.get('fuel_release', {}).get('build_id') + build_id = fuel_release.get('build_id') + packages = structure.get('fuel_packages') # Release not in rules if release not in rules: return True - release_rules = rules.get(release) + filtering_rules = rules.get(release) - # No build_ids specified - if release_rules is None: + # Filtering rules doesn't specified + if filtering_rules is None: return False - # build_id not found in list - if build_id not in release_rules: - return True + filtered_by_build_id = _is_filtered_by_build_info( + build_id, filtering_rules) - build_rules = release_rules.get(build_id) + filtered_by_packages = _is_filtered_by_build_info( + packages, filtering_rules) - # No from_dt specified - if build_rules is None: - return False - - # from_dt in the past - from_dt = parser.parse(build_rules) - cur_dt = datetime.utcnow() - if from_dt <= cur_dt: - return False - - return True + return filtered_by_build_id or filtered_by_packages diff --git a/collector/collector/api/schemas/installation_structure.json b/collector/collector/api/schemas/installation_structure.json index 7abb8ef..e7ab6e9 100644 --- a/collector/collector/api/schemas/installation_structure.json +++ b/collector/collector/api/schemas/installation_structure.json @@ -34,6 +34,10 @@ }, "required": ["release", "api", "feature_groups"] }, + "fuel_packages": { + "type": "array", + "items": {"type": "string"} + }, "clusters": { "type": "array", "items": { diff --git a/collector/collector/test/resources/test_config.py b/collector/collector/test/resources/test_config.py new file mode 100644 index 0000000..ae72fb8 --- /dev/null +++ b/collector/collector/test/resources/test_config.py @@ -0,0 +1,72 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import mock + +from collector.test.base import BaseTest + +from collector.api.app import app +from collector.api.config import index_filtering_rules +from collector.api.config import normalize_build_info + + +class TestConfig(BaseTest): + + def test_filtering_rules_indexed(self): + build_id = 'build_id_0' + filtering_rules = {(3, 2, 1): None, (2, 1): '2016-01-26', + 'build_id': build_id} + release = '8.0' + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: filtering_rules.copy()}} + ): + # Checking filtering rules before sorting + actual_filtering_rules = app.config.get('FILTERING_RULES')[release] + for packages, from_dt in filtering_rules.iteritems(): + if isinstance(packages, tuple): + self.assertNotIn(tuple(sorted(packages)), + actual_filtering_rules) + self.assertIn(packages, actual_filtering_rules) + + # Checking filtering rules after sorting + index_filtering_rules(app) + actual_filtering_rules = app.config.get('FILTERING_RULES')[release] + for build_info in filtering_rules.iterkeys(): + self.assertIn(normalize_build_info(build_info), + actual_filtering_rules) + + def test_index_filtering_rules_idempotent(self): + packages = ('a', 'b', 'c') + release = '8.0' + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {packages: None}}} + ): + index_filtering_rules(app) + expected_rules = copy.copy( + app.config.get('FILTERING_RULES')[release]) + index_filtering_rules(app) + actual_rules = copy.copy( + app.config.get('FILTERING_RULES')[release]) + self.assertIn(normalize_build_info(packages), actual_rules) + self.assertEqual(expected_rules, actual_rules) + + def test_index_filtering_rules(self): + build_id = '2016-xxx.yyy' + self.assertEqual(build_id, normalize_build_info(build_id)) + packages = ['z', 'x', 'a'] + self.assertEqual(tuple(sorted(packages)), + normalize_build_info(packages)) diff --git a/collector/collector/test/resources/test_installation_structure.py b/collector/collector/test/resources/test_installation_structure.py index e4d37ce..6ce1ac1 100644 --- a/collector/collector/test/resources/test_installation_structure.py +++ b/collector/collector/test/resources/test_installation_structure.py @@ -21,6 +21,8 @@ from collector.api.app import app from collector.api.app import db from collector.api.db.model import InstallationStructure from collector.api.resources.installation_structure import _is_filtered +from collector.api.resources.installation_structure import \ + _is_filtered_by_build_info class TestInstallationStructure(DbTest): @@ -229,7 +231,7 @@ class TestInstallationStructure(DbTest): ) self.check_response_ok(resp, codes=(200, 201)) - def test_is_not_filtered(self): + def test_is_not_filtered_by_build_id(self): release = '6.1' build_id = '2014-10-30_14-51-06' struct = { @@ -264,21 +266,93 @@ class TestInstallationStructure(DbTest): {'FILTERING_RULES': {release: {build_id: dt_str}}}): self.assertFalse(_is_filtered(struct)) - def test_is_filtered(self): - release = '6.1_filtered' - build_id = '2014-10-30_14-51-06_filtered' + def test_is_not_filtered_by_packages(self): + release = '8.0' + packages = ['z_filtered', 'a_filtered'] + sorted_packages = tuple(sorted(packages)) + struct = { + 'fuel_release': { + 'release': release + }, + 'fuel_packages': packages + } + + # Have 'packages', no from_dt + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: None}}} + ): + self.assertFalse(_is_filtered(struct)) + + # Have 'packages', from_dt in past + dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) + dt_str = dt.isoformat() + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: dt_str}}}): + self.assertFalse(_is_filtered(struct)) + + def test_is_not_filtered_by_packages_and_build_id(self): + release = '8.0' + packages = ['z_filtered', 'a_filtered'] + sorted_packages = tuple(sorted(packages)) + build_id = '2016-01-26' struct = { 'fuel_release': { 'release': release, 'build_id': build_id - } + }, + 'fuel_packages': packages } - # release not found in rules + # No rules + with mock.patch.dict(app.config, {'FILTERING_RULES': None}): + self.assertFalse(_is_filtered(struct)) + + with mock.patch.dict(app.config, {'FILTERING_RULES': {}}): + self.assertFalse(_is_filtered(struct)) + + # No build info + with mock.patch.dict(app.config, + {'FILTERING_RULES': {release: None}}): + self.assertFalse(_is_filtered(struct)) + + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: None, + build_id: None}}} + ): + self.assertFalse(_is_filtered(struct)) + + # Have build info, from_dt in past + dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) + dt_str = dt.isoformat() + + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: dt_str, + build_id: dt_str}}} + ): + self.assertFalse(_is_filtered(struct)) + + def test_is_filtered_by_packages_and_build_id(self): + release = '8.0_filtered' + packages = ['z_filtered', 'a_filtered'] + build_id = '2016-01-26_filtered' + sorted_packages = tuple(sorted(packages)) + struct = { + 'fuel_release': { + 'release': release, + 'build_id': build_id + }, + 'fuel_packages': packages + } + + # build_info not found in rules with mock.patch.dict(app.config, {'FILTERING_RULES': {'xx': None}}): self.assertTrue(_is_filtered(struct)) - # build_id not found in rules + # build_info not found in rules with mock.patch.dict(app.config, {'FILTERING_RULES': {release: {}}}): self.assertTrue(_is_filtered(struct)) @@ -286,11 +360,23 @@ class TestInstallationStructure(DbTest): # from_dt in future dt = datetime.datetime.utcnow() + datetime.timedelta(days=1) dt_str = dt.isoformat() + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: dt_str}}}): + self.assertTrue(_is_filtered(struct)) + with mock.patch.dict( app.config, {'FILTERING_RULES': {release: {build_id: dt_str}}}): self.assertTrue(_is_filtered(struct)) + with mock.patch.dict( + app.config, + {'FILTERING_RULES': {release: {sorted_packages: dt_str, + build_id: dt_str}}} + ): + self.assertTrue(_is_filtered(struct)) + def test_is_filtered_check_from_dt_formats(self): release = 'release_dt_format' build_id = 'build_id_dt_format' @@ -361,3 +447,63 @@ class TestInstallationStructure(DbTest): obj = db.session.query(InstallationStructure).filter( InstallationStructure.master_node_uid == master_node_uid).one() self.assertTrue(obj.is_filtered) + + def test_is_filtered_by_build_info_by_build_id(self): + build_id = 'build_01' + + # Checking 'build_id' not in filtering rules + filtering_rules = {} + self.assertTrue(_is_filtered_by_build_info(build_id, filtering_rules)) + + # Checking filtering by time + from_dt = datetime.datetime.utcnow() + datetime.timedelta(days=2) + from_dt_str = from_dt.isoformat() + filtering_rules = {build_id: from_dt_str} + self.assertTrue(_is_filtered_by_build_info(build_id, filtering_rules)) + + def test_is_not_filtered_by_build_info_by_build_id(self): + # Checking not filtered if 'build_id' not defined + self.assertFalse(_is_filtered_by_build_info(None, {})) + + # Checking filtering by time + build_id = 'build_02' + from_dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) + from_dt_str = from_dt.isoformat() + filtering_rules = {build_id: from_dt_str} + self.assertFalse(_is_filtered_by_build_info(build_id, filtering_rules)) + + def test_is_filtered_by_build_info_by_packages(self): + packages = ('z', 'a', 'b') + + # Checking 'packages' not in filtering rules + filtering_rules = {} + self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules)) + + # Checking 'packages' doesn't match + filtering_rules = {packages[:-1]: None} + self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules)) + + filtering_rules = {tuple(sorted(packages))[:-1]: None} + self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules)) + + # Checking not sorted 'packages' is filtered + filtering_rules = {packages: None} + self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules)) + + # Checking filtering by time + from_dt = datetime.datetime.utcnow() + datetime.timedelta(days=2) + from_dt_str = from_dt.isoformat() + filtering_rules = {packages: from_dt_str} + self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules)) + + def test_is_not_filtered_by_build_info_by_packages(self): + # Checking not filtered if 'packages' not defined + self.assertFalse(_is_filtered_by_build_info(None, {})) + + # Checking filtering by time + packages = ('z', 'a', 'b') + from_dt = datetime.datetime.utcnow() - datetime.timedelta(days=1) + from_dt_str = from_dt.isoformat() + filtering_rules = {tuple(sorted(packages)): from_dt_str} + self.assertFalse(_is_filtered_by_build_info( + packages, filtering_rules))