Filtering inst info by packages list implemented
From release 8.0 Fuel sends fuel_packages list as build marker instead fuel_release->build_id. Now we are filtering installation info by build_id and fuel_packages. Change-Id: I798ad8261e9982e71afb1c53e5ce39a67b9fe097 Closes-Bug: #1533660 Closes-Bug: #1515579
This commit is contained in:
Alexander Kislitsky
parent
7fccde7be7
commit
4203f56d39
|
|
@ -20,6 +20,8 @@ import flask_sqlalchemy
|
|||
import os
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from collector.api.config import index_filtering_rules
|
||||
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
|
|
@ -28,6 +30,10 @@ app = Flask(__name__)
|
|||
app.config['JSONSCHEMA_DIR'] = os.path.join(app.root_path, 'schemas')
|
||||
flask_jsonschema.JsonSchema(app)
|
||||
|
||||
# We should rebuild packages based keys in the FILTERING_RULES.
|
||||
# Sorted tuples built from packages lists are used as keys.
|
||||
index_filtering_rules(app)
|
||||
|
||||
db = flask_sqlalchemy.SQLAlchemy(app)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,17 @@ class Production(object):
|
|||
# Filtration is performed by release and build_id from installation
|
||||
# info fuel_release data.
|
||||
#
|
||||
# Structure of FILTERING_RULES: {release: {build_id: from_dt}}
|
||||
# Structure of FILTERING_RULES for releases < 8.0:
|
||||
# {release: {build_id: from_dt}}
|
||||
# Structure of FILTERING_RULES for releases >= 8.0:
|
||||
# {release: {('fuel-nailgun-8.0.0-1.mos8212.noarch',
|
||||
# 'fuel-library8.0-8.0.0-1.mos7718.noarch'): from_dt}}
|
||||
#
|
||||
# PAY ATTENTION: you must use tuples as indexes in the FILTERING_RULES
|
||||
#
|
||||
# If packages and build_id are set simultaneously both conditions
|
||||
# will be checked. Installation info will be filtered if any of build_id
|
||||
# or packages filtered.
|
||||
#
|
||||
# Example of FILTERING_RULES:
|
||||
# {'6.1':
|
||||
|
|
@ -42,7 +52,9 @@ class Production(object):
|
|||
# '2015-04-13_06-18-10': None
|
||||
# },
|
||||
# '6.1.1': {}, # All builds of 6.1.1 filtered
|
||||
# '7.0': None # All builds of 7.0 not filtered
|
||||
# '7.0': None, # All builds of 7.0 not filtered
|
||||
# '8.0': {('fuel-nailgun-8.0.0-1.mos8212.noarch',): '2016-02-01T23:00:18',
|
||||
# ('fuel-nailgun-8.0.0-2.mos9345.noarch',): '2016-02-10',}
|
||||
# }
|
||||
#
|
||||
# If you don't need any filtration, please set FILTERING_RULES = None
|
||||
|
|
@ -62,3 +74,39 @@ class Testing(Production):
|
|||
SQLALCHEMY_DATABASE_URI = \
|
||||
'postgresql://collector:collector@localhost/collector'
|
||||
SQLALCHEMY_ECHO = True
|
||||
|
||||
|
||||
def normalize_build_info(build_info):
|
||||
"""Prepare build info for searching in the filtering rules
|
||||
|
||||
:param build_info: build_id or packages list
|
||||
:return: build_id or ordered tuple of packages
|
||||
"""
|
||||
if isinstance(build_info, (list, tuple)):
|
||||
return tuple(sorted(build_info))
|
||||
|
||||
return build_info
|
||||
|
||||
|
||||
def index_filtering_rules(app):
|
||||
"""Rebuilds packages based keys in FILTERING_RULES.
|
||||
|
||||
For accurate search we need to have sorted packages tuples as indexes
|
||||
in the FILTERING_RULES.
|
||||
|
||||
:param app: Flask application
|
||||
"""
|
||||
|
||||
filtering_rules = app.config.get('FILTERING_RULES')
|
||||
if not filtering_rules:
|
||||
return
|
||||
|
||||
for rules in filtering_rules.itervalues():
|
||||
if not rules:
|
||||
continue
|
||||
|
||||
for build_info, from_dt in rules.iteritems():
|
||||
normalized_info = normalize_build_info(build_info)
|
||||
if normalized_info not in rules:
|
||||
rules[normalized_info] = from_dt
|
||||
rules.pop(build_info)
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ from collector.api.app import db
|
|||
from collector.api.common.util import db_transaction
|
||||
from collector.api.common.util import exec_time
|
||||
from collector.api.common.util import handle_response
|
||||
from collector.api.config import normalize_build_info
|
||||
from collector.api.db.model import InstallationStructure
|
||||
|
||||
|
||||
|
|
@ -57,6 +58,40 @@ def post():
|
|||
return status_code, {'status': 'ok'}
|
||||
|
||||
|
||||
def _is_filtered_by_build_info(build_info, filtering_rules):
|
||||
"""Calculates is build_info should be filtered or not.
|
||||
|
||||
:param build_info: build_id or packages from the
|
||||
installation info structure
|
||||
:param filtering_rules: filtering rules for release
|
||||
"""
|
||||
|
||||
# We don't have 'build_id' in structure since release 8.0
|
||||
# and 'packages' before 8.0
|
||||
if build_info is None:
|
||||
return False
|
||||
|
||||
build_info = normalize_build_info(build_info)
|
||||
|
||||
# build info not found
|
||||
if build_info not in filtering_rules:
|
||||
return True
|
||||
|
||||
build_rules = filtering_rules.get(build_info)
|
||||
|
||||
# No from_dt specified
|
||||
if build_rules is None:
|
||||
return False
|
||||
|
||||
# from_dt in the past
|
||||
from_dt = parser.parse(build_rules)
|
||||
cur_dt = datetime.utcnow()
|
||||
if from_dt <= cur_dt:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _is_filtered(structure):
|
||||
"""Checks is structure should be filtered or not.
|
||||
For filtering uses rules defined at app.config['FILTERING_RULES']
|
||||
|
|
@ -71,32 +106,23 @@ def _is_filtered(structure):
|
|||
# Extracting data from structure
|
||||
fuel_release = structure.get('fuel_release', {})
|
||||
release = fuel_release.get('release')
|
||||
build_id = structure.get('fuel_release', {}).get('build_id')
|
||||
build_id = fuel_release.get('build_id')
|
||||
packages = structure.get('fuel_packages')
|
||||
|
||||
# Release not in rules
|
||||
if release not in rules:
|
||||
return True
|
||||
|
||||
release_rules = rules.get(release)
|
||||
filtering_rules = rules.get(release)
|
||||
|
||||
# No build_ids specified
|
||||
if release_rules is None:
|
||||
# Filtering rules doesn't specified
|
||||
if filtering_rules is None:
|
||||
return False
|
||||
|
||||
# build_id not found in list
|
||||
if build_id not in release_rules:
|
||||
return True
|
||||
filtered_by_build_id = _is_filtered_by_build_info(
|
||||
build_id, filtering_rules)
|
||||
|
||||
build_rules = release_rules.get(build_id)
|
||||
filtered_by_packages = _is_filtered_by_build_info(
|
||||
packages, filtering_rules)
|
||||
|
||||
# No from_dt specified
|
||||
if build_rules is None:
|
||||
return False
|
||||
|
||||
# from_dt in the past
|
||||
from_dt = parser.parse(build_rules)
|
||||
cur_dt = datetime.utcnow()
|
||||
if from_dt <= cur_dt:
|
||||
return False
|
||||
|
||||
return True
|
||||
return filtered_by_build_id or filtered_by_packages
|
||||
|
|
|
|||
|
|
@ -34,6 +34,10 @@
|
|||
},
|
||||
"required": ["release", "api", "feature_groups"]
|
||||
},
|
||||
"fuel_packages": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"}
|
||||
},
|
||||
"clusters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import copy
|
||||
import mock
|
||||
|
||||
from collector.test.base import BaseTest
|
||||
|
||||
from collector.api.app import app
|
||||
from collector.api.config import index_filtering_rules
|
||||
from collector.api.config import normalize_build_info
|
||||
|
||||
|
||||
class TestConfig(BaseTest):
|
||||
|
||||
def test_filtering_rules_indexed(self):
|
||||
build_id = 'build_id_0'
|
||||
filtering_rules = {(3, 2, 1): None, (2, 1): '2016-01-26',
|
||||
'build_id': build_id}
|
||||
release = '8.0'
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: filtering_rules.copy()}}
|
||||
):
|
||||
# Checking filtering rules before sorting
|
||||
actual_filtering_rules = app.config.get('FILTERING_RULES')[release]
|
||||
for packages, from_dt in filtering_rules.iteritems():
|
||||
if isinstance(packages, tuple):
|
||||
self.assertNotIn(tuple(sorted(packages)),
|
||||
actual_filtering_rules)
|
||||
self.assertIn(packages, actual_filtering_rules)
|
||||
|
||||
# Checking filtering rules after sorting
|
||||
index_filtering_rules(app)
|
||||
actual_filtering_rules = app.config.get('FILTERING_RULES')[release]
|
||||
for build_info in filtering_rules.iterkeys():
|
||||
self.assertIn(normalize_build_info(build_info),
|
||||
actual_filtering_rules)
|
||||
|
||||
def test_index_filtering_rules_idempotent(self):
|
||||
packages = ('a', 'b', 'c')
|
||||
release = '8.0'
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {packages: None}}}
|
||||
):
|
||||
index_filtering_rules(app)
|
||||
expected_rules = copy.copy(
|
||||
app.config.get('FILTERING_RULES')[release])
|
||||
index_filtering_rules(app)
|
||||
actual_rules = copy.copy(
|
||||
app.config.get('FILTERING_RULES')[release])
|
||||
self.assertIn(normalize_build_info(packages), actual_rules)
|
||||
self.assertEqual(expected_rules, actual_rules)
|
||||
|
||||
def test_index_filtering_rules(self):
|
||||
build_id = '2016-xxx.yyy'
|
||||
self.assertEqual(build_id, normalize_build_info(build_id))
|
||||
packages = ['z', 'x', 'a']
|
||||
self.assertEqual(tuple(sorted(packages)),
|
||||
normalize_build_info(packages))
|
||||
|
|
@ -21,6 +21,8 @@ from collector.api.app import app
|
|||
from collector.api.app import db
|
||||
from collector.api.db.model import InstallationStructure
|
||||
from collector.api.resources.installation_structure import _is_filtered
|
||||
from collector.api.resources.installation_structure import \
|
||||
_is_filtered_by_build_info
|
||||
|
||||
|
||||
class TestInstallationStructure(DbTest):
|
||||
|
|
@ -229,7 +231,7 @@ class TestInstallationStructure(DbTest):
|
|||
)
|
||||
self.check_response_ok(resp, codes=(200, 201))
|
||||
|
||||
def test_is_not_filtered(self):
|
||||
def test_is_not_filtered_by_build_id(self):
|
||||
release = '6.1'
|
||||
build_id = '2014-10-30_14-51-06'
|
||||
struct = {
|
||||
|
|
@ -264,21 +266,93 @@ class TestInstallationStructure(DbTest):
|
|||
{'FILTERING_RULES': {release: {build_id: dt_str}}}):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
def test_is_filtered(self):
|
||||
release = '6.1_filtered'
|
||||
build_id = '2014-10-30_14-51-06_filtered'
|
||||
def test_is_not_filtered_by_packages(self):
|
||||
release = '8.0'
|
||||
packages = ['z_filtered', 'a_filtered']
|
||||
sorted_packages = tuple(sorted(packages))
|
||||
struct = {
|
||||
'fuel_release': {
|
||||
'release': release
|
||||
},
|
||||
'fuel_packages': packages
|
||||
}
|
||||
|
||||
# Have 'packages', no from_dt
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: None}}}
|
||||
):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
# Have 'packages', from_dt in past
|
||||
dt = datetime.datetime.utcnow() - datetime.timedelta(days=1)
|
||||
dt_str = dt.isoformat()
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: dt_str}}}):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
def test_is_not_filtered_by_packages_and_build_id(self):
|
||||
release = '8.0'
|
||||
packages = ['z_filtered', 'a_filtered']
|
||||
sorted_packages = tuple(sorted(packages))
|
||||
build_id = '2016-01-26'
|
||||
struct = {
|
||||
'fuel_release': {
|
||||
'release': release,
|
||||
'build_id': build_id
|
||||
}
|
||||
},
|
||||
'fuel_packages': packages
|
||||
}
|
||||
|
||||
# release not found in rules
|
||||
# No rules
|
||||
with mock.patch.dict(app.config, {'FILTERING_RULES': None}):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
with mock.patch.dict(app.config, {'FILTERING_RULES': {}}):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
# No build info
|
||||
with mock.patch.dict(app.config,
|
||||
{'FILTERING_RULES': {release: None}}):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: None,
|
||||
build_id: None}}}
|
||||
):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
# Have build info, from_dt in past
|
||||
dt = datetime.datetime.utcnow() - datetime.timedelta(days=1)
|
||||
dt_str = dt.isoformat()
|
||||
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: dt_str,
|
||||
build_id: dt_str}}}
|
||||
):
|
||||
self.assertFalse(_is_filtered(struct))
|
||||
|
||||
def test_is_filtered_by_packages_and_build_id(self):
|
||||
release = '8.0_filtered'
|
||||
packages = ['z_filtered', 'a_filtered']
|
||||
build_id = '2016-01-26_filtered'
|
||||
sorted_packages = tuple(sorted(packages))
|
||||
struct = {
|
||||
'fuel_release': {
|
||||
'release': release,
|
||||
'build_id': build_id
|
||||
},
|
||||
'fuel_packages': packages
|
||||
}
|
||||
|
||||
# build_info not found in rules
|
||||
with mock.patch.dict(app.config, {'FILTERING_RULES': {'xx': None}}):
|
||||
self.assertTrue(_is_filtered(struct))
|
||||
|
||||
# build_id not found in rules
|
||||
# build_info not found in rules
|
||||
with mock.patch.dict(app.config,
|
||||
{'FILTERING_RULES': {release: {}}}):
|
||||
self.assertTrue(_is_filtered(struct))
|
||||
|
|
@ -286,11 +360,23 @@ class TestInstallationStructure(DbTest):
|
|||
# from_dt in future
|
||||
dt = datetime.datetime.utcnow() + datetime.timedelta(days=1)
|
||||
dt_str = dt.isoformat()
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: dt_str}}}):
|
||||
self.assertTrue(_is_filtered(struct))
|
||||
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {build_id: dt_str}}}):
|
||||
self.assertTrue(_is_filtered(struct))
|
||||
|
||||
with mock.patch.dict(
|
||||
app.config,
|
||||
{'FILTERING_RULES': {release: {sorted_packages: dt_str,
|
||||
build_id: dt_str}}}
|
||||
):
|
||||
self.assertTrue(_is_filtered(struct))
|
||||
|
||||
def test_is_filtered_check_from_dt_formats(self):
|
||||
release = 'release_dt_format'
|
||||
build_id = 'build_id_dt_format'
|
||||
|
|
@ -361,3 +447,63 @@ class TestInstallationStructure(DbTest):
|
|||
obj = db.session.query(InstallationStructure).filter(
|
||||
InstallationStructure.master_node_uid == master_node_uid).one()
|
||||
self.assertTrue(obj.is_filtered)
|
||||
|
||||
def test_is_filtered_by_build_info_by_build_id(self):
|
||||
build_id = 'build_01'
|
||||
|
||||
# Checking 'build_id' not in filtering rules
|
||||
filtering_rules = {}
|
||||
self.assertTrue(_is_filtered_by_build_info(build_id, filtering_rules))
|
||||
|
||||
# Checking filtering by time
|
||||
from_dt = datetime.datetime.utcnow() + datetime.timedelta(days=2)
|
||||
from_dt_str = from_dt.isoformat()
|
||||
filtering_rules = {build_id: from_dt_str}
|
||||
self.assertTrue(_is_filtered_by_build_info(build_id, filtering_rules))
|
||||
|
||||
def test_is_not_filtered_by_build_info_by_build_id(self):
|
||||
# Checking not filtered if 'build_id' not defined
|
||||
self.assertFalse(_is_filtered_by_build_info(None, {}))
|
||||
|
||||
# Checking filtering by time
|
||||
build_id = 'build_02'
|
||||
from_dt = datetime.datetime.utcnow() - datetime.timedelta(days=1)
|
||||
from_dt_str = from_dt.isoformat()
|
||||
filtering_rules = {build_id: from_dt_str}
|
||||
self.assertFalse(_is_filtered_by_build_info(build_id, filtering_rules))
|
||||
|
||||
def test_is_filtered_by_build_info_by_packages(self):
|
||||
packages = ('z', 'a', 'b')
|
||||
|
||||
# Checking 'packages' not in filtering rules
|
||||
filtering_rules = {}
|
||||
self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules))
|
||||
|
||||
# Checking 'packages' doesn't match
|
||||
filtering_rules = {packages[:-1]: None}
|
||||
self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules))
|
||||
|
||||
filtering_rules = {tuple(sorted(packages))[:-1]: None}
|
||||
self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules))
|
||||
|
||||
# Checking not sorted 'packages' is filtered
|
||||
filtering_rules = {packages: None}
|
||||
self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules))
|
||||
|
||||
# Checking filtering by time
|
||||
from_dt = datetime.datetime.utcnow() + datetime.timedelta(days=2)
|
||||
from_dt_str = from_dt.isoformat()
|
||||
filtering_rules = {packages: from_dt_str}
|
||||
self.assertTrue(_is_filtered_by_build_info(packages, filtering_rules))
|
||||
|
||||
def test_is_not_filtered_by_build_info_by_packages(self):
|
||||
# Checking not filtered if 'packages' not defined
|
||||
self.assertFalse(_is_filtered_by_build_info(None, {}))
|
||||
|
||||
# Checking filtering by time
|
||||
packages = ('z', 'a', 'b')
|
||||
from_dt = datetime.datetime.utcnow() - datetime.timedelta(days=1)
|
||||
from_dt_str = from_dt.isoformat()
|
||||
filtering_rules = {tuple(sorted(packages)): from_dt_str}
|
||||
self.assertFalse(_is_filtered_by_build_info(
|
||||
packages, filtering_rules))
|
||||
|
|
|
|||
Loading…
Reference in New Issue