openstack
/
fuel-stats
Code Issues Proposed changes

Merge "Config format for packages based filtering rules changed"

This commit is contained in:
Jenkins 2016-03-15 07:46:46 +00:00 committed by Gerrit Code Review
parent 019ef0c6e7 fab9e71234
commit e9ec2111a7
6 changed files with 120 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
collector/collector/api/app_prod.py
View File

@ -13,9 +13,11 @@
# under the License.
from collector.api.app import app
from collector.api.config import index_filtering_rules
from collector.api import log
app.config.from_object('collector.api.config.Production')
app.config.from_envvar('COLLECTOR_SETTINGS', silent=True)
index_filtering_rules(app)
log.init_logger()

2
collector/collector/api/app_test.py
View File

@ -13,9 +13,11 @@
# under the License.
from collector.api.app import app
from collector.api.config import index_filtering_rules
from collector.api import log
app.config.from_object('collector.api.config.Testing')
app.config.from_envvar('COLLECTOR_SETTINGS', silent=True)
index_filtering_rules(app)
log.init_logger()

57
collector/collector/api/config.py
View File

@ -15,6 +15,8 @@
import logging
import os
import six
class Production(object):
DEBUG = False
@ -32,8 +34,9 @@ class Production(object):
# Structure of FILTERING_RULES for releases < 8.0:
# {release: {build_id: from_dt}}
# Structure of FILTERING_RULES for releases >= 8.0:
# {release: {('fuel-nailgun-8.0.0-1.mos8212.noarch',
# 'fuel-library8.0-8.0.0-1.mos7718.noarch'): from_dt}}
# {release: [{'packages_list': ['fuel-nailgun-8.0.0-1.mos8212.noarch']},
# {'packages_list': ['fuel-8.0-8.0.0-1.mos7718.noarch'],
# 'from_date': from_dt}]
#
# PAY ATTENTION: you must use tuples as indexes in the FILTERING_RULES
#
@ -53,8 +56,11 @@ class Production(object):
# },
# '6.1.1': {}, # All builds of 6.1.1 filtered
# '7.0': None, # All builds of 7.0 not filtered
# '8.0': {('fuel-nailgun-8.0.0-1.mos8212.noarch',): '2016-02-01T23:00:18',
# ('fuel-nailgun-8.0.0-2.mos9345.noarch',): '2016-02-10',}
# '8.0': [{'packages_list': ['fuel-nailgun-8.0.0-1.mos8212.noarch'],
# 'from_date': '2016-02-01T23:00:18'},
# {'packages_list': ['fuel-nailgun-8.0.0-2.mos9345.noarch']},
# {'build_id': 'build_id_value', 'from_date': '2016-03-01'},
# {'build_id': 'build_id_value'}]
# }
#
# If you don't need any filtration, please set FILTERING_RULES = None
@ -76,16 +82,35 @@ class Testing(Production):
SQLALCHEMY_ECHO = True
def normalize_build_info(build_info):
"""Prepare build info for searching in the filtering rules
def packages_as_index(packages):
if isinstance(packages, (list, tuple)):
return tuple(sorted(packages))
else:
return packages
:param build_info: build_id or packages list
:return: build_id or ordered tuple of packages
def convert_rules_to_dict(rules):
"""Converts filtering rules for release to internal format
:param rules: dict or list of filtering rules for the release
:return: dict of converted filtering rules
"""
if isinstance(build_info, (list, tuple)):
return tuple(sorted(build_info))
return build_info
# Already converted or doesn't need to be converted
if isinstance(rules, dict):
return rules
# If rules is list of dicts
result = {}
for rule in rules:
if 'packages_list' in rule:
build_info = packages_as_index(rule['packages_list'])
else:
build_info = rule['build_id']
result[build_info] = rule.get('from_date')
return result
def index_filtering_rules(app):
@ -98,15 +123,11 @@ def index_filtering_rules(app):
"""
filtering_rules = app.config.get('FILTERING_RULES')
if not filtering_rules:
return
for rules in filtering_rules.itervalues():
for release, rules in six.iteritems(filtering_rules):
if not rules:
continue
for build_info, from_dt in rules.iteritems():
normalized_info = normalize_build_info(build_info)
if normalized_info not in rules:
rules[normalized_info] = from_dt
rules.pop(build_info)
filtering_rules[release] = convert_rules_to_dict(rules)

12
collector/collector/api/resources/installation_structure.py
View File

@ -25,7 +25,7 @@ from collector.api.app import db
from collector.api.common.util import db_transaction
from collector.api.common.util import exec_time
from collector.api.common.util import handle_response
from collector.api.config import normalize_build_info
from collector.api.config import packages_as_index
from collector.api.db.model import InstallationStructure
@ -70,7 +70,7 @@ def _is_filtered_by_build_info(build_info, filtering_rules):
if build_info is None:
return False
build_info = normalize_build_info(build_info)
build_info = packages_as_index(build_info)
# build info not found
if build_info not in filtering_rules:
@ -98,6 +98,7 @@ def _is_filtered(structure):
:return: bool
"""
rules = app.config.get('FILTERING_RULES')
app.logger.debug("Filtering by rules: %s", rules)
# No rules specified
if not rules:
return False
@ -110,18 +111,25 @@ def _is_filtered(structure):
# Release not in rules
if release not in rules:
app.logger.debug("Release: %s not in rules. Not filtered",
release)
return True
filtering_rules = rules.get(release)
# Filtering rules doesn't specified
if filtering_rules is None:
app.logger.debug("Filtering rules are empty. Not filtered")
return False
filtered_by_build_id = _is_filtered_by_build_info(
build_id, filtering_rules)
app.logger.debug("Filtering by build_id: %s, result: %s",
build_id, filtered_by_build_id)
filtered_by_packages = _is_filtered_by_build_info(
packages, filtering_rules)
app.logger.debug("Filtering by packages: %s, result: %s",
packages, filtered_by_packages)
return filtered_by_build_id or filtered_by_packages

84
collector/collector/test/resources/test_config.py
View File

@ -19,34 +19,82 @@ from collector.test.base import BaseTest
from collector.api.app import app
from collector.api.config import index_filtering_rules
from collector.api.config import normalize_build_info
from collector.api.config import packages_as_index
class TestConfig(BaseTest):
def test_filtering_rules_indexed(self):
build_id = 'build_id_0'
filtering_rules = {(3, 2, 1): None, (2, 1): '2016-01-26',
'build_id': build_id}
release = '8.0'
packages_0 = [1, 5, 2]
packages_1 = [6, 4, 3]
from_date_1 = '2016-03-01'
packages_2 = []
raw_rules = {
release: [
{'packages_list': packages_0},
{'packages_list': packages_1, 'from_date': from_date_1},
{'packages_list': packages_2, 'from_date': None}
]
}
expected_rules = {
release: {
packages_as_index(packages_0): None,
packages_as_index(packages_1): from_date_1,
packages_as_index(packages_2): None
}
}
with mock.patch.dict(
app.config,
{'FILTERING_RULES': {release: filtering_rules.copy()}}
{'FILTERING_RULES': copy.deepcopy(raw_rules)}
):
# Checking filtering rules before sorting
actual_filtering_rules = app.config.get('FILTERING_RULES')[release]
for packages, from_dt in filtering_rules.iteritems():
if isinstance(packages, tuple):
self.assertNotIn(tuple(sorted(packages)),
actual_filtering_rules)
self.assertIn(packages, actual_filtering_rules)
actual_rules = app.config.get('FILTERING_RULES')
actual_release_rules = actual_rules[release]
for rule in raw_rules[release]:
packages = packages_as_index(rule['packages_list'])
self.assertNotIn(packages, actual_release_rules)
# Checking filtering rules after sorting
index_filtering_rules(app)
actual_filtering_rules = app.config.get('FILTERING_RULES')[release]
for build_info in filtering_rules.iterkeys():
self.assertIn(normalize_build_info(build_info),
actual_filtering_rules)
actual_rules = app.config.get('FILTERING_RULES')
self.assertEqual(expected_rules, actual_rules)
def test_mix_packages_and_build_id(self):
release_build_id = '7.0'
build_id = 'build_id_0'
release_mixed = '8.0'
build_id_mixed = 'build_id_1'
from_date = '2016-03-01'
packages = [1, 5, 2]
raw_rules = {
release_mixed: [{'packages_list': packages},
{'build_id': build_id_mixed,
'from_date': from_date}],
release_build_id: {build_id: None}
}
with mock.patch.dict(
app.config,
{'FILTERING_RULES': copy.deepcopy(raw_rules)}
):
index_filtering_rules(app)
actual_filtering_rules = app.config.get('FILTERING_RULES')
expected_rules = {
release_mixed: {
packages_as_index(packages): None,
build_id_mixed: from_date
},
release_build_id: {
build_id: None
}
}
self.assertEqual(expected_rules, actual_filtering_rules)
def test_index_filtering_rules_idempotent(self):
packages = ('a', 'b', 'c')
@ -61,12 +109,10 @@ class TestConfig(BaseTest):
index_filtering_rules(app)
actual_rules = copy.copy(
app.config.get('FILTERING_RULES')[release])
self.assertIn(normalize_build_info(packages), actual_rules)
self.assertIn(packages_as_index(packages), actual_rules)
self.assertEqual(expected_rules, actual_rules)
def test_index_filtering_rules(self):
build_id = '2016-xxx.yyy'
self.assertEqual(build_id, normalize_build_info(build_id))
packages = ['z', 'x', 'a']
self.assertEqual(tuple(sorted(packages)),
normalize_build_info(packages))
packages_as_index(packages))

2
collector/manage_collector.py
View File

@ -21,6 +21,7 @@ from flask_script import Manager
from collector.api import log
from collector.api.app import app
from collector.api import app as app_module
from collector.api.config import index_filtering_rules
from collector.api.db.model import *
import flask_sqlalchemy
@ -32,6 +33,7 @@ def configure_app(mode=None):
}
app.config.from_object(mode_map.get(mode))
app.config.from_envvar('COLLECTOR_SETTINGS', silent=True)
index_filtering_rules(app)
setattr(app_module, 'db', flask_sqlalchemy.SQLAlchemy(app))
log.init_logger()
return app