Use pytest for queries
Switches queries testing to use of pytest which provides the following: - test generator for each query (parametrize) - ability to test a single query test - generate html report with test results, making easier to investigate failures. - parallel executions - minor bugfix which prevented running queries from running with py38 as the config parser requires only strings (None being invalid). Change-Id: I982c694a5160a9ecfd117d177d30b911cfe53425
This commit is contained in:
Sorin Sbarnea
Sorin Sbarnea (zbr)
parent
df62575028
commit
c6f07d7f93
|
|
@ -106,7 +106,7 @@ class Config(object):
|
|||
{'es_url': ES_URL,
|
||||
'ls_url': LS_URL,
|
||||
'db_uri': DB_URI,
|
||||
'server_password': None,
|
||||
'server_password': '',
|
||||
'ci_username': CI_USERNAME,
|
||||
'jobs_re': JOBS_RE,
|
||||
'pidfile': PID_FN,
|
||||
|
|
|
|||
|
|
@ -11,82 +11,48 @@
|
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from functools import lru_cache
|
||||
import os
|
||||
import pytest
|
||||
import logging
|
||||
|
||||
from launchpadlib import launchpad
|
||||
import pyelasticsearch
|
||||
|
||||
import elastic_recheck.config as er_conf
|
||||
from elastic_recheck import elasticRecheck
|
||||
import elastic_recheck.query_builder as qb
|
||||
from elastic_recheck import tests
|
||||
# from elastic_recheck import tests
|
||||
|
||||
LPCACHEDIR = os.path.expanduser('~/.launchpadlib/cache')
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class TestQueries(tests.TestCase):
|
||||
"""Make sure queries are valid.
|
||||
class Context():
|
||||
|
||||
Make sure all queries are for open OpenStack bugs in Launchpad or have
|
||||
hits in logstash.openstack.org.
|
||||
This test is used to validate any changes to queries.yaml
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
super(TestQueries, self).setUp()
|
||||
@lru_cache()
|
||||
def __init__(self):
|
||||
config = er_conf.Config(config_file='elasticRecheck.conf')
|
||||
self.classifier = elasticRecheck.Classifier(config.gerrit_query_file,
|
||||
config=config)
|
||||
self.classifier = elasticRecheck.Classifier(
|
||||
config.gerrit_query_file,
|
||||
config=config)
|
||||
self.lp = launchpad.Launchpad.login_anonymously(
|
||||
'grabbing bugs',
|
||||
'production',
|
||||
LPCACHEDIR)
|
||||
|
||||
self.lp = launchpad.Launchpad.login_anonymously('grabbing bugs',
|
||||
'production',
|
||||
LPCACHEDIR)
|
||||
self.openstack_projects = (self.get_group_projects('openstack') +
|
||||
self.get_group_projects('oslo') +
|
||||
# Fix for story 2006737 since os-brick is
|
||||
# not in the openstack group in launchpad.
|
||||
['os-brick'])
|
||||
self.openstack_projects = (
|
||||
self.get_group_projects('openstack') +
|
||||
self.get_group_projects('oslo') +
|
||||
# Fix for story 2006737 since os-brick is
|
||||
# not in the openstack group in launchpad.
|
||||
['os-brick'])
|
||||
|
||||
def get_group_projects(self, group_name):
|
||||
group = self.lp.project_groups[group_name]
|
||||
return list(map(lambda project: project.name,
|
||||
group.projects))
|
||||
|
||||
def test_launchpad(self):
|
||||
bad_bugs = []
|
||||
for x in self.classifier.queries:
|
||||
print("Looking for bug: https://bugs.launchpad.net/bugs/%s"
|
||||
% x['bug'])
|
||||
if not self._is_valid_launchpad_bug(x['bug']):
|
||||
bad_bugs.append("https://bugs.launchpad.net/bugs/%s" %
|
||||
x['bug'])
|
||||
if len(bad_bugs) > 0:
|
||||
self.fail("the following bugs are not targeted to openstack "
|
||||
"on launchpad: %s" % bad_bugs)
|
||||
|
||||
def test_elasticsearch_query(self):
|
||||
for x in self.classifier.queries:
|
||||
print("Looking for bug: https://bugs.launchpad.net/bugs/%s"
|
||||
% x['bug'])
|
||||
self.assertTrue(
|
||||
self._is_valid_ElasticSearch_query(x, x['bug']),
|
||||
("Something is wrong with bug %s" % x['bug']))
|
||||
|
||||
def _is_valid_ElasticSearch_query(self, x, bug):
|
||||
query = qb.generic(x['query'])
|
||||
try:
|
||||
results = self.classifier.es.search(query, size='10')
|
||||
except pyelasticsearch.ElasticHttpError:
|
||||
self.fail("Failure to process query for bug %s" % bug)
|
||||
|
||||
valid_query = len(results) > 0
|
||||
if not valid_query:
|
||||
print("Didn't find any hits for bug %s" % x['bug'])
|
||||
# don't fail tests if no hits for a bug
|
||||
return True
|
||||
|
||||
def _is_valid_launchpad_bug(self, bug):
|
||||
def _is_valid_launchpad_bug(self, bug) -> bool:
|
||||
bug_tasks = self.lp.bugs[bug].bug_tasks
|
||||
projects = map(lambda bug_task: bug_task.bug_target_name, bug_tasks)
|
||||
# Check that at least one bug_tasks is targeted to an OpenStack Project
|
||||
|
|
@ -96,5 +62,57 @@ class TestQueries(tests.TestCase):
|
|||
continue
|
||||
if project in self.openstack_projects:
|
||||
return True
|
||||
print("bug has no targets in the openstack launchpad group")
|
||||
LOGGER.error("bug has no targets in the openstack launchpad group")
|
||||
return False
|
||||
|
||||
def _is_valid_ElasticSearch_query(self, x, bug) -> bool:
|
||||
query = qb.generic(x['query'])
|
||||
results = self.classifier.es.search(query, size='10')
|
||||
|
||||
valid_query = len(results) > 0
|
||||
if not valid_query:
|
||||
LOGGER.error("Didn't find any hits for bug %s" % x['bug'])
|
||||
# don't fail tests if no hits for a bug
|
||||
return True
|
||||
|
||||
|
||||
_ctx = Context()
|
||||
|
||||
|
||||
@pytest.fixture(scope='session')
|
||||
def ctx(request):
|
||||
return _ctx
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
_ctx.classifier.queries,
|
||||
ids=[x['bug'] for x in _ctx.classifier.queries])
|
||||
def test_launchpad(ctx, x):
|
||||
"""Make sure queries are valid.
|
||||
|
||||
Make sure all queries are for open OpenStack bugs in Launchpad or have
|
||||
hits in logstash.openstack.org.
|
||||
This test is used to validate any changes to queries.yaml
|
||||
"""
|
||||
LOGGER.debug(
|
||||
"Looking for bug: https://bugs.launchpad.net/bugs/%s",
|
||||
x['bug'])
|
||||
assert \
|
||||
ctx._is_valid_launchpad_bug(x['bug']) is True, \
|
||||
f"Bug {x['bug']} is not targeted to openstack on launchpad."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
_ctx.classifier.queries,
|
||||
ids=[x['bug'] for x in _ctx.classifier.queries])
|
||||
def test_elasticsearch_query(ctx, x):
|
||||
LOGGER.debug(
|
||||
"Looking for bug: https://bugs.launchpad.net/bugs/%s",
|
||||
x['bug'])
|
||||
try:
|
||||
assert ctx._is_valid_ElasticSearch_query(x, x['bug']), \
|
||||
("Something is wrong with bug %s" % x['bug'])
|
||||
except Exception as e:
|
||||
pytest.fail(f"Exception {e} receive with query: {x}")
|
||||
|
|
|
|||
|
|
@ -5,3 +5,7 @@ testrepository>=0.0.17
|
|||
testscenarios>=0.4
|
||||
testtools>=0.9.36,!=1.2.0
|
||||
mock>=1.0
|
||||
# queries use pytest
|
||||
pytest
|
||||
pytest-xdist
|
||||
pytest-html
|
||||
|
|
|
|||
3
tox.ini
3
tox.ini
|
|
@ -22,7 +22,8 @@ commands = python setup.py testr --slowest --testr-args='{posargs} tests.functio
|
|||
|
||||
[testenv:queries]
|
||||
basepython = python3
|
||||
commands = python setup.py testr --slowest --testr-args='{posargs} tests.functional.test_queries'
|
||||
commands =
|
||||
pytest -k functional {posargs:-n auto --html={envlogdir}/reports.html --self-contained-html -v}
|
||||
|
||||
[testenv:venv]
|
||||
basepython = python3
|
||||
|
|
|
|||
Loading…
Reference in New Issue