openstack
/
fuel-qa
Code Issues Proposed changes

Merge "Refactor gerrit client"

This commit is contained in:
Jenkins 2016-06-08 09:15:54 +00:00 committed by Gerrit Code Review
parent d2d49cde95 b3d498520d
commit 7fa8638d87
5 changed files with 292 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/helpers.rst
View File

@ -157,4 +157,14 @@ Info provider
Utils
-----
.. automodule:: fuelweb_test.helpers.gerrit.utils
:members:
Rules
-----
.. automodule:: fuelweb_test.helpers.gerrit.rules
:members:
Content parser
--------------
.. automodule:: fuelweb_test.helpers.gerrit.content_parser
:members:

71
fuelweb_test/helpers/gerrit/content_parser.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import re
# pylint: disable=redefined-builtin
# noinspection PyUnresolvedReferences
from six.moves import xrange
# pylint: enable=redefined-builtin
class PuppetfileChangesParser(object):
def __init__(self, review, path):
self.review = review
self.filepath = path
def get_changed_modules(self):
content = self.review.get_content_as_dict(self.filepath)
diff = self.review.get_diff_as_dict(self.filepath)
diff_lines_changed = self._get_lines_num_changed_from_diff(diff)
mod_lines_changed = self._get_modules_line_num_changed_from_content(
diff_lines_changed, content)
return self._get_modules_from_lines_changed(mod_lines_changed, content)
@staticmethod
def _get_lines_num_changed_from_diff(diff):
lines_changed = []
cursor = 1
for content in diff['content']:
diff_content = content.values()[0]
if 'ab' in content.keys():
cursor += len(diff_content)
if 'b' in content.keys():
lines_changed.extend(
xrange(cursor, len(diff_content) + cursor))
cursor += len(diff_content)
return lines_changed
@staticmethod
def _get_modules_line_num_changed_from_content(lines, content):
modules_lines_changed = []
for num in lines:
index = num
if content[index] == '' or content[index].startswith('#'):
continue
while not content[index].startswith('mod'):
index -= 1
modules_lines_changed.append(index)
return modules_lines_changed
def _get_modules_from_lines_changed(self, lines, content):
modules = []
pattern = re.compile(r"mod '([a-z]+)',")
for num in lines:
match = pattern.match(content[num])
if match:
module = match.group(1)
modules.append((module, self.filepath))
return modules

49
fuelweb_test/helpers/gerrit/gerrit_client.py
View File

@ -12,12 +12,15 @@
# License for the specific language governing permissions and limitations
# under the License.
import base64
import os
import requests
from requests.utils import quote
from fuelweb_test.helpers.gerrit import utils
class GerritClient(object):
class BaseGerritClient(object):
def __init__(self,
endpoint='https://review.openstack.org',
@ -71,3 +74,47 @@ class GerritClient(object):
def _send_get_request(self):
return requests.get(self.query, verify=False)
class GerritClient(BaseGerritClient):
def __init__(self, *args, **kwargs):
super(GerritClient, self).__init__(*args, **kwargs)
def get_files(self):
r = self._request_file_list()
text = r.text
files = utils.filter_response_text(text)
return set(filter(lambda x: x != '/COMMIT_MSG',
utils.json_to_dict(files).keys()))
def get_content_as_dict(self, filename):
content_decoded = self._request_content(filename).text
content = base64.b64decode(content_decoded)
return {num: line for num, line in enumerate(content.split('\n'), 1)}
def get_diff_as_dict(self, filename):
diff_raw = self._request_diff(filename).text
diff_filtered = utils.filter_response_text(diff_raw)
return utils.json_to_dict(diff_filtered)
def get_dependencies_as_dict(self):
dependencies_raw = self._request_related_changes().text
dependencies_filtered = utils.filter_response_text(dependencies_raw)
return utils.json_to_dict(dependencies_filtered)
@utils.check_status_code(200)
def _request_file_list(self):
return self.list_files()
@utils.check_status_code(200)
def _request_content(self, filename):
return self.get_content(filename)
@utils.check_status_code(200)
def _request_diff(self, filename):
return self.get_diff(filename)
@utils.check_status_code(200)
def _request_related_changes(self):
return self.get_related_changes()

265
fuelweb_test/helpers/gerrit/gerrit_info_provider.py
View File

@ -12,42 +12,55 @@
# License for the specific language governing permissions and limitations
# under the License.
import base64
from collections import namedtuple
import os
import re
# pylint: disable=redefined-builtin
# noinspection PyUnresolvedReferences
from six.moves import xrange
# pylint: enable=redefined-builtin
from fuelweb_test import logger
from fuelweb_test import settings
from fuelweb_test.helpers.gerrit.gerrit_client import GerritClient
from fuelweb_test.helpers.gerrit import utils
from fuelweb_test.helpers.gerrit import rules
class TemplateMap(object):
M_PATH = 'deployment/puppet/'
MAP = [
{'deployment/Puppetfile':
rules.get_changed_modules_inside_file},
{os.path.join(M_PATH, 'osnailyfacter/modular/roles/'):
rules.osnailyfacter_roles_rule},
{os.path.join(M_PATH, 'osnailyfacter/modular/'):
rules.osnailyfacter_modular_rule},
{os.path.join(M_PATH, 'osnailyfacter/manifests/'):
rules.osnailyfacter_manifest_rule},
{os.path.join(M_PATH, 'osnailyfacter/templates/'):
rules.osnailyfacter_templates_rule},
{os.path.join(M_PATH, 'osnailyfacter/'):
rules.no_rule},
{os.path.join(M_PATH, 'openstack_tasks/Puppetfile'):
rules.get_changed_modules_inside_file},
{os.path.join(M_PATH, 'openstack_tasks/lib/facter/'):
rules.openstack_tasks_libfacter_rule},
{os.path.join(M_PATH, 'openstack_tasks/manifests/roles/'):
rules.openstack_tasks_roles_rule},
{os.path.join(M_PATH, 'openstack_tasks/examples/roles/'):
rules.openstack_tasks_roles_rule},
{os.path.join(M_PATH, 'openstack_tasks/manifests/'):
rules.openstack_manifest_rule},
{os.path.join(M_PATH, 'openstack_tasks/examples/'):
rules.openstack_examples_rule},
{os.path.join(M_PATH, 'openstack_tasks/'):
rules.no_rule},
{M_PATH:
rules.common_rule},
]
class FuelLibraryModulesProvider(object):
PROJECT_ROOT_PATH = 'fuel-library'
MODULE_ROOT_PATH = 'deployment/puppet/'
OSNAILYFACTER_NAME = 'osnailyfacter'
OSNAILYFACTER_PATH = \
os.path.join(MODULE_ROOT_PATH, OSNAILYFACTER_NAME, 'modular/')
OSNAILYFACTER_ROLES_PATH = os.path.join(OSNAILYFACTER_PATH, 'roles/')
TASKS_YAML_PATH = os.path.join(OSNAILYFACTER_ROLES_PATH, 'tasks.yaml')
OPENSTACK_TASKS_PATH = os.path.join(MODULE_ROOT_PATH,
'openstack_tasks/manifests/')
OS_TASKS_YAML_PATH = os.path.join(MODULE_ROOT_PATH,
'openstack_tasks/tasks.yaml')
PUPPETFILE_PATH = 'deployment/Puppetfile'
def __init__(self, gerrit_review):
self.gerrit_review = gerrit_review
def __init__(self, review):
self.changed_modules = {}
self._files_list = set()
self.dependency_provider = DependencyProvider(self.gerrit_review)
self.review = review
@classmethod
def from_environment_vars(cls, endpoint='https://review.openstack.org'):
@ -58,190 +71,36 @@ class FuelLibraryModulesProvider(object):
patchset_num=settings.GERRIT_PATCHSET_NUMBER)
return cls(review)
def get_changed_modules(self, dependency_lookup=True):
self._store_file_list()
self._find_modules_in_files()
self._find_modules_in_puppetfile_()
if dependency_lookup:
dependencies = self.dependency_provider.get_dependencies(
self.gerrit_review)
for dependency in dependencies:
self.gerrit_review.change_id = dependency.change_id
self.gerrit_review.patchset_num = str(dependency.patchset_num)
self._store_file_list()
self._find_modules_in_files()
self._find_modules_in_puppetfile_()
def get_changed_modules(self):
logger.debug('Review details: branch={0}, id={1}, patchset={2}'
.format(self.review.branch,
self.review.change_id,
self.review.patchset_num))
files = self.review.get_files()
for _file in files:
self._apply_rule(review=self.review, _file=_file)
return self.changed_modules
def _store_file_list(self):
r = self._request_file_list()
text = r.text
files = utils.filter_response_text(text)
logger.debug('Changed files list {}'.format(files))
self._files_list.update(set(filter(lambda x: x != '/COMMIT_MSG',
utils.json_to_dict(files).keys())))
@utils.check_status_code(200)
def _request_file_list(self):
return self.gerrit_review.list_files()
def _find_modules_in_files(self):
for f in self._files_list:
if f.startswith(FuelLibraryModulesProvider.MODULE_ROOT_PATH):
split_path = f.split('/')
module = split_path[-1]
logger.debug('Process next module {}'.format(module))
self._add_module_from_files(module, split_path)
self._add_module_from_osnailyfacter(f, split_path)
self._add_module_from_openstack_tasks(f, split_path)
def _add_module_from_files(self, module, split_path):
if module != FuelLibraryModulesProvider.OSNAILYFACTER_NAME:
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH, *split_path[:3]
)
logger.debug('Add module {0} from files by path {1}'.format(
module, module_path))
self._add_module(module, module_path)
def _add_module_from_openstack_tasks(self, filename, split_path):
if filename.startswith(
FuelLibraryModulesProvider.OPENSTACK_TASKS_PATH) \
and filename != FuelLibraryModulesProvider.OS_TASKS_YAML_PATH:
module = split_path[4]
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH,
*split_path[0:-1])
logger.debug('Add module {0} from openstack_tasks '
'by path {1}'.format(module, module_path))
self._add_module(module, module_path)
def _add_module(self, module, module_path):
logger.debug('Changed modules are {}'.format(self.changed_modules))
logger.debug("Add module '{}' to changed modules".format(module))
if module in self.changed_modules:
logger.debug('Add module {} to changed modules'.format(module))
self.changed_modules[module].add(module_path)
else:
self.changed_modules[module] = {module_path}
logger.debug('Add module {} to changed modules'.format(module))
def _add_module_from_osnailyfacter(self, filename, split_path):
if filename.startswith(FuelLibraryModulesProvider.OSNAILYFACTER_PATH) \
and filename != FuelLibraryModulesProvider.TASKS_YAML_PATH:
module = split_path[4]
if module == 'roles':
module = 'roles/{}'.format(os.path.basename(filename))
module_path = os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH, *split_path[:5]
)
logger.debug('Add module {0} from osnailyfacter '
'by path {1}'.format(module, module_path))
def _add_modules(self, modules):
for module, module_path in modules:
self._add_module(module, module_path)
def _get_puppetfile_content_as_dict(self):
content_decoded = self._request_content(
FuelLibraryModulesProvider.PUPPETFILE_PATH
).text
content = base64.b64decode(content_decoded)
return {num: line for num, line in enumerate(content.split('\n'), 1)}
@utils.check_status_code(200)
def _request_content(self, filename):
return self.gerrit_review.get_content(filename)
def _get_puppetfile_diff_as_dict(self):
diff_raw = self._request_diff(
FuelLibraryModulesProvider.PUPPETFILE_PATH
).text
diff_filtered = utils.filter_response_text(diff_raw)
return utils.json_to_dict(diff_filtered)
@utils.check_status_code(200)
def _request_diff(self, filename):
return self.gerrit_review.get_diff(filename)
@staticmethod
def _get_lines_num_changed_from_diff(diff):
lines_changed = []
cursor = 1
for content in diff['content']:
diff_content = content.values()[0]
if 'ab' in content.keys():
cursor += len(diff_content)
if 'b' in content.keys():
lines_changed.extend(
xrange(cursor, len(diff_content) + cursor))
cursor += len(diff_content)
return lines_changed
@staticmethod
def _get_modules_line_num_changed_from_content(lines, content):
modules_lines_changed = []
for num in lines:
index = num
if content[index] == '' or content[index].startswith('#'):
continue
while not content[index].startswith('mod'):
index -= 1
modules_lines_changed.append(index)
return modules_lines_changed
def _add_modules_from_lines_changed(self, lines, content):
pattern = re.compile(r"mod '([a-z]+)',")
for num in lines:
match = pattern.match(content[num])
if match:
module = match.group(1)
self._add_module(
module,
os.path.join(
FuelLibraryModulesProvider.PROJECT_ROOT_PATH,
FuelLibraryModulesProvider.PUPPETFILE_PATH
)
)
def _find_modules_in_puppetfile_(self):
if FuelLibraryModulesProvider.PUPPETFILE_PATH in self._files_list:
content = self._get_puppetfile_content_as_dict()
diff = self._get_puppetfile_diff_as_dict()
diff_lines_changed = self._get_lines_num_changed_from_diff(diff)
mod_lines_changed = \
self._get_modules_line_num_changed_from_content(
diff_lines_changed, content)
self._add_modules_from_lines_changed(mod_lines_changed, content)
class DependencyProvider(object):
Dependency = namedtuple('Dependency', ['change_id', 'patchset_num'])
def __init__(self, review=None):
self.review = review
self.dependent_reviews = set()
@utils.check_status_code(200)
def _request_related_changes(self):
return self.review.get_related_changes()
def _get_dependencies_as_dict(self):
dependencies_raw = self._request_related_changes().text
dependencies_filtered = utils.filter_response_text(dependencies_raw)
return utils.json_to_dict(dependencies_filtered)
def _store_dependent_reviews(self, dependencies):
for dependency in dependencies['changes']:
if 'change_id' in dependency and \
'_current_revision_number' in dependency:
d = DependencyProvider.Dependency(
change_id=dependency['change_id'],
patchset_num=dependency['_current_revision_number']
)
if d.change_id != self.review.change_id:
self.dependent_reviews.add(d)
def get_dependencies(self, review=None):
if review:
self.review = review
dependencies = self._get_dependencies_as_dict()
self._store_dependent_reviews(dependencies)
return self.dependent_reviews
def _apply_rule(self, review, _file):
for path_rule in TemplateMap.MAP:
tmpl, rule = next(iter(path_rule.items()))
if _file.startswith(tmpl):
logger.debug("Using '{0}' rule with '{1}' template "
"for '{2}' filename".format(rule.__name__,
tmpl,
_file))
modules = rules.invoke_rule(review, _file, rule)
if modules:
self._add_modules(modules)
return

101
fuelweb_test/helpers/gerrit/rules.py Normal file
View File

@ -0,0 +1,101 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from fuelweb_test.helpers.gerrit.content_parser import PuppetfileChangesParser
FUEL_LIBRARY_PROJECT_NAME = 'fuel-library'
def invoke_rule(review, path, rule):
if rule.__name__ == 'get_changed_modules_inside_file':
return rule(review, path)
else:
return rule(path)
def get_changed_modules_inside_file(review, filename):
parser = PuppetfileChangesParser(review=review, path=filename)
return [(module, os.path.join(FUEL_LIBRARY_PROJECT_NAME, module_path))
for module, module_path in parser.get_changed_modules()]
def no_rule(path):
return []
def common_rule(path):
return _apply_standard_rule(path=path, mod_depth=2)
def osnailyfacter_roles_rule(path):
return _apply_subdir_rule(path=path, subdir='roles', mod_depth=5)
def osnailyfacter_modular_rule(path):
return _apply_standard_rule(path=path)
def osnailyfacter_manifest_rule(path):
return _apply_standard_rule(path=path)
def osnailyfacter_templates_rule(path):
return _apply_standard_rule(path=path)
def openstack_tasks_libfacter_rule(path):
return _apply_standard_rule(path=path, mod_depth=5)
def openstack_tasks_roles_rule(path):
return _apply_subdir_rule(path=path, subdir='roles', mod_depth=4)
def openstack_manifest_rule(path):
return _apply_standard_rule(path=path)
def openstack_examples_rule(path):
return _apply_standard_rule(path=path)
def _join_module_path(split_path, depth):
return os.path.join(FUEL_LIBRARY_PROJECT_NAME, *split_path[:depth])
def _apply_subdir_rule(path, subdir, mod_depth=4):
"""Returns module name and module path if not given subdir, otherwise
returns module combined with given subdir.
"""
split_path = path.split('/')
module = split_path[mod_depth]
if module == subdir:
filename, _ = os.path.splitext(os.path.basename(path))
module = '{}/{}'.format(subdir, filename)
module_path = _join_module_path(split_path, mod_depth + 2)
return [(module, module_path)]
def _apply_standard_rule(path, mod_depth=4):
"""Returns module name and module path by applying the following rule:
if this is a directory, then use directory name as the module name,
otherwise use filename without extension as the module name.
"""
split_path = path.split('/')
module, _ = os.path.splitext(split_path[mod_depth])
module_path = _join_module_path(split_path, mod_depth + 1)
return [(module, module_path)]