Use hash of test ID to pick Gerrit ports in tests

This eliminates need for 127.* IPs mapped to loopback interface (which
is true only for Linux) and makes test site generation even more
predictable (name of site dir for a test is the same on any machine).

Hash function used is md5(test_id) % 10000 which is enough for now. This
function is checked for collisions before every run in tox.ini, so if
someone happens to add test that will cause a collision, it'll be
immediately visible. Hash function can be changed to anything else that
maps test ID string to a number in [0,10000].

Change-Id: Ib05d9b489a80e4f55c84db2f8bea20b88e959649
This commit is contained in:
Yuriy Taraday 2016-02-27 06:55:46 +03:00 committed by Darragh Bailey
parent cf4e7cdb5f
commit 33081ce1d6
3 changed files with 81 additions and 5 deletions

View File

@ -13,9 +13,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import hashlib
import os
import shutil
import stat
import struct
import sys
if sys.version < '3':
@ -41,6 +43,20 @@ WAR_URL = 'http://tarballs.openstack.org/' \
GOLDEN_SITE_VER = '4'
# NOTE(yorik-sar): This function needs to be a perfect hash function for
# existing test IDs. This is verified by running check_test_id_hashes script
# prior to running tests. Note that this doesn't imply any cryptographic
# requirements for underlying algorithm, so we can use weak hash here.
# Range of results for this function is limited by port numbers selection
# in _pick_gerrit_port_and_dir method (it can't be greater than 10000 now).
def _hash_test_id(test_id):
if not isinstance(test_id, bytes):
test_id = test_id.encode('utf-8')
hash_ = hashlib.md5(test_id).digest()
num = struct.unpack("=I", hash_[:4])[0]
return num % 10000
class GerritHelpers(object):
def _dir(self, base, *args):
@ -143,7 +159,6 @@ class GerritHelpers(object):
class BaseGitReviewTestCase(testtools.TestCase, GerritHelpers):
"""Base class for the git-review tests."""
_test_counter = 0
_remote = 'gerrit'
@property
@ -158,7 +173,6 @@ class BaseGitReviewTestCase(testtools.TestCase, GerritHelpers):
"""
super(BaseGitReviewTestCase, self).setUp()
self.useFixture(fixtures.Timeout(2 * 60, True))
BaseGitReviewTestCase._test_counter += 1
# ensures git-review command runs in local mode (for functional tests)
self.useFixture(
@ -326,9 +340,13 @@ class BaseGitReviewTestCase(testtools.TestCase, GerritHelpers):
self._run_git('config', 'gitreview.username', 'test_user')
def _pick_gerrit_port_and_dir(self):
pid = os.getpid()
host = '127.%s.%s.%s' % (self._test_counter, pid >> 8, pid & 255)
return host, 29418, host, 8080, self._dir('gerrit', 'site-' + host)
hash_ = _hash_test_id(self.id())
host = '127.0.0.1'
return (
host, 12000 + hash_, # avoid 11211 that is memcached port on CI
host, 22000 + hash_, # avoid ephemeral ports at 32678+
self._dir('gerrit', 'site-' + str(hash_)),
)
def _create_gitreview_file(self, **kwargs):
cfg = ('[gerrit]\n'

View File

@ -0,0 +1,57 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import print_function
import sys
from git_review import tests
from git_review.tests import utils
def list_test_ids(argv):
res = utils.run_cmd(sys.executable, '-m', 'testtools.run', *argv[1:])
return res.split('\n')
def find_collisions(test_ids):
hashes = {}
for test_id in test_ids:
hash_ = tests._hash_test_id(test_id)
if hash_ in hashes:
return (hashes[hash_], test_id)
hashes[hash_] = test_id
return None
def main(argv):
test_ids = list_test_ids(argv)
if not test_ids:
print("No tests found, check command line arguments", file=sys.stderr)
return 1
collision = find_collisions(test_ids)
if collision is None:
return 0
print(
"Found a collision for test ids hash function: %s and %s\n"
"You should change _hash_test_id function in"
" git_review/tests/__init__.py module to fit new set of test ids."
% collision,
file=sys.stderr,
)
return 2
if __name__ == "__main__":
sys.exit(main(sys.argv))

View File

@ -7,6 +7,7 @@ setenv =
VIRTUAL_ENV={envdir}
commands =
python -m git_review.tests.check_test_id_hashes discover --list
python -m git_review.tests.prepare
python setup.py testr --slowest --testr-args='--concurrency=2 {posargs}'