Add some threading improvements

Two improvements have been added:
- display the name of the thread in the logging
- record the thread exit status and return it upon completion
This commit is contained in:
Craig Tracey 2015-10-15 12:18:46 -04:00
parent 68ae12caf3
commit 503d3f0b7b
4 changed files with 46 additions and 29 deletions

View File

@ -32,6 +32,7 @@ class Builder(object):
self._temp_dir = None
self._temp_src_dir = None
self._spec = spec
self._thread_exit = []
def _get_venv_pip_path(self, venv_path):
return os.path.join(venv_path, 'bin/pip')
@ -46,38 +47,44 @@ class Builder(object):
return []
def _build_project(self, project):
self._prepare_project_build(project)
self._make_dir(project.install_path)
try:
self._prepare_project_build(project)
self._make_dir(project.install_path)
# clone the source
src_clone_dir = os.path.join(self._temp_src_dir, project.name)
repo = self._clone_project(project.giturl, project.name,
project.gitref, project.gitdepth,
src_clone_dir)
# clone the source
src_clone_dir = os.path.join(self._temp_src_dir, project.name)
repo = self._clone_project(project.giturl, project.name,
project.gitref, project.gitdepth,
src_clone_dir)
# create and build the virtualenv
self._create_virtualenv(project.venv_command, project.install_path)
dependencies = []
if project.pip_dependencies:
dependencies = project.pip_dependencies
if self._spec.settings.gerrit_dependencies:
dependencies += self._get_gerrit_dependencies(repo, project)
# create and build the virtualenv
self._create_virtualenv(project.venv_command, project.install_path)
dependencies = []
if project.pip_dependencies:
dependencies = project.pip_dependencies
if self._spec.settings.gerrit_dependencies:
dependencies += self._get_gerrit_dependencies(repo, project)
if len(dependencies):
self._install_pip_dependencies(project.install_path,
dependencies)
if len(dependencies):
self._install_pip_dependencies(project.install_path,
dependencies)
if self._spec.settings.include_config:
self._copy_sample_config(src_clone_dir, project)
if self._spec.settings.include_config:
self._copy_sample_config(src_clone_dir, project)
self._install_project(project.install_path, src_clone_dir)
self._install_project(project.install_path, src_clone_dir)
if project.postinstall_dependencies:
dependencies = project.postinstall_dependencies
self._install_pip_dependencies(project, dependencies)
if project.postinstall_dependencies:
dependencies = project.postinstall_dependencies
self._install_pip_dependencies(project.install_path,
dependencies)
# finish up
self._finalize_project_build(project)
# finish up
self._finalize_project_build(project)
except Exception as e:
LOG.error("Oops. Problem building %s: %s", project.name, e)
self._thread_exit.append(-1)
self._thread_exit.append(0)
def build(self):
spec = self._spec
@ -93,17 +100,23 @@ class Builder(object):
for project in spec.projects:
if spec.settings.parallel_build:
t = threading.Thread(target=self._build_project,
args=(project,))
name=project.name, args=(project,))
threads.append(t)
t.start()
else:
self._build_project(project)
rc = 0
if spec.settings.parallel_build:
for thread in threads:
thread.join()
for thread_exit in self._thread_exit:
if thread_exit != 0:
rc = thread_exit
self._finalize_build()
return rc
def cleanup(self):
self._cleanup_build()

View File

@ -30,8 +30,9 @@ def _setup_logger(level=logging.INFO):
logger = logging.getLogger()
logger.setLevel(level)
log_handler = ColorStreamHandler(sys.stdout)
fmt = logging.Formatter(fmt='%(asctime)s %(name)s %(levelname)s: '
'%(message)s', datefmt='%F %H:%M:%S')
fmt = logging.Formatter(fmt='%(asctime)s %(threadName)s %(name)s '
'%(levelname)s: %(message)s',
datefmt='%F %H:%M:%S')
log_handler.setFormatter(fmt)
logger.addHandler(log_handler)
@ -55,7 +56,7 @@ def build(args):
sys.exit()
signal.signal(signal.SIGINT, _signal_handler)
builder.build()
rc = builder.build()
except Exception as e:
LOG.exception("Oops something went wrong: %s", e)
fail = True
@ -63,6 +64,7 @@ def build(args):
builder.cleanup()
if fail:
sys.exit(-1)
sys.exit(rc)
def main():

View File

@ -22,6 +22,7 @@ from nose.plugins.logcapture import LogCapture
class TestLog(unittest.TestCase):
def test_get_logger(self):
lc = LogCapture()
lc.begin()

View File

@ -27,6 +27,7 @@ SAMPLE_SETTINGS = {
class TestSettings(unittest.TestCase):
def test_factory(self):
settings_dict = SAMPLE_SETTINGS
s = settings.Settings.factory(settings_dict)