From 2742f4bfc296d3ccae3531e328c2fb31053cdd97 Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Mon, 17 Oct 2016 18:35:04 +0300 Subject: [PATCH] Allow to tune image spec for every image This requires changes in all component repos to replace FROM line to FROM {{ image_spec('someimage') }} Change-Id: I18281bdb41e91cd5c9160055f1617d7ee9d3b548 --- fuel_ccp/build.py | 57 ++++++++++++++---------------------- fuel_ccp/config/images.py | 29 ++++++++++++++++++ fuel_ccp/templates.py | 13 ++------ fuel_ccp/tests/test_build.py | 40 +++++-------------------- 4 files changed, 61 insertions(+), 78 deletions(-) diff --git a/fuel_ccp/build.py b/fuel_ccp/build.py index ae043d25..df84921a 100644 --- a/fuel_ccp/build.py +++ b/fuel_ccp/build.py @@ -13,6 +13,7 @@ import git from fuel_ccp.common import jinja_utils from fuel_ccp import config +from fuel_ccp.config import images BUILD_TIMEOUT = 2 ** 16 # in seconds @@ -26,6 +27,7 @@ _SHUTDOWN = False def render_dockerfile(path, name, config): LOG.info('%s: Rendering dockerfile', name) sources = set() + parent = [] # Could've been None if we could use nonlocal def copy_sources(source_name, cont_dir): if source_name not in config['sources']: @@ -33,9 +35,16 @@ def render_dockerfile(path, name, config): sources.add(source_name) return 'COPY %s %s' % (source_name, cont_dir) - content = jinja_utils.jinja_render(path, config['render'], [copy_sources]) + def image_spec(image_name): + if parent: + raise RuntimeError('You can use image_spec only once in FROM line') + parent.append(image_name) + return images.image_spec(image_name, add_address=CONF.builder.push) - return content, sources + content = jinja_utils.jinja_render(path, config['render'], + [copy_sources, image_spec]) + + return content, sources, parent[0] if parent else None def prepare_source(source_name, name, dest_dir, config): @@ -84,19 +93,16 @@ def find_dockerfiles(repository_name, match=True): dockerfiles = {} repository_dir = os.path.join(CONF.repositories.path, repository_name) - namespace = CONF.images.namespace - if CONF.builder.push and CONF.registry.address: - namespace = '%s/%s' % (CONF.registry.address, namespace) - for root, __, files in os.walk(repository_dir): if 'Dockerfile.j2' in files: path = os.path.join(root, 'Dockerfile.j2') else: continue name = os.path.basename(os.path.dirname(path)) + spec = images.image_spec(name, add_address=CONF.builder.push) dockerfiles[name] = { 'name': name, - 'full_name': '%s/%s:%s' % (namespace, name, CONF.images.tag), + 'full_name': spec, 'path': path, 'parent': None, 'children': [], @@ -120,44 +126,25 @@ def find_dockerfiles(repository_name, match=True): def render_dockerfiles(dockerfiles, config): for dockerfile in dockerfiles.values(): - content, sources = \ + content, sources, parent = \ render_dockerfile(dockerfile['path'], dockerfile['name'], config) dockerfile['content'] = content dockerfile['sources'] = sources + dockerfile['parent'] = parent IMAGE_FULL_NAME_RE = r"((?P[\w:\.-]+)/){0,2}" \ "(?P[\w_-]+)" \ "(:(?P[\w_\.-]+))?" IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE) -# This regex is needed for matching not yet rendered images -NOT_RENDERED_IMAGE_PATTERN = (r"((?P[\w:\.\-}{ ]+)/){0,2}" - r"(?P[\w_\-}{ ]+)" - r"(:(?P[\w_\.\-}{ ]+))?") - -DOCKER_FILE_FROM_PATTERN = re.compile( - r"^\s?FROM\s+{}\s?$".format(NOT_RENDERED_IMAGE_PATTERN), re.MULTILINE -) -def find_dependencies(dockerfiles): - for name, dockerfile in dockerfiles.items(): - with open(dockerfile['path']) as f: - content = f.read() - - matcher = DOCKER_FILE_FROM_PATTERN.search(content) - if not matcher: - raise RuntimeError( - "FROM clause was not found in dockerfile for image: " + name - ) - - parent_ns = matcher.group("namespace") - if not parent_ns: - continue - parent_name = matcher.group("name") - - dockerfile['parent'] = dockerfiles[parent_name] - dockerfiles[parent_name]['children'].append(dockerfile) +def connect_children(dockerfiles): + for dockerfile in dockerfiles.values(): + parent = dockerfile['parent'] + if parent: + dockerfiles[parent]['children'].append(dockerfile) + dockerfile['parent'] = dockerfiles[parent] def build_dockerfile(dc, dockerfile): @@ -381,7 +368,7 @@ def build_components(components=None): find_dockerfiles(repository_def['name'], match=match)) render_dockerfiles(dockerfiles, config) - find_dependencies(dockerfiles) + connect_children(dockerfiles) ready_images = get_ready_image_names() diff --git a/fuel_ccp/config/images.py b/fuel_ccp/config/images.py index a9b96a84..310003a6 100644 --- a/fuel_ccp/config/images.py +++ b/fuel_ccp/config/images.py @@ -5,6 +5,7 @@ DEFAULTS = { 'base_distro': 'debian', 'base_tag': 'jessie', 'maintainer': 'MOS Microservices ', + 'image_specs': {}, }, } @@ -18,6 +19,34 @@ SCHEMA = { 'base_distro': {'type': 'string'}, 'base_tag': {'type': 'string'}, 'maintainer': {'type': 'string'}, + 'image_specs': { + 'type': 'object', + 'additionalProperties': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'tag': {'type': 'string'}, + 'namespace': {'type': 'string'}, + }, + }, + }, }, }, } + + +def image_spec(image_name, add_address=True): + from fuel_ccp import config + CONF = config._REAL_CONF + spec = { + 'name': image_name, + 'namespace': CONF.images.namespace, + 'tag': CONF.images.tag, + } + image_spec = CONF.images.image_specs.get(image_name) + if image_spec: + spec.update(image_spec._items()) + spec_str = '{namespace}/{name}:{tag}'.format(**spec) + if add_address and CONF.registry.address: + spec_str = '{}/{}'.format(CONF.registry.address, spec_str) + return spec_str diff --git a/fuel_ccp/templates.py b/fuel_ccp/templates.py index 31a3de0b..e1fc3a26 100644 --- a/fuel_ccp/templates.py +++ b/fuel_ccp/templates.py @@ -2,6 +2,7 @@ import json from fuel_ccp.common import utils from fuel_ccp import config +from fuel_ccp.config import images CONF = config.CONF @@ -16,14 +17,6 @@ ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py" PYTHON_PATH = "/usr/bin/python" -def _get_image_name(image_name): - image_name = "%s/%s:%s" % (CONF.images.namespace, image_name, - CONF.images.tag) - if CONF.registry.address: - image_name = "%s/%s" % (CONF.registry.address, image_name) - return image_name - - def _get_start_cmd(role_name): return ["dumb-init", PYTHON_PATH, ENTRYPOINT_PATH, "provision", role_name] @@ -105,7 +98,7 @@ def serialize_env_variables(container): def serialize_daemon_container_spec(container): cont_spec = { "name": container["name"], - "image": _get_image_name(container["image"]), + "image": images.image_spec(container["image"]), "command": _get_start_cmd(container["name"]), "volumeMounts": serialize_volume_mounts(container), "readinessProbe": { @@ -136,7 +129,7 @@ def serialize_daemon_container_spec(container): def serialize_job_container_spec(container, job): return { "name": job["name"], - "image": _get_image_name(container["image"]), + "image": images.image_spec(container["image"]), "command": _get_start_cmd(job["name"]), "volumeMounts": serialize_volume_mounts(container), "env": serialize_env_variables(container) diff --git a/fuel_ccp/tests/test_build.py b/fuel_ccp/tests/test_build.py index 112a93c2..6f7f8767 100644 --- a/fuel_ccp/tests/test_build.py +++ b/fuel_ccp/tests/test_build.py @@ -1,5 +1,4 @@ import collections -import io import os import fixtures @@ -52,36 +51,6 @@ class TestBuild(base.TestCase): },) ]) - def test_find_dependencies_no_registry(self): - m_open = mock.mock_open() - m_open.side_effect = [ - io.StringIO(BASE_DOCKERFILE), - io.StringIO(COMPONENT_DOCKERFILE.format('')) - ] - dockerfiles = self.__create_dockerfile_objects() - with mock.patch('fuel_ccp.build.open', m_open, create=True): - build.find_dependencies(dockerfiles) - - self.assertListEqual([dockerfiles['ms-mysql']], - dockerfiles['ms-debian-base']['children']) - self.assertDictEqual(dockerfiles['ms-debian-base'], - dockerfiles['ms-mysql']['parent']) - - def test_find_dependencies_registry(self): - m_open = mock.mock_open() - m_open.side_effect = [ - io.StringIO(BASE_DOCKERFILE), - io.StringIO(COMPONENT_DOCKERFILE.format('example.com:8909/')) - ] - dockerfiles = self.__create_dockerfile_objects() - with mock.patch('fuel_ccp.build.open', m_open, create=True): - build.find_dependencies(dockerfiles) - - self.assertListEqual([dockerfiles['ms-mysql']], - dockerfiles['ms-debian-base']['children']) - self.assertDictEqual(dockerfiles['ms-debian-base'], - dockerfiles['ms-mysql']['parent']) - @mock.patch("docker.Client") @mock.patch("fuel_ccp.build.build_dockerfile") @mock.patch("fuel_ccp.build.submit_dockerfile_processing") @@ -266,18 +235,23 @@ class TestRenderDockerfile(testscenarios.WithScenarios, base.TestCase): ('empty', { 'config': {'render': {}}, 'source': '', - 'result': ('', set()), + 'result': ('', set(), None), }), ('one_source', { 'config': {'render': {}, 'sources': {'one': {}}}, 'source': '{{ copy_sources("one", "/tmp") }}', - 'result': ('COPY one /tmp', {'one'}), + 'result': ('COPY one /tmp', {'one'}, None), }), ('wrong_source', { 'config': {'render': {}, 'sources': {'one': {}}}, 'source': '{{ copy_sources("wrong", "/tmp") }}', 'exception': Exception('No such source: wrong'), }), + ('one_from', { + 'config': {'render': {}}, + 'source': 'FROM {{ image_spec("one") }}', + 'result': ('FROM ccp/one:latest', set(), 'one'), + }), ] config = None