Merge "Allow to tune image spec for every image"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
e1a311a2fa
|
|
@ -13,6 +13,7 @@ import git
|
|||
|
||||
from fuel_ccp.common import jinja_utils
|
||||
from fuel_ccp import config
|
||||
from fuel_ccp.config import images
|
||||
|
||||
BUILD_TIMEOUT = 2 ** 16 # in seconds
|
||||
|
||||
|
|
@ -26,6 +27,7 @@ _SHUTDOWN = False
|
|||
def render_dockerfile(path, name, config):
|
||||
LOG.info('%s: Rendering dockerfile', name)
|
||||
sources = set()
|
||||
parent = [] # Could've been None if we could use nonlocal
|
||||
|
||||
def copy_sources(source_name, cont_dir):
|
||||
if source_name not in config['sources']:
|
||||
|
|
@ -33,9 +35,16 @@ def render_dockerfile(path, name, config):
|
|||
sources.add(source_name)
|
||||
return 'COPY %s %s' % (source_name, cont_dir)
|
||||
|
||||
content = jinja_utils.jinja_render(path, config['render'], [copy_sources])
|
||||
def image_spec(image_name):
|
||||
if parent:
|
||||
raise RuntimeError('You can use image_spec only once in FROM line')
|
||||
parent.append(image_name)
|
||||
return images.image_spec(image_name, add_address=CONF.builder.push)
|
||||
|
||||
return content, sources
|
||||
content = jinja_utils.jinja_render(path, config['render'],
|
||||
[copy_sources, image_spec])
|
||||
|
||||
return content, sources, parent[0] if parent else None
|
||||
|
||||
|
||||
def prepare_source(source_name, name, dest_dir, config):
|
||||
|
|
@ -84,19 +93,16 @@ def find_dockerfiles(repository_name, match=True):
|
|||
dockerfiles = {}
|
||||
repository_dir = os.path.join(CONF.repositories.path, repository_name)
|
||||
|
||||
namespace = CONF.images.namespace
|
||||
if CONF.builder.push and CONF.registry.address:
|
||||
namespace = '%s/%s' % (CONF.registry.address, namespace)
|
||||
|
||||
for root, __, files in os.walk(repository_dir):
|
||||
if 'Dockerfile.j2' in files:
|
||||
path = os.path.join(root, 'Dockerfile.j2')
|
||||
else:
|
||||
continue
|
||||
name = os.path.basename(os.path.dirname(path))
|
||||
spec = images.image_spec(name, add_address=CONF.builder.push)
|
||||
dockerfiles[name] = {
|
||||
'name': name,
|
||||
'full_name': '%s/%s:%s' % (namespace, name, CONF.images.tag),
|
||||
'full_name': spec,
|
||||
'path': path,
|
||||
'parent': None,
|
||||
'children': [],
|
||||
|
|
@ -120,44 +126,25 @@ def find_dockerfiles(repository_name, match=True):
|
|||
|
||||
def render_dockerfiles(dockerfiles, config):
|
||||
for dockerfile in dockerfiles.values():
|
||||
content, sources = \
|
||||
content, sources, parent = \
|
||||
render_dockerfile(dockerfile['path'], dockerfile['name'], config)
|
||||
dockerfile['content'] = content
|
||||
dockerfile['sources'] = sources
|
||||
dockerfile['parent'] = parent
|
||||
|
||||
|
||||
IMAGE_FULL_NAME_RE = r"((?P<namespace>[\w:\.-]+)/){0,2}" \
|
||||
"(?P<name>[\w_-]+)" \
|
||||
"(:(?P<tag>[\w_\.-]+))?"
|
||||
IMAGE_FULL_NAME_PATTERN = re.compile(IMAGE_FULL_NAME_RE)
|
||||
# This regex is needed for matching not yet rendered images
|
||||
NOT_RENDERED_IMAGE_PATTERN = (r"((?P<namespace>[\w:\.\-}{ ]+)/){0,2}"
|
||||
r"(?P<name>[\w_\-}{ ]+)"
|
||||
r"(:(?P<tag>[\w_\.\-}{ ]+))?")
|
||||
|
||||
DOCKER_FILE_FROM_PATTERN = re.compile(
|
||||
r"^\s?FROM\s+{}\s?$".format(NOT_RENDERED_IMAGE_PATTERN), re.MULTILINE
|
||||
)
|
||||
|
||||
|
||||
def find_dependencies(dockerfiles):
|
||||
for name, dockerfile in dockerfiles.items():
|
||||
with open(dockerfile['path']) as f:
|
||||
content = f.read()
|
||||
|
||||
matcher = DOCKER_FILE_FROM_PATTERN.search(content)
|
||||
if not matcher:
|
||||
raise RuntimeError(
|
||||
"FROM clause was not found in dockerfile for image: " + name
|
||||
)
|
||||
|
||||
parent_ns = matcher.group("namespace")
|
||||
if not parent_ns:
|
||||
continue
|
||||
parent_name = matcher.group("name")
|
||||
|
||||
dockerfile['parent'] = dockerfiles[parent_name]
|
||||
dockerfiles[parent_name]['children'].append(dockerfile)
|
||||
def connect_children(dockerfiles):
|
||||
for dockerfile in dockerfiles.values():
|
||||
parent = dockerfile['parent']
|
||||
if parent:
|
||||
dockerfiles[parent]['children'].append(dockerfile)
|
||||
dockerfile['parent'] = dockerfiles[parent]
|
||||
|
||||
|
||||
def build_dockerfile(dc, dockerfile):
|
||||
|
|
@ -381,7 +368,7 @@ def build_components(components=None):
|
|||
find_dockerfiles(repository_def['name'], match=match))
|
||||
|
||||
render_dockerfiles(dockerfiles, config)
|
||||
find_dependencies(dockerfiles)
|
||||
connect_children(dockerfiles)
|
||||
|
||||
ready_images = get_ready_image_names()
|
||||
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ DEFAULTS = {
|
|||
'base_distro': 'debian',
|
||||
'base_tag': 'jessie',
|
||||
'maintainer': 'MOS Microservices <mos-microservices@mirantis.com>',
|
||||
'image_specs': {},
|
||||
},
|
||||
}
|
||||
|
||||
|
|
@ -18,6 +19,34 @@ SCHEMA = {
|
|||
'base_distro': {'type': 'string'},
|
||||
'base_tag': {'type': 'string'},
|
||||
'maintainer': {'type': 'string'},
|
||||
'image_specs': {
|
||||
'type': 'object',
|
||||
'additionalProperties': {
|
||||
'type': 'object',
|
||||
'additionalProperties': False,
|
||||
'properties': {
|
||||
'tag': {'type': 'string'},
|
||||
'namespace': {'type': 'string'},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def image_spec(image_name, add_address=True):
|
||||
from fuel_ccp import config
|
||||
CONF = config._REAL_CONF
|
||||
spec = {
|
||||
'name': image_name,
|
||||
'namespace': CONF.images.namespace,
|
||||
'tag': CONF.images.tag,
|
||||
}
|
||||
image_spec = CONF.images.image_specs.get(image_name)
|
||||
if image_spec:
|
||||
spec.update(image_spec._items())
|
||||
spec_str = '{namespace}/{name}:{tag}'.format(**spec)
|
||||
if add_address and CONF.registry.address:
|
||||
spec_str = '{}/{}'.format(CONF.registry.address, spec_str)
|
||||
return spec_str
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import json
|
|||
|
||||
from fuel_ccp.common import utils
|
||||
from fuel_ccp import config
|
||||
from fuel_ccp.config import images
|
||||
|
||||
|
||||
CONF = config.CONF
|
||||
|
|
@ -16,14 +17,6 @@ ENTRYPOINT_PATH = "/opt/ccp_start_script/bin/start_script.py"
|
|||
PYTHON_PATH = "/usr/bin/python"
|
||||
|
||||
|
||||
def _get_image_name(image_name):
|
||||
image_name = "%s/%s:%s" % (CONF.images.namespace, image_name,
|
||||
CONF.images.tag)
|
||||
if CONF.registry.address:
|
||||
image_name = "%s/%s" % (CONF.registry.address, image_name)
|
||||
return image_name
|
||||
|
||||
|
||||
def _get_start_cmd(role_name):
|
||||
return ["dumb-init", PYTHON_PATH, ENTRYPOINT_PATH, "provision", role_name]
|
||||
|
||||
|
|
@ -105,7 +98,7 @@ def serialize_env_variables(container):
|
|||
def serialize_daemon_container_spec(container):
|
||||
cont_spec = {
|
||||
"name": container["name"],
|
||||
"image": _get_image_name(container["image"]),
|
||||
"image": images.image_spec(container["image"]),
|
||||
"command": _get_start_cmd(container["name"]),
|
||||
"volumeMounts": serialize_volume_mounts(container),
|
||||
"readinessProbe": {
|
||||
|
|
@ -136,7 +129,7 @@ def serialize_daemon_container_spec(container):
|
|||
def serialize_job_container_spec(container, job):
|
||||
return {
|
||||
"name": job["name"],
|
||||
"image": _get_image_name(container["image"]),
|
||||
"image": images.image_spec(container["image"]),
|
||||
"command": _get_start_cmd(job["name"]),
|
||||
"volumeMounts": serialize_volume_mounts(container),
|
||||
"env": serialize_env_variables(container)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import collections
|
||||
import io
|
||||
import os
|
||||
|
||||
import fixtures
|
||||
|
|
@ -52,36 +51,6 @@ class TestBuild(base.TestCase):
|
|||
},)
|
||||
])
|
||||
|
||||
def test_find_dependencies_no_registry(self):
|
||||
m_open = mock.mock_open()
|
||||
m_open.side_effect = [
|
||||
io.StringIO(BASE_DOCKERFILE),
|
||||
io.StringIO(COMPONENT_DOCKERFILE.format(''))
|
||||
]
|
||||
dockerfiles = self.__create_dockerfile_objects()
|
||||
with mock.patch('fuel_ccp.build.open', m_open, create=True):
|
||||
build.find_dependencies(dockerfiles)
|
||||
|
||||
self.assertListEqual([dockerfiles['ms-mysql']],
|
||||
dockerfiles['ms-debian-base']['children'])
|
||||
self.assertDictEqual(dockerfiles['ms-debian-base'],
|
||||
dockerfiles['ms-mysql']['parent'])
|
||||
|
||||
def test_find_dependencies_registry(self):
|
||||
m_open = mock.mock_open()
|
||||
m_open.side_effect = [
|
||||
io.StringIO(BASE_DOCKERFILE),
|
||||
io.StringIO(COMPONENT_DOCKERFILE.format('example.com:8909/'))
|
||||
]
|
||||
dockerfiles = self.__create_dockerfile_objects()
|
||||
with mock.patch('fuel_ccp.build.open', m_open, create=True):
|
||||
build.find_dependencies(dockerfiles)
|
||||
|
||||
self.assertListEqual([dockerfiles['ms-mysql']],
|
||||
dockerfiles['ms-debian-base']['children'])
|
||||
self.assertDictEqual(dockerfiles['ms-debian-base'],
|
||||
dockerfiles['ms-mysql']['parent'])
|
||||
|
||||
@mock.patch("docker.Client")
|
||||
@mock.patch("fuel_ccp.build.build_dockerfile")
|
||||
@mock.patch("fuel_ccp.build.submit_dockerfile_processing")
|
||||
|
|
@ -266,18 +235,23 @@ class TestRenderDockerfile(testscenarios.WithScenarios, base.TestCase):
|
|||
('empty', {
|
||||
'config': {'render': {}},
|
||||
'source': '',
|
||||
'result': ('', set()),
|
||||
'result': ('', set(), None),
|
||||
}),
|
||||
('one_source', {
|
||||
'config': {'render': {}, 'sources': {'one': {}}},
|
||||
'source': '{{ copy_sources("one", "/tmp") }}',
|
||||
'result': ('COPY one /tmp', {'one'}),
|
||||
'result': ('COPY one /tmp', {'one'}, None),
|
||||
}),
|
||||
('wrong_source', {
|
||||
'config': {'render': {}, 'sources': {'one': {}}},
|
||||
'source': '{{ copy_sources("wrong", "/tmp") }}',
|
||||
'exception': Exception('No such source: wrong'),
|
||||
}),
|
||||
('one_from', {
|
||||
'config': {'render': {}},
|
||||
'source': 'FROM {{ image_spec("one") }}',
|
||||
'result': ('FROM ccp/one:latest', set(), 'one'),
|
||||
}),
|
||||
]
|
||||
|
||||
config = None
|
||||
|
|
|
|||
Loading…
Reference in New Issue