openstack
/
zun
Code Issues Proposed changes

Merge "Support to provide 'image_driver' during container create"

This commit is contained in:
Jenkins 2017-01-30 13:56:57 +00:00 committed by Gerrit Code Review
parent 9448e462e7 dbe6293022
commit 3b6070dc3f
23 changed files with 213 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
setup.cfg
View File

@ -64,5 +64,9 @@ zun.scheduler.driver =
chance_scheduler = zun.scheduler.chance_scheduler:ChanceScheduler
fake_scheduler = zun.tests.unit.scheduler.fake_scheduler:FakeScheduler
zun.image.driver =
glance = zun.image.glance.driver:GlanceDriver
docker = zun.image.docker.driver:DockerDriver
tempest.test_plugins =
zun_tests = zun.tests.tempest.plugin:ZunTempestPlugin

8
zun/api/controllers/v1/containers.py
View File

@ -202,13 +202,19 @@ class ContainersController(rest.RestController):
'"false", True, False, "True" and "False"')
raise exception.InvalidValue(msg)
# Valiadtion accepts 'None' so need to convert it to None
if container_dict.get('image_driver'):
container_dict['image_driver'] = api_utils.string_or_none(
container_dict.get('image_driver'))
# NOTE(mkrai): Intent here is to check the existence of image
# before proceeding to create container. If image is not found,
# container create will fail with 400 status.
images = compute_api.image_search(context, container_dict['image'],
container_dict.get('image_driver'),
True)
if not images:
raise exception.ImageNotFound(container_dict['image'])
raise exception.ImageNotFound(image=container_dict['image'])
container_dict['project_id'] = context.project_id
container_dict['user_id'] = context.user_id
name = container_dict.get('name') or \

8
zun/api/controllers/v1/images.py
View File

@ -117,7 +117,8 @@ class ImagesController(rest.RestController):
@pecan.expose('json')
@exception.wrap_pecan_controller_exception
def search(self, image, exact_match=False):
@validation.validate_query_param(pecan.request, schema.query_param_search)
def search(self, image, image_driver=None, exact_match=False):
context = pecan.request.context
policy.enforce(context, "image:search",
action="image:search")
@ -129,5 +130,10 @@ class ImagesController(rest.RestController):
msg = _("Valid exact_match values are true,"
" false, 0, 1, yes and no")
raise exception.InvalidValue(msg)
# Valiadtion accepts 'None' so need to convert it to None
if image_driver:
image_driver = api_utils.string_or_none(image_driver)
return pecan.request.compute_api.image_search(context, image,
image_driver,
exact_match)

1
zun/api/controllers/v1/schemas/containers.py
View File

@ -27,6 +27,7 @@ _container_properties = {
'restart_policy': parameter_types.restart_policy,
'tty': parameter_types.boolean,
'stdin_open': parameter_types.boolean,
'image_driver': parameter_types.image_driver
}
container_create = {

9
zun/api/controllers/v1/schemas/images.py
View File

@ -25,3 +25,12 @@ image_create = {
'required': ['repo'],
'additionalProperties': False
}
query_param_search = {
'type': 'object',
'properties': {
'image_driver': parameter_types.image_driver,
'exact_match': parameter_types.boolean
},
'additionalProperties': False
}

3
zun/api/controllers/v1/views/containers_view.py
View File

@ -37,7 +37,8 @@ _basic_keys = (
'restart_policy',
'status_detail',
'tty',
'stdin_open'
'stdin_open',
'image_driver'
)

7
zun/api/utils.py
View File

@ -33,6 +33,13 @@ JSONPATCH_EXCEPTIONS = (jsonpatch.JsonPatchException,
DOCKER_MINIMUM_MEMORY = 4 * 1024 * 1024
def string_or_none(value):
if value in [None, 'None']:
return None
else:
return value
def validate_limit(limit):
try:
if limit is not None and int(limit) <= 0:

11
zun/common/validation/parameter_types.py
View File

@ -11,7 +11,13 @@
# under the License.
import copy
import zun.conf
CONF = zun.conf.CONF
image_driver_list = [driver for driver in CONF.image_driver_list]
image_driver_list_with_none = image_driver_list + [None, 'None']
non_negative_integer = {
'type': ['integer', 'string'],
@ -31,6 +37,11 @@ boolean = {
'enum': [True, 'True', 'true', False, 'False', 'false'],
}
image_driver = {
'type': ['string', 'null'],
'enum': image_driver_list_with_none
}
container_name = {
'type': ['string', 'null'],
'minLength': 2,

4
zun/compute/api.py
View File

@ -90,5 +90,5 @@ class API(object):
def image_pull(self, context, image, *args):
return self.rpcapi.image_pull(context, image, *args)
def image_search(self, context, image, *args):
return self.rpcapi.image_search(context, image, *args)
def image_search(self, context, image, image_driver, *args):
return self.rpcapi.image_search(context, image, image_driver, *args)

23
zun/compute/manager.py
View File

@ -21,11 +21,12 @@ from zun.common import exception
from zun.common.i18n import _LE
from zun.common import utils
from zun.common.utils import translate_exception
import zun.conf
from zun.container import driver
from zun.image import driver as image_driver
from zun.objects import fields
CONF = zun.conf.CONF
LOG = logging.getLogger(__name__)
@ -67,10 +68,14 @@ class Manager(object):
container.task_state = fields.TaskState.SANDBOX_CREATING
container.save(context)
sandbox_id = None
sandbox_image = 'kubernetes/pause'
sandbox_image = CONF.sandbox_image
sandbox_image_driver = CONF.sandbox_image_driver
sandbox_image_pull_policy = CONF.sandbox_image_pull_policy
repo, tag = utils.parse_image_name(sandbox_image)
try:
image = image_driver.pull_image(context, repo, tag, 'ifnotpresent')
image = image_driver.pull_image(context, repo, tag,
sandbox_image_pull_policy,
sandbox_image_driver)
sandbox_id = self.driver.create_sandbox(context, container,
image=sandbox_image)
except Exception as e:
@ -86,9 +91,11 @@ class Manager(object):
repo, tag = utils.parse_image_name(container.image)
image_pull_policy = utils.get_image_pull_policy(
container.image_pull_policy, tag)
image_driver_name = container.image_driver
try:
image = image_driver.pull_image(context, repo,
tag, image_pull_policy)
image = image_driver.pull_image(context, repo, tag,
image_pull_policy,
image_driver_name)
except exception.ImageNotFound as e:
with excutils.save_and_reraise_exception(reraise=reraise):
LOG.error(six.text_type(e))
@ -112,6 +119,7 @@ class Manager(object):
return
container.task_state = fields.TaskState.CONTAINER_CREATING
container.image_driver = image.get('driver')
container.save(context)
try:
container = self.driver.create(context, container,
@ -372,10 +380,11 @@ class Manager(object):
raise
@translate_exception
def image_search(self, context, image, exact_match):
def image_search(self, context, image, image_driver_name, exact_match):
LOG.debug('Searching image...', image=image)
try:
return image_driver.search_image(context, image, exact_match)
return image_driver.search_image(context, image,
image_driver_name, exact_match)
except Exception as e:
LOG.exception(_LE("Unexpected exception while searching "
"image: %s"), six.text_type(e))

3
zun/compute/rpcapi.py
View File

@ -89,10 +89,11 @@ class API(rpc_service.API):
host = None
self._cast(host, 'image_pull', image=image)
def image_search(self, context, image, exact_match):
def image_search(self, context, image, image_driver, exact_match):
# NOTE(hongbin): Image API doesn't support multiple compute nodes
# scenario yet, so we temporarily set host to None and rpc will
# choose an arbitrary host.
host = None
return self._call(host, 'image_search', image=image,
image_driver_name=image_driver,
exact_match=exact_match)

24
zun/conf/image_driver.py
View File

@ -18,11 +18,11 @@ from zun.conf import path
image_driver_opts = [
cfg.ListOpt(
'image_driver_list',
default=['glance.driver.GlanceDriver', 'docker.driver.DockerDriver'],
default=['glance', 'docker'],
help="""Defines the list of image driver to use for downloading image.
Possible values:
* ``docker.driver.DockerDriver``
* ``glance.driver.GlanceDriver``
* ``docker``
* ``glance``
Services which consume this:
* ``zun-compute``
Interdependencies to other options:
@ -30,6 +30,21 @@ Interdependencies to other options:
""")
]
sandbox_opts = [
cfg.StrOpt(
'sandbox_image',
default='kubernetes/pause',
help='Container image for sandbox container.'),
cfg.StrOpt(
'sandbox_image_driver',
default='docker',
help='Image driver for sandbox container.'),
cfg.StrOpt(
'sandbox_image_pull_policy',
default='ifnotpresent',
help='Image pull policy for sandbox image.'),
]
glance_driver_opts = [
cfg.StrOpt(
'images_directory',
@ -42,13 +57,14 @@ glance_driver_opts = [
glance_opt_group = cfg.OptGroup(name='glance',
title='Glance options for image management')
ALL_OPTS = (glance_driver_opts + image_driver_opts)
ALL_OPTS = (glance_driver_opts + image_driver_opts + sandbox_opts)
def register_opts(conf):
conf.register_group(glance_opt_group)
conf.register_opts(glance_driver_opts, group=glance_opt_group)
conf.register_opts(image_driver_opts)
conf.register_opts(sandbox_opts)
def list_opts():

35
zun/db/sqlalchemy/alembic/versions/5458f8394206_add_image_driver_field.py Normal file
View File

@ -0,0 +1,35 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add image driver field
Revision ID: 5458f8394206
Revises: d1ef05fd92c8
Create Date: 2017-01-25 19:01:46.033461
"""
# revision identifiers, used by Alembic.
revision = '5458f8394206'
down_revision = 'd1ef05fd92c8'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('container',
sa.Column('image_driver', sa.Text(),
nullable=True))

1
zun/db/sqlalchemy/models.py
View File

@ -150,6 +150,7 @@ class Container(Base):
status_detail = Column(String(50))
tty = Column(Boolean, default=False)
stdin_open = Column(Boolean, default=False)
image_driver = Column(String(255))
class Image(Base):

28
zun/image/driver.py
View File

@ -16,7 +16,7 @@ import six
import sys
from oslo_log import log as logging
from oslo_utils import importutils
import stevedore
from zun.common import exception
from zun.common.i18n import _
@ -45,26 +45,34 @@ def load_image_driver(image_driver=None):
LOG.info(_LI("Loading container image driver '%s'"), image_driver)
try:
driver = importutils.import_object(
'zun.image.%s' % image_driver)
driver = stevedore.driver.DriverManager(
"zun.image.driver",
image_driver,
invoke_on_load=True).driver
if not isinstance(driver, ContainerImageDriver):
raise Exception(_('Expected driver of type: %s') %
str(ContainerImageDriver))
return driver
except ImportError:
except Exception:
LOG.exception(_LE("Unable to load the container image driver"))
sys.exit(1)
def pull_image(context, repo, tag, image_pull_policy):
image_driver_list = CONF.image_driver_list
def pull_image(context, repo, tag, image_pull_policy, image_driver):
if image_driver:
image_driver_list = [image_driver.lower()]
else:
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
image = image_driver.pull_image(context, repo,
tag, image_pull_policy)
if image:
image['driver'] = driver.split('.')[0]
break
except exception.ImageNotFound:
image = None
@ -77,10 +85,14 @@ def pull_image(context, repo, tag, image_pull_policy):
return image
def search_image(context, image_name, exact_match):
def search_image(context, image_name, image_driver, exact_match):
images = []
repo, tag = parse_image_name(image_name)
for driver in CONF.image_driver_list:
if image_driver:
image_driver_list = [image_driver.lower()]
else:
image_driver_list = CONF.image_driver_list
for driver in image_driver_list:
try:
image_driver = load_image_driver(driver)
imgs = image_driver.search_image(context, repo, tag,

1
zun/image/glance/driver.py
View File

@ -98,7 +98,6 @@ class GlanceDriver(driver.ContainerImageDriver):
try:
# TODO(hongbin): find image by both repo and tag
images = utils.find_images(context, repo, exact_match)
LOG.debug('Image %s was found in glance' % repo)
return images
except Exception as e:
raise exception.ZunException(six.text_type(e))

4
zun/objects/container.py
View File

@ -30,7 +30,8 @@ class Container(base.ZunPersistentObject, base.ZunObject):
# Version 1.8: Add restart_policy
# Version 1.9: Add status_detail column
# Version 1.10: Add tty, stdin_open
VERSION = '1.10'
# Version 1.11: Add image_driver
VERSION = '1.11'
fields = {
'id': fields.IntegerField(),
@ -59,6 +60,7 @@ class Container(base.ZunPersistentObject, base.ZunObject):
'status_detail': fields.StringField(nullable=True),
'tty': fields.BooleanField(nullable=True),
'stdin_open': fields.BooleanField(nullable=True),
'image_driver': fields.StringField(nullable=True)
}
@staticmethod

3
zun/tests/tempest/api/common/datagen.py
View File

@ -53,7 +53,8 @@ def container_data(**kwargs):
'image': 'cirros:latest',
'command': 'sleep 10000',
'memory': '100',
'environment': {}
'environment': {},
'image_driver': 'docker'
}
data.update(kwargs)

17
zun/tests/unit/api/controllers/v1/test_containers.py
View File

@ -1101,6 +1101,23 @@ class TestContainerController(api_base.FunctionalTest):
'/v1/containers/%s/%s/' % (container_uuid, 'kill'))
self.assertTrue(mock_container_kill.called)
@patch('zun.compute.api.API.container_create')
@patch('zun.compute.api.API.image_search')
def test_create_container_resp_has_image_driver(self, mock_search,
mock_container_create):
mock_container_create.side_effect = lambda x, y: y
# Create a container with a command
params = ('{"name": "MyDocker", "image": "ubuntu",'
'"command": "env", "memory": "512",'
'"environment": {"key1": "val1", "key2": "val2"},'
'"image_driver": "glance"}')
response = self.app.post('/v1/containers/',
params=params,
content_type='application/json')
self.assertEqual(202, response.status_int)
self.assertIn('image_driver', response.json.keys())
self.assertEqual('glance', response.json.get('image_driver'))
class TestContainerEnforcement(api_base.FunctionalTest):

29
zun/tests/unit/api/controllers/v1/test_images.py
View File

@ -138,7 +138,7 @@ class TestImageController(api_base.FunctionalTest):
response = self.app.get('/v1/images/redis/search/')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', None, False)
@patch('zun.compute.api.API.image_search')
def test_search_image_with_tag(self, mock_image_search):
@ -146,37 +146,46 @@ class TestImageController(api_base.FunctionalTest):
response = self.app.get('/v1/images/redis:test/search/')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis:test', False)
mock.ANY, 'redis:test', None, False)
@patch('zun.compute.api.API.image_search')
def test_search_image_not_found(self, mock_image_search):
mock_image_search.side_effect = exception.ImageNotFound
self.assertRaises(AppError, self.app.get, '/v1/images/redis/search/')
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', None, False)
@patch('zun.compute.rpcapi.API.image_search')
def test_search_image_with_exact_match_true(self, mock_image_search):
mock_image_search.return_value = {'name': 'redis', 'stars': 2000}
response = self.app.get('/v1/images/redis/search?exact_match=true')
response = self.app.get(
'/v1/images/redis/search?exact_match=true&image_driver=docker')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', True)
mock.ANY, 'redis', 'docker', True)
@patch('zun.compute.rpcapi.API.image_search')
def test_search_image_with_exact_match_false(self, mock_image_search):
mock_image_search.return_value = {'name': 'redis', 'stars': 2000}
response = self.app.get('/v1/images/redis/search?exact_match=false')
response = self.app.get(
'/v1/images/redis/search?exact_match=false&image_driver=glance')
self.assertEqual(200, response.status_int)
mock_image_search.assert_called_once_with(
mock.ANY, 'redis', False)
mock.ANY, 'redis', 'glance', False)
@patch('zun.compute.api.API.image_search')
def test_search_image_with_exact_match_wrong(self, mock_image_search):
mock_image_search.side_effect = exception.InvalidValue
self.assertRaises(AppError, self.app.get,
'/v1/images/redis/search?exact_match=wrong')
self.assertTrue(mock_image_search.not_called)
with self.assertRaisesRegexp(AppError,
"Invalid input for query parameters"):
self.app.get('/v1/images/redis/search?exact_match=wrong')
@patch('zun.compute.api.API.image_search')
def test_search_image_with_image_driver_wrong(self, mock_image_search):
mock_image_search.side_effect = exception.InvalidValue
with self.assertRaisesRegexp(AppError,
"Invalid input for query parameters"):
self.app.get('/v1/images/redis/search?image_driver=wrong')
class TestImageEnforcement(api_base.FunctionalTest):

16
zun/tests/unit/common/test_validations.py
View File

@ -28,7 +28,8 @@ CONTAINER_CREATE = {
'image_pull_policy': parameter_types.image_pull_policy,
'labels': parameter_types.labels,
'environment': parameter_types.environment,
'restart_policy': parameter_types.restart_policy
'restart_policy': parameter_types.restart_policy,
'image_driver': parameter_types.image_driver
},
'required': ['image'],
'additionalProperties': False,
@ -48,7 +49,8 @@ class TestSchemaValidations(base.BaseTestCase):
'labels': {'abc': 12, 'bcd': 'xyz'},
'environment': {'xyz': 'pqr', 'pqr': 2},
'restart_policy': {'Name': 'no',
'MaximumRetryCount': '0'}}
'MaximumRetryCount': '0'},
'image_driver': 'docker'}
self.schema_validator.validate(request_to_validate)
def test_create_schema_with_all_parameters_none(self):
@ -58,7 +60,8 @@ class TestSchemaValidations(base.BaseTestCase):
'image_pull_policy': None,
'labels': None,
'environment': None,
'restart_policy': None
'restart_policy': None,
'image_driver': None
}
self.schema_validator.validate(request_to_validate)
@ -154,3 +157,10 @@ class TestSchemaValidations(base.BaseTestCase):
with self.assertRaisesRegexp(exception.SchemaValidationError,
"'Name' is a required property"):
self.schema_validator.validate(request_to_validate)
def test_create_schema_wrong_image_driver(self):
request_to_validate = {'image_driver': 'xyz', 'image': 'nginx'}
with self.assertRaisesRegexp(exception.SchemaValidationError,
"Invalid input for field"
" 'image_driver'"):
self.schema_validator.validate(request_to_validate)

24
zun/tests/unit/compute/test_compute_manager.py
View File

@ -50,14 +50,15 @@ class TestManager(base.TestCase):
def test_container_create(self, mock_create_sandbox, mock_create,
mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.return_value = 'fake_path'
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_create_sandbox.return_value = 'fake_id'
self.compute_manager._do_container_create(self.context, container)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container,
'fake_id', 'fake_path')
'fake_id', image)
@mock.patch.object(Container, 'save')
@mock.patch.object(fake_driver, 'create_sandbox')
@ -109,6 +110,8 @@ class TestManager(base.TestCase):
mock_create, mock_pull,
mock_save):
container = Container(self.context, **utils.get_test_container())
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_pull.return_value = image
mock_create.side_effect = exception.DockerError("Creation Failed")
mock_create_sandbox.return_value = mock.MagicMock()
self.compute_manager._do_container_create(self.context, container)
@ -122,15 +125,16 @@ class TestManager(base.TestCase):
def test_container_run(self, mock_start,
mock_create, mock_pull, mock_save):
container = Container(self.context, **utils.get_test_container())
mock_pull.return_value = 'fake_path'
image = {'image': 'repo', 'path': 'out_path', 'driver': 'glance'}
mock_create.return_value = container
mock_pull.return_value = image
container.status = 'Stopped'
self.compute_manager._do_container_run(self.context, container)
mock_save.assert_called_with(self.context)
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container,
None, 'fake_path')
None, image)
mock_start.assert_called_once_with(container)
@mock.patch.object(Container, 'save')
@ -147,7 +151,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -163,7 +167,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Image Not Found')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -179,7 +183,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Docker Error occurred')
mock_pull.assert_called_once_with(self.context, 'kubernetes/pause',
'latest', 'ifnotpresent')
'latest', 'ifnotpresent', 'docker')
@mock.patch.object(Container, 'save')
@mock.patch('zun.image.driver.pull_image')
@ -198,7 +202,7 @@ class TestManager(base.TestCase):
mock_fail.assert_called_with(self.context,
container, 'Docker Error occurred')
mock_pull.assert_any_call(self.context, container.image, 'latest',
'always')
'always', 'glance')
mock_create.assert_called_once_with(self.context, container, None,
{'name': 'nginx', 'path': None})

1
zun/tests/unit/db/utils.py
View File

@ -62,6 +62,7 @@ def get_test_container(**kw):
'status_detail': kw.get('status_detail', 'up from 5 hours'),
'tty': kw.get('tty', True),
'stdin_open': kw.get('stdin_open', True),
'image_driver': 'glance'
}