Merge "Implement Image synchronization in Kingbird."

This commit is contained in:
Jenkins 2017-07-19 14:50:58 +00:00 committed by Gerrit Code Review
commit 6264342d2b
11 changed files with 1020 additions and 63 deletions

View File

@ -98,6 +98,14 @@ class JobNotFound(NotFound):
message = _("Job doesn't exist.")
class DependentImageNotFound(NotFound):
message = _("Dependent image doesn't exist.")
class ImageFormatNotSupported(KingbirdException):
message = _("An invalid version was provided")
class ConnectionRefused(KingbirdException):
message = _("Connection to the service endpoint is refused")

View File

@ -0,0 +1,63 @@
# Copyright 2017 Ericsson AB.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
from kingbird.common import exceptions
from kingbird.drivers.openstack.glance_v2 import GlanceClient
LOG = log.getLogger(__name__)
API_VERSION = '2'
def check_dependent_images(context, region, image_id):
"""Check Dependent Images for a given image.
:param region: source_region
:param image_id: ID of the image to get dependent image details.
"""
source_glance_client = GlanceClient(region, context)
source_image = source_glance_client.get_image(image_id)
if source_image.disk_format == 'ami':
try:
kernel_image = source_glance_client.\
get_image(source_image.kernel_id)
kernel_disk_format = getattr(kernel_image, 'disk_format', None)
if kernel_disk_format is None or kernel_disk_format != 'aki':
raise exceptions.DependentImageNotFound()
LOG.info('Kernel Image Found: %(kernel_image)s in %(region)s'
% {'kernel_image': kernel_image.id,
'region': region})
ramdisk_image = source_glance_client.\
get_image(source_image.ramdisk_id)
ramdisk_disk_format = getattr(ramdisk_image, 'disk_format', None)
if ramdisk_disk_format is None or ramdisk_disk_format != 'ari':
raise exceptions.DependentImageNotFound()
LOG.info('ram disk Image Found: %(ramdisk_image)s in %(region)s'
% {'ramdisk_image': ramdisk_image.id,
'region': region})
except exceptions.DependentImageNotFound():
raise
return {
'kernel_image': kernel_image,
'ramdisk_image': ramdisk_image
}
elif source_image.disk_format in ['qcow2', 'aki', 'ari']:
return None
else:
raise exceptions.ImageFormatNotSupported()

View File

@ -12,6 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_log import log
import six
from kingbird.common import exceptions
from kingbird.drivers.openstack.keystone_v3 import KeystoneClient
@ -75,6 +76,7 @@ class GlanceClient(object):
have to be retrieved.
"""
LOG.info("Checking for image in source_region.")
try:
image = self.glance_client.images.get(resource_identifier)
LOG.info("Source image: %s", image.name)
@ -83,3 +85,81 @@ class GlanceClient(object):
LOG.error('Exception Occurred: Source Image %s not available.',
resource_identifier)
pass
def get_image(self, resource_identifier):
"""Get the image details for the specified resource_identifier.
:param resource_identifier: resource_id for which the details
have to be retrieved.
"""
LOG.info("Fetching image: %s", resource_identifier)
return self.glance_client.images.get(resource_identifier)
def get_image_data(self, resource_identifier):
"""Get the image data of the specified resource.
:param resource_identifier: resource_id for which the details
have to be retrieved.
"""
LOG.info("Fetching data: %s", resource_identifier)
return self.glance_client.images.data(resource_identifier)
def create_image(self, image, force, kernel_image=None,
ramdisk_image=None):
"""Create image with the same properties of the source image.
:param kwargs: properties of the image to create in target
region.
"""
kwargs = {}
fields_after_creation = ['status', 'created_at', 'size',
'updated_at', 'file', 'checksum',
'virtual_size', 'schema']
# split out the usual key and the properties which are top-level
for key in six.iterkeys(image):
if key not in fields_after_creation:
kwargs[key] = image.get(key)
LOG.warning('Image with id same as that of source_image will be'
' created.If any issue while syncing an image'
' It is because the id already has an entry in'
' target region use --force to avoid this issue.')
if force:
LOG.info("Image with a new-id is created.")
kwargs.pop('id')
if kernel_image:
kwargs["kernel-id"] = kernel_image
if ramdisk_image:
kwargs["ramdisk-id"] = ramdisk_image
LOG.info("Creating Image: %s", image.name)
return self.glance_client.images.create(**kwargs)
def image_upload(self, image_id, image_data):
"""Upload image to glance.
:param image_id: UUID of the image to be uploaded.
"""
LOG.info("Uploading image: %s", image_id)
return self.glance_client.images.upload(image_id, image_data)
class GlanceUpload(object):
def __init__(self, data):
self.received = data
def read(self, chunk_size):
"""Replace the actual read method in glance.
Please note that we are using this to replace the actual
read in glance.
when we download an image a chunk of 65536 will be written into
the destination file and
when we upload an image using a file it reads the entire file
in the form of chunks(65536 KB). So our approach is to get
entire imagedata into an iterator and send this 65536kb chunk to
the glance image upload and there by omitting the usage of file.
"""
return self.received.next()

View File

@ -0,0 +1,209 @@
# Copyright 2017 Ericsson AB.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import threading
from oslo_log import log as logging
from kingbird.common import consts
from kingbird.common import exceptions
from kingbird.db.sqlalchemy import api as db_api
from kingbird.drivers.openstack import glance_adapter
from kingbird.drivers.openstack.glance_v2 import GlanceClient
from kingbird.drivers.openstack.glance_v2 import GlanceUpload
LOG = logging.getLogger(__name__)
class ImageSyncManager(object):
"""Manages tasks related to resource management."""
def __init__(self, *args, **kwargs):
super(ImageSyncManager, self).__init__()
def create_resources_in_region(self, job_id, target_regions,
source_region, context, resource, force):
"""Create Region Specific threads."""
regions_thread = list()
for region in target_regions:
thread = threading.Thread(target=self.create_resources,
args=(job_id, region, source_region,
context, resource, force))
regions_thread.append(thread)
thread.start()
for region_thread in regions_thread:
region_thread.join()
def create_resources(self, job_id, region, source_region, context,
resource, force):
"""Check dependent images and create resources in target regions."""
source_glance_client = GlanceClient(source_region, context)
target_glance_client = GlanceClient(region, context)
dependent_images = glance_adapter.check_dependent_images(
context, source_region, resource)
if dependent_images is not None:
result = self.create_dependent_image(
resource, dependent_images, target_glance_client,
source_glance_client, region, force)
self.update_result_in_database(context, job_id, region, resource,
result)
else:
result = self.create_independent_image(
resource, target_glance_client, source_glance_client,
region, force)
self.update_result_in_database(context, job_id, region, resource,
result)
def update_result_in_database(self, context, job_id, region, resource,
result):
"""Update result in database based on the sync operation."""
job_result = consts.JOB_SUCCESS if result else consts.JOB_FAILURE
try:
db_api.resource_sync_update(context, job_id, region,
resource, job_result)
except exceptions.JobNotFound():
raise
pass
def create_dependent_image(self, resource, dependent_images,
target_client, source_client, region, force):
"""Create dependent images along with base image.
Base image here is Amazon Machine Image(AMI) and
Dependent images are Amazon Kernel Image(AKI),
Amazon Ramdisk Image(ARI).
:param resource: Resource to be synced.
:param dependent_images: Dependent images for the base image.
:param target_client: Glance client object for the target_region.
:param source_client: Glance client object for source_region.
:param region: Target region in which resource has to be synced.
:param force: Default force option is False. If '--force'
is given then force is set to True.
"""
try:
kernel_image = dependent_images['kernel_image']
ramdisk_image = dependent_images['ramdisk_image']
source_image = source_client.get_image(resource)
# Create images in target regions.
target_kernel_image = target_client.\
create_image(kernel_image, force)
target_ramdisk_image = target_client.\
create_image(ramdisk_image, force)
target_source_image = target_client.\
create_image(source_image, force, target_kernel_image.id,
target_ramdisk_image.id)
# Fetch and Upload image into glance.
# Kernel Image upload.
kernel_image_data = source_client.\
get_image_data(kernel_image.id)
upload_kernel_image = GlanceUpload(kernel_image_data)
target_client.image_upload(target_kernel_image.id,
upload_kernel_image)
LOG.info('Kernel_image %(image)s uploaded in %(region)s'
% {'image': kernel_image.id, 'region': region})
# Ramdisk image upload.
ramdisk_image_data = source_client.\
get_image_data(ramdisk_image.id)
upload_ram_disk_image = GlanceUpload(ramdisk_image_data)
target_client.image_upload(target_ramdisk_image.id,
upload_ram_disk_image)
LOG.info('ramdisk_image %(image)s uploaded in %(region)s'
% {'image': ramdisk_image.id, 'region': region})
# Base 'AMI' image upload.
source_image_data = source_client.get_image_data(source_image.id)
upload_source_image = GlanceUpload(source_image_data)
target_client.image_upload(target_source_image.id,
upload_source_image)
LOG.info('source_image %(image)s uploaded in %(region)s'
% {'image': source_image.id, 'region': region})
return True
except Exception as exc:
LOG.error('Exception Occurred: %(msg)s in %(region)s'
% {'msg': exc.message, 'region': region})
return False
def create_independent_image(self, resource, target_client,
source_client, region, force):
"""Create independent images.
Base image here is Qcow2.
:param resource: Resource to be synced.
:param target_client: Glance client object for the target_region.
:param source_client: Glance client object for source_region.
:param region: Target region in which resource has to be synced.
:param force: Default force option is False. If '--force'
is given then force is set to True.
"""
try:
source_image = source_client.get_image(resource)
target_source_image = target_client.create_image(source_image,
force)
source_image_data = source_client.get_image_data(source_image.id)
upload_source_image = GlanceUpload(source_image_data)
target_client.image_upload(target_source_image.id,
upload_source_image)
LOG.info('source_image %(image)s uploaded in %(region)s'
% {'image': source_image.id, 'region': region})
return True
except Exception as exc:
LOG.error('Exception Occurred: %(msg)s in %(region)s'
% {'msg': exc.message, 'region': region})
return False
def resource_sync(self, context, job_id, payload):
"""Create resources in target regions.
Image with same id is created in target_regions and therefore
if a user wants to syncs the same resource as the ID is already
used glance throws 409 error in order to avoid that we use --force
and set force flag to true and there by creates resource without
fail.
:param context: request context object.
:param job_id: ID of the job which triggered image_sync.
:payload: request payload.
"""
LOG.info('Triggered image sync.')
images_thread = list()
target_regions = payload['target']
force = eval(str(payload.get('force', False)))
resource_ids = payload.get('resources')
source_region = payload['source']
for resource in resource_ids:
thread = threading.Thread(target=self.create_resources_in_region,
args=(job_id, target_regions,
source_region, context,
resource, force))
images_thread.append(thread)
thread.start()
for image_thread in images_thread:
image_thread.join()
try:
resource_sync_details = db_api.\
resource_sync_status(context, job_id)
except exceptions.JobNotFound:
raise
result = consts.JOB_SUCCESS
if consts.JOB_FAILURE in resource_sync_details:
result = consts.JOB_FAILURE
try:
db_api.sync_job_update(context, job_id, result)
except exceptions.JobNotFound:
raise

View File

@ -26,17 +26,18 @@ from kingbird.drivers.openstack.nova_v2 import NovaClient
LOG = logging.getLogger(__name__)
class SyncManager(object):
"""Manages tasks related to resource management"""
class KeypairSyncManager(object):
"""Manages tasks related to resource management."""
def __init__(self, *args, **kwargs):
super(SyncManager, self).__init__()
super(KeypairSyncManager, self).__init__()
def _create_keypairs_in_region(self, job_id, force, target_regions,
def create_resources_in_region(self, job_id, force, target_regions,
source_keypair, session, context):
"""Create Region specific threads."""
regions_thread = list()
for region in target_regions:
thread = threading.Thread(target=self._create_keypairs,
thread = threading.Thread(target=self.create_resources,
args=(job_id, force, region,
source_keypair, session,
context))
@ -45,8 +46,9 @@ class SyncManager(object):
for region_thread in regions_thread:
region_thread.join()
def _create_keypairs(self, job_id, force, region, source_keypair,
def create_resources(self, job_id, force, region, source_keypair,
session, context):
"""Create resources using threads."""
target_nova_client = NovaClient(region, session)
try:
target_nova_client.create_keypairs(force, source_keypair)
@ -69,7 +71,14 @@ class SyncManager(object):
raise
pass
def keypair_sync_for_user(self, context, job_id, payload):
def resource_sync(self, context, job_id, payload):
"""Create resources in target regions.
:param context: request context object.
:param job_id: ID of the job which triggered image_sync.
:payload: request payload.
"""
LOG.info("Triggered Keypair Sync.")
keypairs_thread = list()
target_regions = payload['target']
force = eval(str(payload.get('force', False)))
@ -81,7 +90,7 @@ class SyncManager(object):
source_nova_client = NovaClient(source_region, session)
for keypair in resource_ids:
source_keypair = source_nova_client.get_keypairs(keypair)
thread = threading.Thread(target=self._create_keypairs_in_region,
thread = threading.Thread(target=self.create_resources_in_region,
args=(job_id, force, target_regions,
source_keypair, session,
context,))
@ -89,8 +98,6 @@ class SyncManager(object):
thread.start()
for keypair_thread in keypairs_thread:
keypair_thread.join()
# Update the parent_db after the sync
try:
resource_sync_details = db_api.\
resource_sync_status(context, job_id)

View File

@ -23,9 +23,10 @@ from kingbird.common import context
from kingbird.common import exceptions
from kingbird.common.i18n import _
from kingbird.common import messaging as rpc_messaging
from kingbird.engine.image_sync_manager import ImageSyncManager
from kingbird.engine.keypair_sync_manager import KeypairSyncManager
from kingbird.engine.quota_manager import QuotaManager
from kingbird.engine import scheduler
from kingbird.engine.sync_manager import SyncManager
from kingbird.objects import service as service_obj
from oslo_service import service
from oslo_utils import timeutils
@ -49,14 +50,14 @@ def request_context(func):
class EngineService(service.Service):
'''Lifecycle manager for a running service engine.
"""Lifecycle manager for a running service engine.
- All the methods in here are called from the RPC client.
- If a RPC call does not have a corresponding method here, an exceptions
will be thrown.
- Arguments to these calls are added dynamically and will be treated as
keyword arguments by the RPC client.
'''
"""
def __init__(self, host, topic, manager=None):
@ -73,7 +74,8 @@ class EngineService(service.Service):
self.target = None
self._rpc_server = None
self.qm = None
self.sm = None
self.ksm = None
self.ism = None
def init_tgm(self):
self.TG = scheduler.ThreadGroupManager()
@ -81,14 +83,18 @@ class EngineService(service.Service):
def init_qm(self):
self.qm = QuotaManager()
def init_sm(self):
self.sm = SyncManager()
def init_ksm(self):
self.ksm = KeypairSyncManager()
def init_ism(self):
self.ism = ImageSyncManager()
def start(self):
self.engine_id = uuidutils.generate_uuid()
self.init_tgm()
self.init_qm()
self.init_sm()
self.init_ksm()
self.init_ism()
target = oslo_messaging.Target(version=self.rpc_api_version,
server=self.host,
topic=self.topic)
@ -156,7 +162,12 @@ class EngineService(service.Service):
@request_context
def keypair_sync_for_user(self, ctxt, job_id, payload):
# Keypair Sync for a user, will be triggered by KB-API
self.sm.keypair_sync_for_user(ctxt, job_id, payload)
self.ksm.resource_sync(ctxt, job_id, payload)
@request_context
def image_sync(self, ctxt, job_id, payload):
# Image Sync triggered by KB_API.
self.ism.resource_sync(ctxt, job_id, payload)
def _stop_rpc_server(self):
# Stop RPC connection to prevent new requests

View File

@ -0,0 +1,190 @@
# Copyright 2017 Ericsson AB.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from mock import patch
from kingbird.common import exceptions
from kingbird.drivers.openstack import glance_adapter
from kingbird.tests import base
from kingbird.tests import utils
FAKE_ID = utils.UUID1
FAKE_KERNEL_ID = utils.UUID2
FAKE_RAMDISK_ID = utils.UUID3
class FakeQCOW2Image(object):
"""Fake QCOW2 image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class FakeAMIimage(object):
"""Fake AMI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, kernel_id, ramdisk_id, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.kernel_id = kernel_id
self.ramdisk_id = ramdisk_id
self.id = id
class FakeAKIimage(object):
"""Fake AKI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class FakeARIimage(object):
"""Fake ARI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class TestGlanceAdapter(base.KingbirdTestCase):
def setUp(self):
super(TestGlanceAdapter, self).setUp()
self.ctx = utils.dummy_context()
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_check_dependent_images_for_qcow2(self, mock_glance_client):
fake_image = FakeQCOW2Image(0, 'False', 0, 'fake_image', 'bare',
'qcow2', FAKE_ID)
mock_glance_client().get_image.return_value = fake_image
glance_adapter.check_dependent_images('fake_region', self.ctx,
fake_image.id)
mock_glance_client().get_image.assert_called_once_with(fake_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_check_dependent_images_for_aki(self, mock_glance_client):
fake_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image', 'aki',
'aki', FAKE_ID)
mock_glance_client().get_image.return_value = fake_image
glance_adapter.check_dependent_images('fake_region', self.ctx,
fake_image.id)
mock_glance_client().get_image.assert_called_once_with(fake_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_check_dependent_images_for_ari(self, mock_glance_client):
fake_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image', 'ari',
'ari', FAKE_ID)
mock_glance_client().get_image.return_value = fake_image
glance_adapter.check_dependent_images('fake_region', self.ctx,
fake_image.id)
mock_glance_client().get_image.assert_called_once_with(fake_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_check_dependent_images_for_ami(self, mock_glance_client):
fake_ami_image = FakeAMIimage(0, 'False', 0, 'fake_image', 'ami',
'ami', FAKE_KERNEL_ID,
FAKE_RAMDISK_ID, FAKE_ID)
fake_aki_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image',
'aki', 'aki', FAKE_ID)
fake_ari_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image',
'ari', 'ari', FAKE_ID)
mock_glance_client().get_image.side_effect = [
fake_ami_image, fake_aki_image, fake_ari_image]
dependent_images = glance_adapter.check_dependent_images(
'fake_region', self.ctx, fake_ami_image.id)
self.assertEqual(mock_glance_client().get_image.call_count, 3)
expected_result = {
'kernel_image': fake_aki_image,
'ramdisk_image': fake_ari_image
}
self.assertEqual(dependent_images, expected_result)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_ami_image_without_dependent_images(self, mock_glance_client):
fake_ami_image = FakeAMIimage(0, 'False', 0, 'fake_image', 'ami',
'ami', None, None, FAKE_ID)
mock_glance_client().get_image.side_effect = [
fake_ami_image, None, None]
self.assertRaises(exceptions.DependentImageNotFound,
glance_adapter.check_dependent_images,
'fake_region', self.ctx,
fake_ami_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_ami_image_with_wrong_aki_image(self, mock_glance_client):
fake_ami_image = FakeAMIimage(0, 'False', 0, 'fake_image', 'ami',
'ami', FAKE_KERNEL_ID,
FAKE_RAMDISK_ID, FAKE_ID)
fake_aki_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image',
'aki', 'fake_aki', FAKE_ID)
fake_ari_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image',
'ari', 'ari', FAKE_ID)
mock_glance_client().get_image.side_effect = [
fake_ami_image, fake_aki_image, fake_ari_image]
self.assertRaises(exceptions.DependentImageNotFound,
glance_adapter.check_dependent_images,
'fake_region', self.ctx,
fake_ami_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_ami_image_with_wrong_ari_image(self, mock_glance_client):
fake_ami_image = FakeAMIimage(0, 'False', 0, 'fake_image', 'ami',
'ami', FAKE_KERNEL_ID, FAKE_RAMDISK_ID,
FAKE_ID)
fake_aki_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image',
'aki', 'aki', FAKE_ID)
fake_ari_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image',
'ari', 'fake_ari', FAKE_ID)
mock_glance_client().get_image.side_effect = [
fake_ami_image, fake_aki_image, fake_ari_image]
self.assertRaises(exceptions.DependentImageNotFound,
glance_adapter.check_dependent_images,
'fake_region', self.ctx,
fake_ami_image.id)
@patch('kingbird.drivers.openstack.glance_adapter.GlanceClient')
def test_image_with_wrong_format_image(self, mock_glance_client):
fake_image = FakeQCOW2Image(0, 'False', 0, 'fake_image', 'bare',
'fake_format', FAKE_ID)
mock_glance_client().get_image.return_value = fake_image
self.assertRaises(exceptions.ImageFormatNotSupported,
glance_adapter.check_dependent_images,
'fake_region', self.ctx,
fake_image.id)

View File

@ -16,9 +16,13 @@
from mock import patch
from kingbird.drivers.openstack.glance_v2 import GlanceClient
from kingbird.drivers.openstack.glance_v2 import GlanceUpload
from kingbird.tests import base
from kingbird.tests import utils
FAKE_ITERATOR = iter([1, 2, 3])
FAKE_ID = utils.UUID4
class FakeService(object):
"""Fake service class used to test service enable testcase."""
@ -29,6 +33,33 @@ class FakeService(object):
self.id = id
class FakeImage(object):
"""Fake service class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, visibility, tags,
owner, architecture, os_version, os_distro, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.visibility = visibility
self.tags = tags
self.owner = owner
self.architecture = architecture
self.os_version = os_version
self.os_distro = os_distro
self.container_format = container_format
self.disk_format = disk_format
self.id = id
def get(self, attr):
return getattr(self, attr)
def iterkeys(self):
return ",".join(self.__dict__.keys()).split(",")
class FakeEndpoint(object):
"""Fake Endpoints class used to test service enable testcase."""
@ -44,28 +75,125 @@ class TestGlanceClient(base.KingbirdTestCase):
super(TestGlanceClient, self).setUp()
self.ctx = utils.dummy_context()
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_init(self, mock_glance_client, mock_keystone_client):
"""Mock init method of glance."""
def common_init(self, mock_glance_client, mock_keystone_client):
"""Keep commonly used variables."""
fake_service = FakeService('image', 'fake_type', 'fake_id')
fake_endpoint = FakeEndpoint('fake_url', fake_service.id,
'fake_region', 'public')
mock_keystone_client().services_list = [fake_service]
mock_keystone_client().endpoints_list = [fake_endpoint]
GlanceClient('fake_region', self.ctx)
return GlanceClient('fake_region', self.ctx)
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_init(self, mock_glance_client, mock_keystone_client):
"""Test init method of glance."""
self.common_init(mock_glance_client, mock_keystone_client)
self.assertEqual(1, mock_glance_client.call_count)
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_get_image(self, mock_glance_client, mock_keystone_client):
"""Test get_image method of glance."""
Glance_client = self.common_init(mock_glance_client,
mock_keystone_client)
Glance_client.get_image('fake_resource')
mock_glance_client().images.get.\
assert_called_once_with('fake_resource')
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_check_image(self, mock_glance_client, mock_keystone_client):
"""Test get_image method of glance."""
fake_service = FakeService('image', 'fake_type', 'fake_id')
fake_endpoint = FakeEndpoint('fake_url', fake_service.id,
'fake_region', 'public')
mock_keystone_client().services_list = [fake_service]
mock_keystone_client().endpoints_list = [fake_endpoint]
glance_client = GlanceClient('fake_region', self.ctx)
glance_client.check_image('fake_resource')
Glance_client = self.common_init(mock_glance_client,
mock_keystone_client)
Glance_client.check_image('fake_resource')
mock_glance_client().images.get.\
assert_called_once_with('fake_resource')
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_get_image_data(self, mock_glance_client, mock_keystone_client):
"""Test get_image_data method of glance."""
Glance_client = self.common_init(mock_glance_client,
mock_keystone_client)
Glance_client.get_image_data('fake_resource')
mock_glance_client().images.data.\
assert_called_once_with('fake_resource')
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_image_create_force_false(self, mock_glance_client,
mock_keystone_client):
"""Test create_image method of glance."""
Glance_client = self.common_init(mock_glance_client,
mock_keystone_client)
fake_image = FakeImage(0, 'False', 0, 'fake_image', 'public',
'fake_tag', 'fake_owner', 'qemu',
'fake_version', 'fake_distribution', 'bare',
'qcow2', FAKE_ID)
fake_kwargs = {
"min_ram": fake_image.min_ram,
"protected": fake_image.protected,
"min_disk": fake_image.min_disk,
"name": fake_image.name,
"visibility": fake_image.visibility,
"tags": fake_image.tags,
"owner": fake_image.owner,
"architecture": fake_image.architecture,
"os_version": fake_image.os_version,
"os_distro": fake_image.os_distro,
"container_format": fake_image.container_format,
"disk_format": fake_image.disk_format,
"id": fake_image.id
}
Glance_client.create_image(fake_image, False)
mock_glance_client().images.create.\
assert_called_once_with(**fake_kwargs)
@patch('kingbird.drivers.openstack.glance_v2.KeystoneClient')
@patch('kingbird.drivers.openstack.glance_v2.Client')
def test_image_create_force_true(self, mock_glance_client,
mock_keystone_client):
"""Test create_image method of glance."""
Glance_client = self.common_init(mock_glance_client,
mock_keystone_client)
fake_image = FakeImage(0, 'False', 0, 'fake_image', 'public',
'fake_tag', 'fake_owner', 'qemu',
'fake_version', 'fake_distribution', 'bare',
'qcow2', FAKE_ID)
fake_kwargs = {
"min_ram": fake_image.min_ram,
"protected": fake_image.protected,
"min_disk": fake_image.min_disk,
"name": fake_image.name,
"visibility": fake_image.visibility,
"tags": fake_image.tags,
"owner": fake_image.owner,
"architecture": fake_image.architecture,
"os_version": fake_image.os_version,
"os_distro": fake_image.os_distro,
"container_format": fake_image.container_format,
"disk_format": fake_image.disk_format
}
Glance_client.create_image(fake_image, True)
mock_glance_client().images.create.\
assert_called_once_with(**fake_kwargs)
class TestGlanceUpload(base.KingbirdTestCase):
def test_init(self):
"""Test init method of GlanceUpload."""
glance_upload = GlanceUpload(FAKE_ITERATOR)
self.assertEqual(glance_upload.received, FAKE_ITERATOR)
def test_read(self):
"""Test read methos of GlanceUpload.
We send 65536 even though we don't use it.
Because the read method in GlanceUpload is the replacement
of read method in glance.
"""
glance_upload = GlanceUpload(FAKE_ITERATOR).read(65536)
self.assertEqual(glance_upload, 1)

View File

@ -0,0 +1,246 @@
# Copyright 2017 Ericsson AB.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from mock import patch
from kingbird.engine import image_sync_manager
from kingbird.tests import base
from kingbird.tests import utils
DEFAULT_FORCE = False
FAKE_USER_ID = 'user123'
FAKE_TARGET_REGION = 'fake_target_region'
FAKE_SOURCE_REGION = 'fake_source_region'
FAKE_RESOURCE_ID = 'fake_id'
FAKE_JOB_ID = utils.UUID1
FAKE_KERNEL_ID = utils.UUID2
FAKE_RAMDISK_ID = utils.UUID3
FAKE_ID = utils.UUID4
FAKE_RESULT = 'SUCCESS'
FAKE_RESULT_FAIL = 'FAILURE'
FAKE_RESOURCE = 'fake_image'
class FakeQCOW2Image(object):
"""Fake QCOW2 image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class FakeAMIimage(object):
"""Fake AMI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, kernel_id, ramdisk_id, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.kernel_id = kernel_id
self.ramdisk_id = ramdisk_id
self.id = id
class FakeAKIimage(object):
"""Fake AKI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class FakeARIimage(object):
"""Fake ARI image class used to test service enable testcase."""
def __init__(self, min_ram, protected, min_disk, name, container_format,
disk_format, id):
self.min_ram = min_ram
self.protected = protected
self.min_disk = min_disk
self.name = name
self.container_format = container_format
self.disk_format = disk_format
self.id = id
class TestImageSyncManager(base.KingbirdTestCase):
def setUp(self):
super(TestImageSyncManager, self).setUp()
self.ctxt = utils.dummy_context()
@patch('kingbird.engine.image_sync_manager.db_api')
@patch('kingbird.engine.image_sync_manager.glance_adapter')
@patch('kingbird.engine.image_sync_manager.GlanceClient')
@patch('kingbird.engine.image_sync_manager.GlanceUpload')
def test_ami_image_sync(self, mock_glance_upload, mock_glance_client,
mock_glance_adapter, mock_db_api):
fake_ami_image = FakeAMIimage(0, 'False', 0, FAKE_RESOURCE, 'ami',
'ami', FAKE_KERNEL_ID,
FAKE_RAMDISK_ID, FAKE_ID)
fake_aki_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image',
'aki', 'aki', FAKE_ID)
fake_ari_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image',
'ari', 'ari', FAKE_ID)
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
payload['force'] = DEFAULT_FORCE
payload['source'] = FAKE_SOURCE_REGION
payload['resources'] = [fake_ami_image.id]
expected_resources = {
'kernel_image': fake_aki_image,
'ramdisk_image': fake_ari_image
}
mock_glance_adapter.check_dependent_images.\
return_value = expected_resources
ism = image_sync_manager.ImageSyncManager()
ism.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_glance_adapter.check_dependent_images.\
assert_called_once_with(self.ctxt, FAKE_SOURCE_REGION,
fake_ami_image.id)
self.assertEqual(mock_glance_client().get_image_data.call_count, 3)
self.assertEqual(mock_glance_client().create_image.call_count, 3)
self.assertEqual(mock_glance_upload.call_count, 3)
self.assertEqual(mock_glance_client().image_upload.call_count, 3)
mock_db_api.resource_sync_status.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID)
mock_db_api.sync_job_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID, FAKE_RESULT)
@patch('kingbird.engine.image_sync_manager.db_api')
@patch('kingbird.engine.image_sync_manager.glance_adapter')
@patch('kingbird.engine.image_sync_manager.GlanceClient')
@patch('kingbird.engine.image_sync_manager.GlanceUpload')
def test_qcow2_image_sync(self, mock_glance_upload, mock_glance_client,
mock_glance_adapter, mock_db_api):
fake_qcow2_image = FakeQCOW2Image(0, 'False', 0, FAKE_RESOURCE, 'bare',
'qcow2', FAKE_ID)
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
payload['force'] = DEFAULT_FORCE
payload['source'] = FAKE_SOURCE_REGION
payload['resources'] = [fake_qcow2_image.id]
expected_resources = None
mock_glance_adapter.check_dependent_images.\
return_value = expected_resources
ism = image_sync_manager.ImageSyncManager()
ism.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_glance_adapter.check_dependent_images.\
assert_called_once_with(self.ctxt, FAKE_SOURCE_REGION,
fake_qcow2_image.id)
self.assertEqual(mock_glance_client().get_image_data.call_count, 1)
self.assertEqual(mock_glance_client().create_image.call_count, 1)
self.assertEqual(mock_glance_upload.call_count, 1)
self.assertEqual(mock_glance_client().image_upload.call_count, 1)
mock_db_api.resource_sync_status.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID)
mock_db_api.sync_job_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID, FAKE_RESULT)
@patch('kingbird.engine.image_sync_manager.db_api')
@patch('kingbird.engine.image_sync_manager.glance_adapter')
@patch('kingbird.engine.image_sync_manager.GlanceClient')
@patch('kingbird.engine.image_sync_manager.GlanceUpload')
def test_aki_image_sync(self, mock_glance_upload, mock_glance_client,
mock_glance_adapter, mock_db_api):
fake_aki_image = FakeAKIimage(0, 'False', 0, 'fake_kernel_image',
'aki', 'aki', FAKE_ID)
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
payload['force'] = DEFAULT_FORCE
payload['source'] = FAKE_SOURCE_REGION
payload['resources'] = [fake_aki_image.id]
expected_resources = None
mock_glance_adapter.check_dependent_images.\
return_value = expected_resources
ism = image_sync_manager.ImageSyncManager()
ism.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_glance_adapter.check_dependent_images.\
assert_called_once_with(self.ctxt, FAKE_SOURCE_REGION,
fake_aki_image.id)
self.assertEqual(mock_glance_client().get_image_data.call_count, 1)
self.assertEqual(mock_glance_client().create_image.call_count, 1)
self.assertEqual(mock_glance_upload.call_count, 1)
self.assertEqual(mock_glance_client().image_upload.call_count, 1)
mock_db_api.resource_sync_status.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID)
mock_db_api.sync_job_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID, FAKE_RESULT)
@patch('kingbird.engine.image_sync_manager.db_api')
@patch('kingbird.engine.image_sync_manager.glance_adapter')
@patch('kingbird.engine.image_sync_manager.GlanceClient')
@patch('kingbird.engine.image_sync_manager.GlanceUpload')
def test_ari_image_sync(self, mock_glance_upload, mock_glance_client,
mock_glance_adapter, mock_db_api):
fake_ari_image = FakeARIimage(0, 'False', 0, 'fake_ramdisk_image',
'ari', 'ari', FAKE_ID)
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
payload['force'] = DEFAULT_FORCE
payload['source'] = FAKE_SOURCE_REGION
payload['resources'] = [fake_ari_image.id]
expected_resources = None
mock_glance_adapter.check_dependent_images.\
return_value = expected_resources
ism = image_sync_manager.ImageSyncManager()
ism.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_glance_adapter.check_dependent_images.\
assert_called_once_with(self.ctxt, FAKE_SOURCE_REGION,
fake_ari_image.id)
self.assertEqual(mock_glance_client().get_image_data.call_count, 1)
self.assertEqual(mock_glance_client().create_image.call_count, 1)
self.assertEqual(mock_glance_upload.call_count, 1)
self.assertEqual(mock_glance_client().image_upload.call_count, 1)
mock_db_api.resource_sync_status.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID)
mock_db_api.sync_job_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID, FAKE_RESULT)
@patch('kingbird.engine.image_sync_manager.db_api')
def test_update_success_result_in_db(self, mock_db_api):
ism = image_sync_manager.ImageSyncManager()
ism.update_result_in_database(self.ctxt, FAKE_JOB_ID,
FAKE_TARGET_REGION, FAKE_RESOURCE,
True)
mock_db_api.resource_sync_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID,
FAKE_TARGET_REGION, FAKE_RESOURCE,
FAKE_RESULT)
@patch('kingbird.engine.image_sync_manager.db_api')
def test_update_fail_result_in_db(self, mock_db_api):
ism = image_sync_manager.ImageSyncManager()
ism.update_result_in_database(self.ctxt, FAKE_JOB_ID,
FAKE_TARGET_REGION, FAKE_RESOURCE,
False)
mock_db_api.resource_sync_update.\
assert_called_once_with(self.ctxt, FAKE_JOB_ID,
FAKE_TARGET_REGION, FAKE_RESOURCE,
FAKE_RESULT_FAIL)

View File

@ -13,7 +13,7 @@
# under the License.
import mock
from kingbird.engine import sync_manager
from kingbird.engine import keypair_sync_manager
from kingbird.tests import base
from kingbird.tests import utils
@ -32,16 +32,17 @@ class FakeKeypair(object):
self.public_key = public_key
class TestSyncManager(base.KingbirdTestCase):
class TestKeypairSyncManager(base.KingbirdTestCase):
def setUp(self):
super(TestSyncManager, self).setUp()
super(TestKeypairSyncManager, self).setUp()
self.ctxt = utils.dummy_context()
@mock.patch.object(sync_manager, 'NovaClient')
@mock.patch.object(sync_manager, 'EndpointCache')
@mock.patch.object(sync_manager.SyncManager, '_create_keypairs')
@mock.patch.object(sync_manager, 'db_api')
def test_keypair_sync_force_false(self, mock_db_api, mock_create_keypair,
@mock.patch.object(keypair_sync_manager, 'NovaClient')
@mock.patch.object(keypair_sync_manager, 'EndpointCache')
@mock.patch.object(keypair_sync_manager.KeypairSyncManager,
'create_resources')
@mock.patch.object(keypair_sync_manager, 'db_api')
def test_keypair_sync_force_false(self, mock_db_api, mock_create_resource,
mock_endpoint_cache, mock_nova):
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
@ -52,17 +53,18 @@ class TestSyncManager(base.KingbirdTestCase):
mock_endpoint_cache().get_session_from_token.\
return_value = 'fake_session'
mock_nova().get_keypairs.return_value = fake_key
sm = sync_manager.SyncManager()
sm.keypair_sync_for_user(self.ctxt, FAKE_JOB_ID, payload)
mock_create_keypair.assert_called_once_with(
ksm = keypair_sync_manager.KeypairSyncManager()
ksm.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_create_resource.assert_called_once_with(
FAKE_JOB_ID, payload['force'], payload['target'][0], fake_key,
'fake_session', self.ctxt)
@mock.patch.object(sync_manager, 'NovaClient')
@mock.patch.object(sync_manager, 'EndpointCache')
@mock.patch.object(sync_manager.SyncManager, '_create_keypairs')
@mock.patch.object(sync_manager, 'db_api')
def test_keypair_sync_force_true(self, mock_db_api, mock_create_keypair,
@mock.patch.object(keypair_sync_manager, 'NovaClient')
@mock.patch.object(keypair_sync_manager, 'EndpointCache')
@mock.patch.object(keypair_sync_manager.KeypairSyncManager,
'create_resources')
@mock.patch.object(keypair_sync_manager, 'db_api')
def test_keypair_sync_force_true(self, mock_db_api, mock_create_resource,
mock_endpoint_cache, mock_nova):
payload = dict()
payload['target'] = [FAKE_TARGET_REGION]
@ -73,18 +75,18 @@ class TestSyncManager(base.KingbirdTestCase):
mock_endpoint_cache().get_session_from_token.\
return_value = 'fake_session'
mock_nova().get_keypairs.return_value = fake_key
sm = sync_manager.SyncManager()
sm.keypair_sync_for_user(self.ctxt, FAKE_JOB_ID, payload)
mock_create_keypair.assert_called_once_with(
ksm = keypair_sync_manager.KeypairSyncManager()
ksm.resource_sync(self.ctxt, FAKE_JOB_ID, payload)
mock_create_resource.assert_called_once_with(
FAKE_JOB_ID, payload['force'], payload['target'][0], fake_key,
'fake_session', self.ctxt)
@mock.patch.object(sync_manager, 'NovaClient')
@mock.patch.object(sync_manager, 'db_api')
@mock.patch.object(keypair_sync_manager, 'NovaClient')
@mock.patch.object(keypair_sync_manager, 'db_api')
def test_create_keypair(self, mock_db_api, mock_nova):
fake_key = FakeKeypair('fake_name', 'fake-rsa')
sm = sync_manager.SyncManager()
sm._create_keypairs(FAKE_JOB_ID, DEFAULT_FORCE, FAKE_TARGET_REGION,
fake_key, 'fake_session', self.ctxt)
ksm = keypair_sync_manager.KeypairSyncManager()
ksm.create_resources(FAKE_JOB_ID, DEFAULT_FORCE, FAKE_TARGET_REGION,
fake_key, 'fake_session', self.ctxt)
mock_nova().create_keypairs.\
assert_called_once_with(DEFAULT_FORCE, fake_key)

View File

@ -53,10 +53,15 @@ class TestEngineService(base.KingbirdTestCase):
self.service_obj.init_qm()
self.assertIsNotNone(self.service_obj.qm)
@mock.patch.object(service, 'SyncManager')
def test_init_sm(self, mock_resource_manager):
self.service_obj.init_sm()
self.assertIsNotNone(self.service_obj.sm)
@mock.patch.object(service, 'KeypairSyncManager')
def test_init_ksm(self, mock_keypair_sync_manager):
self.service_obj.init_ksm()
self.assertIsNotNone(self.service_obj.ksm)
@mock.patch.object(service, 'ImageSyncManager')
def test_init_ism(self, mock_image_sync_manager):
self.service_obj.init_ism()
self.assertIsNotNone(self.service_obj.ism)
@mock.patch.object(service.EngineService, 'service_registry_cleanup')
@mock.patch.object(service, 'QuotaManager')
@ -110,11 +115,19 @@ class TestEngineService(base.KingbirdTestCase):
self.service_obj.stop()
mock_rpc.get_rpc_server().stop.assert_called_once_with()
@mock.patch.object(service, 'SyncManager')
def test_resource_sync_for_user(self, mock_sync_manager):
@mock.patch.object(service, 'KeypairSyncManager')
def test_keypair_sync_for_user(self, mock_keypair_sync_manager):
self.service_obj.init_tgm()
self.service_obj.init_sm()
self.service_obj.init_ksm()
self.service_obj.keypair_sync_for_user(
self.context, self.job_id, self.payload,)
mock_sync_manager().keypair_sync_for_user.\
self.context, self.job_id, self.payload)
mock_keypair_sync_manager().resource_sync.\
assert_called_once_with(self.context, self.job_id, self.payload)
@mock.patch.object(service, 'ImageSyncManager')
def test_image_sync(self, mock_image_sync_manager):
self.service_obj.init_tgm()
self.service_obj.init_ism()
self.service_obj.image_sync(self.context, self.job_id, self.payload)
mock_image_sync_manager().resource_sync.\
assert_called_once_with(self.context, self.job_id, self.payload)