Add Sync template feature to KB-server
This commit from kingbird server is to enhance the existing "sync create" job. Existing feature "sync create" is to sync single resource-type to single target region i.e one resource off one resource-type to one region. Issue with "sync create": Kingbird cannot sync more than one resource-type to multiple target regions in a single request. This issue is solved by "sync template" feature. Working of sync template feature: User have to provide a input file with .yaml or .yml or .json extension. This input file should consists of ->resource-type ->resources ->source_region ->target_region. Change-Id: Iaa8e9568f97581f74e688d15a11c7e7fd832e019
This commit is contained in:
parent
fd6b4cba10
commit
a364c4b11c
|
@ -55,11 +55,12 @@ class ResourceSyncController(object):
|
|||
pass
|
||||
|
||||
def _entries_to_database(self, context, target_regions, source_region,
|
||||
resources, resource_type, job_id):
|
||||
resources, resource_type, job_id, job_name):
|
||||
"""Manage the entries to database for both Keypair and image."""
|
||||
# Insert into the parent table
|
||||
try:
|
||||
result = db_api.sync_job_create(context, job_id=job_id)
|
||||
|
||||
result = db_api.sync_job_create(context, job_name, job_id=job_id)
|
||||
except exceptions.InternalError:
|
||||
pecan.abort(500, _('Internal Server Error.'))
|
||||
# Insert into the child table
|
||||
|
@ -107,17 +108,32 @@ class ResourceSyncController(object):
|
|||
context = restcomm.extract_context_from_environ()
|
||||
if not uuidutils.is_uuid_like(project) or project != context.project:
|
||||
pecan.abort(400, _('Invalid request URL'))
|
||||
payload = eval(request.body)
|
||||
if not payload:
|
||||
request_data = eval(request.body)
|
||||
if not request_data:
|
||||
pecan.abort(400, _('Body required'))
|
||||
payload = payload.get('resource_set')
|
||||
if not payload:
|
||||
request_data = request_data.get('resource_set')
|
||||
if not request_data:
|
||||
pecan.abort(400, _('resource_set required'))
|
||||
job_name = None
|
||||
if 'name' in request_data.keys():
|
||||
job_name = request_data.get('name')
|
||||
for iteration in range(len(request_data['Sync'])):
|
||||
payload = request_data['Sync'][iteration]
|
||||
response = self._get_post_data(payload,
|
||||
context, job_name)
|
||||
else:
|
||||
response = self._get_post_data(request_data,
|
||||
context, job_name)
|
||||
return response
|
||||
|
||||
def _get_post_data(self, payload, context, job_name):
|
||||
resource_type = payload.get('resource_type')
|
||||
target_regions = payload.get('target')
|
||||
if not target_regions or not isinstance(target_regions, list):
|
||||
pecan.abort(400, _('Target regions required'))
|
||||
source_region = payload.get('source')
|
||||
if(isinstance(source_region, list)):
|
||||
source_region = "".join(source_region)
|
||||
if not source_region or not isinstance(source_region, str):
|
||||
pecan.abort(400, _('Source region required'))
|
||||
source_resources = payload.get('resources')
|
||||
|
@ -138,7 +154,8 @@ class ResourceSyncController(object):
|
|||
result = self._entries_to_database(context, target_regions,
|
||||
source_region,
|
||||
source_resources,
|
||||
resource_type, job_id)
|
||||
resource_type,
|
||||
job_id, job_name)
|
||||
return self._keypair_sync(job_id, payload, context, result)
|
||||
|
||||
elif resource_type == consts.IMAGE:
|
||||
|
@ -152,7 +169,8 @@ class ResourceSyncController(object):
|
|||
result = self._entries_to_database(context, target_regions,
|
||||
source_region,
|
||||
source_resources,
|
||||
resource_type, job_id)
|
||||
resource_type,
|
||||
job_id, job_name)
|
||||
return self._image_sync(job_id, payload, context, result)
|
||||
|
||||
elif resource_type == consts.FLAVOR:
|
||||
|
@ -169,7 +187,8 @@ class ResourceSyncController(object):
|
|||
result = self._entries_to_database(context, target_regions,
|
||||
source_region,
|
||||
source_resources,
|
||||
resource_type, job_id)
|
||||
resource_type,
|
||||
job_id, job_name)
|
||||
return self._flavor_sync(job_id, payload, context, result)
|
||||
|
||||
else:
|
||||
|
@ -212,8 +231,8 @@ class ResourceSyncController(object):
|
|||
:param result: Result object to return an output.
|
||||
"""
|
||||
self.rpc_client.keypair_sync_for_user(context, job_id, payload)
|
||||
|
||||
return {'job_status': {'id': result.id, 'status': result.sync_status,
|
||||
return {'job_status': {'name': result.name, 'id': result.id,
|
||||
'status': result.sync_status,
|
||||
'created_at': result.created_at}}
|
||||
|
||||
def _image_sync(self, job_id, payload, context, result):
|
||||
|
@ -226,7 +245,8 @@ class ResourceSyncController(object):
|
|||
:param result: Result object to return an output.
|
||||
"""
|
||||
self.rpc_client.image_sync(context, job_id, payload)
|
||||
return {'job_status': {'id': result.id, 'status': result.sync_status,
|
||||
return {'job_status': {'name': result.name, 'id': result.id,
|
||||
'status': result.sync_status,
|
||||
'created_at': result.created_at}}
|
||||
|
||||
def _flavor_sync(self, job_id, payload, context, result):
|
||||
|
@ -239,5 +259,6 @@ class ResourceSyncController(object):
|
|||
:param result: Result object to return an output.
|
||||
"""
|
||||
self.rpc_client.flavor_sync(context, job_id, payload)
|
||||
return {'job_status': {'id': result.id, 'status': result.sync_status,
|
||||
return {'job_status': {'name': result.name, 'id': result.id,
|
||||
'status': result.sync_status,
|
||||
'created_at': result.created_at}}
|
||||
|
|
|
@ -383,9 +383,11 @@ def service_get_all(context):
|
|||
##########################
|
||||
|
||||
@require_context
|
||||
def sync_job_create(context, job_id):
|
||||
def sync_job_create(context, job_name, job_id):
|
||||
with write_session() as session:
|
||||
sjc = models.SyncJob()
|
||||
if job_name is not None:
|
||||
sjc.name = job_name
|
||||
sjc.id = job_id
|
||||
sjc.user_id = context.user
|
||||
sjc.project_id = context.project
|
||||
|
|
|
@ -22,6 +22,7 @@ def upgrade(migrate_engine):
|
|||
|
||||
sync_job = sqlalchemy.Table(
|
||||
'sync_job', meta,
|
||||
sqlalchemy.Column('name', sqlalchemy.String(255)),
|
||||
sqlalchemy.Column('id', sqlalchemy.String(36),
|
||||
primary_key=True),
|
||||
sqlalchemy.Column('sync_status', sqlalchemy.String(length=36),
|
||||
|
|
|
@ -154,6 +154,8 @@ class SyncJob(BASE, KingbirdBase):
|
|||
|
||||
__tablename__ = 'sync_job'
|
||||
|
||||
name = Column('name', String(255))
|
||||
|
||||
id = Column('id', String(36), primary_key=True)
|
||||
|
||||
sync_status = Column(String(36), default=consts.JOB_PROGRESS,
|
||||
|
|
|
@ -136,6 +136,7 @@ class NovaClient(base.DriverBase):
|
|||
def get_flavor(self, res_id):
|
||||
"""Get Flavor for a specified context."""
|
||||
try:
|
||||
res_id = self.nova_client.flavors.find(name=res_id)
|
||||
flavor = self.nova_client.flavors.get(res_id)
|
||||
LOG.info("Source Flavor: %s", flavor.name)
|
||||
return flavor
|
||||
|
|
|
@ -58,7 +58,7 @@ class FlavorSyncManager(object):
|
|||
% {'flavor': source_flavor.name, 'region': region})
|
||||
try:
|
||||
db_api.resource_sync_update(context, job_id, region,
|
||||
source_flavor.id,
|
||||
source_flavor.name,
|
||||
consts.JOB_SUCCESS)
|
||||
except exceptions.JobNotFound():
|
||||
raise
|
||||
|
@ -67,7 +67,7 @@ class FlavorSyncManager(object):
|
|||
% {'msg': exc.message, 'region': region})
|
||||
try:
|
||||
db_api.resource_sync_update(context, job_id, region,
|
||||
source_flavor.id,
|
||||
source_flavor.name,
|
||||
consts.JOB_FAILURE)
|
||||
except exceptions.JobNotFound():
|
||||
raise
|
||||
|
@ -87,6 +87,8 @@ class FlavorSyncManager(object):
|
|||
force = eval(str(payload.get('force', False)))
|
||||
resource_ids = payload.get('resources')
|
||||
source_region = payload['source']
|
||||
if(isinstance(source_region, list)):
|
||||
source_region = "".join(source_region)
|
||||
session = EndpointCache().get_session_from_token(
|
||||
context.auth_token, context.project)
|
||||
# Create Source Region object
|
||||
|
|
|
@ -186,6 +186,8 @@ class ImageSyncManager(object):
|
|||
force = eval(str(payload.get('force', False)))
|
||||
resource_ids = payload.get('resources')
|
||||
source_region = payload['source']
|
||||
if(isinstance(source_region, list)):
|
||||
source_region = "".join(source_region)
|
||||
for resource in resource_ids:
|
||||
thread = threading.Thread(target=self.create_resources_in_region,
|
||||
args=(job_id, target_regions,
|
||||
|
|
|
@ -84,6 +84,8 @@ class KeypairSyncManager(object):
|
|||
force = eval(str(payload.get('force', False)))
|
||||
resource_ids = payload.get('resources')
|
||||
source_region = payload['source']
|
||||
if(isinstance(source_region, list)):
|
||||
source_region = "".join(source_region)
|
||||
session = EndpointCache().get_session_from_token(
|
||||
context.auth_token, context.project)
|
||||
# Create Source Region object
|
||||
|
|
|
@ -25,6 +25,7 @@ from kingbird.tests.unit.api import test_root_controller as testroot
|
|||
from kingbird.tests import utils
|
||||
|
||||
DEFAULT_FORCE = False
|
||||
JOB_NAME = 'fake_job_name'
|
||||
SOURCE_KEYPAIR = 'fake_key1'
|
||||
SOURCE_FLAVOR = 'fake_flavor1'
|
||||
SOURCE_IMAGE_NAME = 'fake_image'
|
||||
|
@ -70,15 +71,17 @@ class FakeImage(object):
|
|||
|
||||
|
||||
class Result(object):
|
||||
def __init__(self, job_id, status, time):
|
||||
def __init__(self, job_id, name, status, time):
|
||||
self.job_id = job_id
|
||||
self.name = name
|
||||
self.status = status
|
||||
self.time = time
|
||||
|
||||
|
||||
class SyncJob(object):
|
||||
def __init__(self, id, sync_status, created_at):
|
||||
def __init__(self, id, name, sync_status, created_at):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.sync_status = sync_status
|
||||
self.created_at = created_at
|
||||
|
||||
|
@ -88,6 +91,21 @@ class TestResourceManager(testroot.KBApiTest):
|
|||
super(TestResourceManager, self).setUp()
|
||||
self.ctx = utils.dummy_context()
|
||||
|
||||
@mock.patch.object(rpc_client, 'EngineClient')
|
||||
@mock.patch.object(sync_manager, 'NovaClient')
|
||||
@mock.patch.object(sync_manager, 'EndpointCache')
|
||||
@mock.patch.object(sync_manager, 'db_api')
|
||||
def test_post_request_data(self, mock_db_api, mock_endpoint_cache,
|
||||
mock_nova, mock_rpc_client):
|
||||
payload = {"resources": [SOURCE_KEYPAIR],
|
||||
"resource_type": "keypair",
|
||||
"source": FAKE_SOURCE_REGION,
|
||||
"target": [FAKE_TARGET_REGION]}
|
||||
result = sync_manager.ResourceSyncController().\
|
||||
_get_post_data(payload, self.ctx, JOB_NAME)
|
||||
self.assertEqual(result['job_status'].get('status'),
|
||||
mock_db_api.sync_job_create().sync_status)
|
||||
|
||||
@mock.patch.object(rpc_client, 'EngineClient')
|
||||
@mock.patch.object(sync_manager, 'NovaClient')
|
||||
@mock.patch.object(sync_manager, 'EndpointCache')
|
||||
|
@ -101,7 +119,8 @@ class TestResourceManager(testroot.KBApiTest):
|
|||
"source": FAKE_SOURCE_REGION,
|
||||
"target": [FAKE_TARGET_REGION]}}
|
||||
fake_key = FakeKeypair('fake_name', 'fake-rsa')
|
||||
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
|
||||
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
|
||||
consts.JOB_PROGRESS, time_now)
|
||||
mock_endpoint_cache().get_session_from_token.\
|
||||
return_value = 'fake_session'
|
||||
mock_nova().get_keypairs.return_value = fake_key
|
||||
|
@ -129,8 +148,10 @@ class TestResourceManager(testroot.KBApiTest):
|
|||
"force": "True",
|
||||
"source": FAKE_SOURCE_REGION,
|
||||
"target": [FAKE_TARGET_REGION]}}
|
||||
fake_flavor = Fake_Flavor('fake_id', 512, 2, 30, 'fake_flavor', 1)
|
||||
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
|
||||
fake_flavor = Fake_Flavor('fake_id', 512, 2,
|
||||
30, 'fake_flavor', 1)
|
||||
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
|
||||
consts.JOB_PROGRESS, time_now)
|
||||
mock_endpoint_cache().get_session_from_token.\
|
||||
return_value = 'fake_session'
|
||||
mock_nova().get_flavor.return_value = fake_flavor
|
||||
|
@ -172,7 +193,8 @@ class TestResourceManager(testroot.KBApiTest):
|
|||
"source": FAKE_SOURCE_REGION,
|
||||
"target": [FAKE_TARGET_REGION]}}
|
||||
fake_image = FakeImage(SOURCE_IMAGE_ID, SOURCE_IMAGE_NAME)
|
||||
sync_job_result = SyncJob(FAKE_JOB, consts.JOB_PROGRESS, time_now)
|
||||
sync_job_result = SyncJob(JOB_NAME, FAKE_JOB,
|
||||
consts.JOB_PROGRESS, time_now)
|
||||
mock_glance().check_image.return_value = fake_image.id
|
||||
mock_db_api.sync_job_create.return_value = sync_job_result
|
||||
response = self.app.post_json(FAKE_URL,
|
||||
|
@ -368,11 +390,11 @@ class TestResourceManager(testroot.KBApiTest):
|
|||
@mock.patch.object(sync_manager, 'db_api')
|
||||
def test_entries_to_database(self, mock_db_api, mock_rpc_client):
|
||||
time_now = timeutils.utcnow()
|
||||
result = Result(FAKE_JOB, FAKE_STATUS, time_now)
|
||||
result = Result(JOB_NAME, FAKE_JOB, FAKE_STATUS, time_now)
|
||||
mock_db_api.sync_job_create.return_value = result
|
||||
sync_manager.ResourceSyncController()._entries_to_database(
|
||||
self.ctx, FAKE_TARGET_REGION, FAKE_SOURCE_REGION,
|
||||
FAKE_RESOURCE_ID, FAKE_RESOURCE_TYPE, FAKE_JOB)
|
||||
FAKE_RESOURCE_ID, FAKE_RESOURCE_TYPE, FAKE_TENANT, JOB_NAME)
|
||||
mock_db_api.resource_sync_create.assert_called_once_with(
|
||||
self.ctx, result, FAKE_TARGET_REGION[0], FAKE_SOURCE_REGION,
|
||||
FAKE_RESOURCE_ID[0], FAKE_RESOURCE_TYPE)
|
||||
|
|
|
@ -69,7 +69,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
self.ctx = utils.dummy_context()
|
||||
|
||||
def test_create_sync_job(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
self.assertEqual(consts.JOB_PROGRESS, job.sync_status)
|
||||
created_job = db_api.sync_job_list(self.ctx, "active")
|
||||
|
@ -77,38 +78,44 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
created_job[0].get('sync_status'))
|
||||
|
||||
def test_primary_key_sync_job(self):
|
||||
self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
self.sync_job_create(self.ctx, job_name='fake_job_name', job_id=UUID1)
|
||||
self.assertRaises(oslo_db.exception.DBDuplicateEntry,
|
||||
self.sync_job_create, self.ctx, job_id=UUID1)
|
||||
self.sync_job_create, self.ctx,
|
||||
job_name='fake_job_name', job_id=UUID1)
|
||||
|
||||
def test_sync_job_update(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
db_api.sync_job_update(self.ctx, UUID1, consts.JOB_SUCCESS)
|
||||
updated_job = db_api.sync_job_list(self.ctx)
|
||||
self.assertEqual(consts.JOB_SUCCESS, updated_job[0].get('sync_status'))
|
||||
|
||||
def test_active_jobs(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
query = db_api.sync_job_list(self.ctx, 'active')
|
||||
self.assertEqual(query[0].get('sync_status'), job.sync_status)
|
||||
|
||||
def test_sync_job_status(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
query = db_api.sync_job_status(self.ctx, job_id=UUID1)
|
||||
self.assertEqual(query, consts.JOB_PROGRESS)
|
||||
|
||||
def test_update_invalid_job(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
self.assertRaises(exceptions.JobNotFound,
|
||||
db_api.sync_job_update, self.ctx, 'fake_job',
|
||||
consts.JOB_SUCCESS)
|
||||
|
||||
def test_resource_sync_create(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
resource_sync_create = self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
source_region='Fake_region2', resource='fake_key',
|
||||
|
@ -117,7 +124,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
self.assertEqual(consts.JOB_PROGRESS, resource_sync_create.sync_status)
|
||||
|
||||
def test_resource_sync_status(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
resource_sync_create = self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
source_region='Fake_region2', resource='fake_key',
|
||||
|
@ -127,7 +135,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
self.assertEqual(consts.JOB_PROGRESS, status[0])
|
||||
|
||||
def test_resource_sync_update(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
resource_sync_create = self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
source_region='Fake_region2', resource='fake_key',
|
||||
|
@ -140,7 +149,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
self.assertEqual(consts.JOB_SUCCESS, updated_job[0].get('sync_status'))
|
||||
|
||||
def test_foreign_key(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
resource_sync_create = self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
|
@ -151,7 +161,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
|
||||
def test_delete_sync_job(self):
|
||||
job_id = UUID1
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.assertIsNotNone(job)
|
||||
self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
|
@ -162,7 +173,8 @@ class DBAPIResourceSyncTest(base.KingbirdTestCase):
|
|||
self.assertEqual(0, len(updated_job))
|
||||
|
||||
def test_composite_primary_key(self):
|
||||
job = self.sync_job_create(self.ctx, job_id=UUID1)
|
||||
job = self.sync_job_create(self.ctx, job_name='fake_job_name',
|
||||
job_id=UUID1)
|
||||
self.resource_sync_create(
|
||||
self.ctx, job=job, region='Fake_region',
|
||||
source_region='Fake_region2', resource='fake_key',
|
||||
|
|
|
@ -175,11 +175,15 @@ class TestNovaClient(base.KingbirdTestCase):
|
|||
@mock.patch.object(nova_v2, 'client')
|
||||
def test_get_flavor(self, mock_novaclient):
|
||||
"""Test get_flavor method of nova."""
|
||||
fake_flavor = Fake_Flavor('fake_id', 512, 2, 30, 'fake_flavor', 1,
|
||||
1.0, False)
|
||||
nv_client = nova_v2.NovaClient('fake_region', self.session,
|
||||
DISABLED_QUOTAS)
|
||||
nv_client.get_flavor('fake_id')
|
||||
|
||||
nv_client.get_flavor(fake_flavor.name)
|
||||
result = mock_novaclient.Client().flavors.find(fake_flavor.name)
|
||||
mock_novaclient.Client().flavors.get.\
|
||||
assert_called_once_with('fake_id')
|
||||
assert_called_once_with(result)
|
||||
|
||||
@mock.patch.object(nova_v2, 'client')
|
||||
def test_get_flavor_access_tenants(self, mock_novaclient):
|
||||
|
|
|
@ -212,5 +212,5 @@ class TestFlavorSyncManager(base.KingbirdTestCase):
|
|||
mock_nova().create_flavor.assert_called_once_with(
|
||||
payload['force'], fake_flavor, access_tenants)
|
||||
mock_db_api.resource_sync_update.assert_called_once_with(
|
||||
self.ctxt, FAKE_JOB_ID, payload['target'][0], fake_flavor.id,
|
||||
self.ctxt, FAKE_JOB_ID, payload['target'][0], fake_flavor.name,
|
||||
JOB_RESULT)
|
||||
|
|
Loading…
Reference in New Issue