Add cloudfoundry datasource driver

This patch adds a datasource driver to congress that integrates with
cloudfoundry. This driver exports the following tables with cloudfoundry
data: organizations, apps, spaces.

Implements blueprint: cloudfoundry-datasource-driver

Change-Id: I632fbf95f6a24ec975448aaa4929fa8a290c3cc3
Co-authored-by: Sabha Parameswaran <sabhap@pivotal.io>
This commit is contained in:
Aaron Rosen 2015-01-30 11:42:46 -08:00
parent 00701940a0
commit cf89c453a2
4 changed files with 471 additions and 0 deletions

View File

@ -0,0 +1,164 @@
#!/usr/bin/env python
# Copyright (c) 2014 VMware, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from cloudfoundryclient.v2 import client
from congress.datasources.datasource_driver import DataSourceDriver
from congress.datasources import datasource_utils
from congress.openstack.common import log as logging
LOG = logging.getLogger(__name__)
def d6service(name, keys, inbox, datapath, args):
"""This method is called by d6cage to create a dataservice instance."""
return CloudFoundryV2Driver(name, keys, inbox, datapath, args)
class CloudFoundryV2Driver(DataSourceDriver):
# This is the most common per-value translator, so define it once here.
value_trans = {'translation-type': 'VALUE'}
organizations_translator = {
'translation-type': 'HDICT',
'table-name': 'organizations',
'selector-type': 'DICT_SELECTOR',
'field-translators':
({'fieldname': 'guid', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans})}
apps_translator = {
'translation-type': 'HDICT',
'table-name': 'apps',
'in-list': True,
'parent-key': 'guid',
'parent-col-name': 'space_guid',
'selector-type': 'DICT_SELECTOR',
'field-translators':
({'fieldname': 'guid', 'translator': value_trans},
{'fieldname': 'buildpack', 'translator': value_trans},
{'fieldname': 'command', 'translator': value_trans},
{'fieldname': 'console', 'translator': value_trans},
{'fieldname': 'debug', 'translator': value_trans},
{'fieldname': 'detect_buildpack', 'translator': value_trans},
{'fieldname': 'detect_start_command', 'translator': value_trans},
{'fieldname': 'disk_quota', 'translator': value_trans},
{'fieldname': 'docker_image', 'translator': value_trans},
{'fieldname': 'enviroment_json', 'translator': value_trans},
{'fieldname': 'health_check_timeout', 'translator': value_trans},
{'fieldname': 'instances', 'translator': value_trans},
{'fieldname': 'memory', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'package_state', 'translator': value_trans},
{'fieldname': 'package_updated_at', 'translator': value_trans},
{'fieldname': 'production', 'translator': value_trans},
{'fieldname': 'staging_failed_reason', 'translator': value_trans},
{'fieldname': 'staging_task_id', 'translator': value_trans},
{'fieldname': 'state', 'translator': value_trans},
{'fieldname': 'version', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans})}
spaces_translator = {
'translation-type': 'HDICT',
'table-name': 'spaces',
'selector-type': 'DICT_SELECTOR',
'field-translators':
({'fieldname': 'guid', 'translator': value_trans},
{'fieldname': 'name', 'translator': value_trans},
{'fieldname': 'created_at', 'translator': value_trans},
{'fieldname': 'updated_at', 'translator': value_trans},
{'fieldname': 'apps', 'translator': apps_translator})}
def __init__(self, name='', keys='', inbox=None,
datapath=None, args=None):
super(CloudFoundryV2Driver, self).__init__(name, keys, inbox,
datapath, args)
self.creds = datasource_utils.get_credentials(name, args)
self.register_translator(CloudFoundryV2Driver.organizations_translator)
self.register_translator(CloudFoundryV2Driver.spaces_translator)
self.cloudfoundry = client.Client(username=self.creds['username'],
password=self.creds['password'],
base_url=self.creds['auth_url'])
self.cloudfoundry.login()
# Store raw state (result of API calls) so that we can
# avoid re-translating and re-sending if no changes occurred.
# Because translation is not deterministic (we're generating
# UUIDs), it's hard to tell if no changes occurred
# after performing the translation.
self.raw_state = {}
self.initialized = True
self._cached_organizations = []
def _save_organizations(self, organizations):
temp_organizations = []
for organization in organizations['resources']:
temp_organizations.append(organization['metadata']['guid'])
self._cached_organizations = temp_organizations
def update_from_datasource(self):
LOG.debug("CloudFoundry grabbing Data")
organizations = self.cloudfoundry.get_organizations()
if ('organizations' not in self.raw_state or
organizations != self.raw_state['organizations']):
self.raw_state['organizations'] = organizations
self._translate_organizations(organizations)
self._save_organizations(organizations)
spaces = []
for org in self._cached_organizations:
temp_spaces = self.cloudfoundry.get_organization_spaces(org)
for temp_space in temp_spaces['resources']:
spaces.append(dict(temp_space['metadata'].items() +
temp_space['entity'].items()))
for space in spaces:
space['apps'] = []
temp_apps = self.cloudfoundry.get_apps_in_space(space['guid'])
for temp_app in temp_apps['resources']:
space['apps'].append(dict(temp_app['metadata'].items() +
temp_app['entity'].items()))
if ('spaces' not in self.raw_state or
spaces != self.raw_state['spaces']):
self.raw_state['spaces'] = spaces
self._translate_spaces(spaces)
def _translate_organizations(self, obj):
LOG.debug("organziations: %s", obj)
# convert_objs needs the data structured a specific way so we
# do this here. Perhaps we can improve convert_objs later to be
# more flexiable.
results = [dict(o['metadata'].items() + o['entity'].items())
for o in obj['resources']]
row_data = CloudFoundryV2Driver.convert_objs(
results,
self.organizations_translator)
self.state['organizations'] = set()
for table, row in row_data:
self.state[table].add(row)
def _translate_spaces(self, obj):
LOG.debug("spaces: %s", obj)
row_data = CloudFoundryV2Driver.convert_objs(
obj,
self.spaces_translator)
self.state['spaces'] = set()
for table, row in row_data:
self.state[table].add(row)

View File

@ -0,0 +1,299 @@
# Copyright (c) 2013 VMware, Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import contextlib
import sys
# NOTE(arosen): done to avoid the fact that cloudfoundryclient
# isn't in the openstack global reqirements.
import mock
sys.modules['cloudfoundryclient.v2.client'] = mock.Mock()
sys.modules['cloudfoundryclient.v2'] = mock.Mock()
sys.modules['cloudfoundryclient'] = mock.Mock()
from congress.datasources import cloudfoundryv2_driver
from congress.tests import base
from congress.tests import helper
ORG1_GUID = '5187136c-ef7d-47e6-9e6b-ac7780bab3db'
ORG_DATA = (
{"total_results": 1,
"next_url": 'null',
"total_pages": 1,
"prev_url": 'null',
"resources": [{
"entity":
{"status": "active",
"spaces_url": "/v2/organizations/" + ORG1_GUID + "/spaces",
"private_domains_url":
"/v2/organizations/" + ORG1_GUID + "/private_domains",
"name": "foo.com",
"domains_url":
"/v2/organizations/" + ORG1_GUID + "/domains",
"billing_enabled": 'true',
"quota_definition_guid":
"b72b1acb-ff4f-468d-99c0-05cd91012b62",
"app_events_url":
"/v2/organizations/" + ORG1_GUID + "/app_events",
"space_quota_definitions_url":
"/v2/organizations/" + ORG1_GUID + "/space_quota_definitions",
"quota_definition_url":
"/v2/quota_definitions/b72b1acb-ff4f-468d-99c0-05cd91012b62",
"auditors_url":
"/v2/organizations/" + ORG1_GUID + "/auditors",
"managers_url":
"/v2/organizations/" + ORG1_GUID + "/managers",
"users_url":
"/v2/organizations/" + ORG1_GUID + "/users",
"billing_managers_url":
"/v2/organizations/" + ORG1_GUID + "/billing_managers"
},
"metadata":
{"url":
"/v2/organizations/5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"created_at": "2015-01-21T02:17:28+00:00",
"guid": "5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"updated_at": "2015-01-21T02:17:28+00:00"
}
}
]
}
)
SPACE1_GUID = "8da5477d-340e-4bb4-808a-54d9f72017d1"
SPACE2_GUID = "79479021-1e77-473a-8c63-28de9d2ca697"
ORG1_SPACES_DATA = (
{"total_results": 2,
"next_url": "null",
"total_pages": 1,
"prev_url": "null",
"resources": [{
"entity":
{"developers_url": "/v2/spaces/" + SPACE1_GUID + "/developers",
"service_instances_url":
"/v2/spaces/" + SPACE1_GUID + "/service_instances",
"events_url": "/v2/spaces/" + SPACE1_GUID + "/events",
"name": "development",
"domains_url": "/v2/spaces/" + SPACE1_GUID + "/domains",
"app_events_url": "/v2/spaces/" + SPACE1_GUID + "/app_events",
"routes_url": "/v2/spaces/" + SPACE1_GUID + "/routes",
"organization_guid": "5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"space_quota_definition_guid": "null",
"apps_url": "/v2/spaces/" + SPACE1_GUID + "/apps",
"auditors_url": "/v2/spaces/" + SPACE1_GUID + "/auditors",
"managers_url": "/v2/spaces/" + SPACE1_GUID + "/managers",
"organization_url":
"/v2/organizations/5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"security_groups_url":
"/v2/spaces/" + SPACE1_GUID + "/security_groups"
},
"metadata":
{"url": "/v2/spaces/" + SPACE1_GUID,
"created_at": "2015-01-21T02:17:28+00:00",
"guid": SPACE1_GUID,
"updated_at": "null"
}
},
{"entity":
{"developers_url": "/v2/spaces/" + SPACE2_GUID + "/developers",
"service_instances_url":
"/v2/spaces/" + SPACE2_GUID + "/service_instances",
"events_url": "/v2/spaces/" + SPACE2_GUID + "/events",
"name": "test2",
"domains_url": "/v2/spaces/" + SPACE2_GUID + "/domains",
"app_events_url": "/v2/spaces/" + SPACE2_GUID + "/app_events",
"routes_url": "/v2/spaces/" + SPACE2_GUID + "/routes",
"organization_guid": "5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"space_quota_definition_guid": "null",
"apps_url": "/v2/spaces/" + SPACE2_GUID + "/apps",
"auditors_url": "/v2/spaces/" + SPACE2_GUID + "/auditors",
"managers_url": "/v2/spaces/" + SPACE2_GUID + "/managers",
"organization_url":
"/v2/organizations/5187136c-ef7d-47e6-9e6b-ac7780bab3db",
"security_groups_url":
"/v2/spaces/" + SPACE2_GUID + "/security_groups"
},
"metadata":
{"url": "/v2/spaces/" + SPACE2_GUID,
"created_at": "2015-01-22T19:02:32+00:00",
"guid": SPACE2_GUID,
"updated_at": "null"
}
}
]
}
)
APP1_GUID = "c3bd7fc1-73b4-4cc7-a6c8-9976c30edad5"
APP2_GUID = "f7039cca-95ac-49a6-b116-e32a53ddda69"
APPS_IN_SPACE1 = (
{"total_results": 2,
"next_url": "null",
"total_pages": 1,
"prev_url": "null",
"resources": [{
"entity":
{"version": "fec00ce7-a980-49e1-abec-beed5516618f",
"staging_failed_reason": "null",
"instances": 1,
"routes_url": "/v2/apps" + APP1_GUID + "routes",
"space_url": "/v2/spaces/8da5477d-340e-4bb4-808a-54d9f72017d1",
"docker_image": "null",
"console": "false",
"package_state": "STAGED",
"state": "STARTED",
"production": "false",
"detected_buildpack": "Ruby",
"memory": 256,
"package_updated_at": "2015-01-21T21:00:40+00:00",
"staging_task_id": "71f75ad3cad64884a92c4e7738eaae16",
"buildpack": "null",
"stack_url": "/v2/stacks/50688ae5-9bfc-4bf6-a4bf-caadb21a32c6",
"events_url": "/v2/apps" + APP1_GUID + "events",
"service_bindings_url":
"/v2/apps" + APP1_GUID + "service_bindings",
"detected_start_command":
"bundle exec rake db:migrate && bundle exec rails s -p $PORT",
"disk_quota": 1024,
"stack_guid": "50688ae5-9bfc-4bf6-a4bf-caadb21a32c6",
"space_guid": "8da5477d-340e-4bb4-808a-54d9f72017d1",
"name": "rails_sample_app",
"health_check_type": "port",
"command":
"bundle exec rake db:migrate && bundle exec rails s -p $PORT",
"debug": "null",
"environment_json": {},
"health_check_timeout": "null"
},
"metadata":
{"url": "/v2/apps/c3bd7fc1-73b4-4cc7-a6c8-9976c30edad5",
"created_at": "2015-01-21T21:01:19+00:00",
"guid": "c3bd7fc1-73b4-4cc7-a6c8-9976c30edad5",
"updated_at": "2015-01-21T21:01:19+00:00"
}
},
{"entity":
{"version": "a1b52559-32f3-4765-9fd3-6e35293fb6d0",
"staging_failed_reason": "null",
"instances": 1,
"routes_url": "/v2/apps" + APP2_GUID + "routes",
"space_url": "/v2/spaces/8da5477d-340e-4bb4-808a-54d9f72017d1",
"docker_image": "null",
"console": "false",
"package_state": "PENDING",
"state": "STOPPED",
"production": "false",
"detected_buildpack": "null",
"memory": 1024,
"package_updated_at": "null",
"staging_task_id": "null",
"buildpack": "null",
"stack_url": "/v2/stacks/50688ae5-9bfc-4bf6-a4bf-caadb21a32c6",
"events_url": "/v2/apps" + APP2_GUID + "events",
"service_bindings_url":
"/v2/apps" + APP2_GUID + "service_bindings",
"detected_start_command": "",
"disk_quota": 1024,
"stack_guid": "50688ae5-9bfc-4bf6-a4bf-caadb21a32c6",
"space_guid": "8da5477d-340e-4bb4-808a-54d9f72017d1",
"name": "help",
"health_check_type": "port",
"command": "null",
"debug": "null",
"environment_json": {},
"health_check_timeout": "null"
},
"metadata":
{"url": "/v2/apps/f7039cca-95ac-49a6-b116-e32a53ddda69",
"created_at": "2015-01-21T18:48:34+00:00",
"guid": "f7039cca-95ac-49a6-b116-e32a53ddda69",
"updated_at": "null"
}
}
]
}
)
APPS_IN_SPACE2 = {"total_results": 0,
"next_url": "null",
"total_pages": 1,
"prev_url": "null",
"resources": []}
EXPECTED_STATE = {
'organizations': set([
('5187136c-ef7d-47e6-9e6b-ac7780bab3db', 'foo.com',
'2015-01-21T02:17:28+00:00', '2015-01-21T02:17:28+00:00')]),
'spaces': set([
('8da5477d-340e-4bb4-808a-54d9f72017d1', 'development',
'2015-01-21T02:17:28+00:00', 'null'),
('79479021-1e77-473a-8c63-28de9d2ca697', 'test2',
'2015-01-22T19:02:32+00:00', 'null')]),
'apps': set([
('8da5477d-340e-4bb4-808a-54d9f72017d1',
'c3bd7fc1-73b4-4cc7-a6c8-9976c30edad5', 'null',
'bundle exec rake db:migrate && bundle exec rails s -p $PORT',
'false', 'null', 'None', 'None', 1024, 'null', 'None', 'null', 1,
256, 'rails_sample_app', 'STAGED', '2015-01-21T21:00:40+00:00',
'false', 'null', '71f75ad3cad64884a92c4e7738eaae16', 'STARTED',
'fec00ce7-a980-49e1-abec-beed5516618f', '2015-01-21T21:01:19+00:00',
'2015-01-21T21:01:19+00:00'),
('8da5477d-340e-4bb4-808a-54d9f72017d1',
'f7039cca-95ac-49a6-b116-e32a53ddda69', 'null', 'null', 'false',
'null', 'None', 'None', 1024, 'null', 'None', 'null', 1, 1024,
'help', 'PENDING', 'null', 'false', 'null', 'null', 'STOPPED',
'a1b52559-32f3-4765-9fd3-6e35293fb6d0',
'2015-01-21T18:48:34+00:00', 'null')])}
class TestCloudFoundryV2Driver(base.TestCase):
def setUp(self):
super(TestCloudFoundryV2Driver, self).setUp()
args = helper.datasource_openstack_args()
args['poll_time'] = 0
args['client'] = mock.MagicMock()
self.driver = cloudfoundryv2_driver.CloudFoundryV2Driver(args=args)
def test_update_from_datasource(self):
def _side_effect_get_org_spaces(org):
if org == ORG1_GUID:
return ORG1_SPACES_DATA
raise ValueError("This should occur...")
def _side_effect_get_apps_in_space(space):
if space == SPACE1_GUID:
return APPS_IN_SPACE1
elif space == SPACE2_GUID:
return APPS_IN_SPACE2
else:
raise ValueError("This should not occur....")
with contextlib.nested(
mock.patch.object(self.driver.cloudfoundry,
"get_organizations",
return_value=ORG_DATA),
mock.patch.object(self.driver.cloudfoundry,
"get_organization_spaces",
side_effect=_side_effect_get_org_spaces),
mock.patch.object(self.driver.cloudfoundry,
"get_apps_in_space",
side_effect=_side_effect_get_apps_in_space),
) as (get_organizations, get_organization_spaces,
get_apps_in_space):
self.driver.update_from_datasource()
self.assertEqual(self.driver.state, EXPECTED_STATE)

View File

@ -68,3 +68,10 @@ username: admin
password: password
auth_url: http://127.0.0.1:5000/v2.0
tenant_name: admin
[cloudfoundryv2]
module: datasources/cloudfoundryv2_driver.py
username: foo@bar.com
password: cloudfoundry_password
auth_url: https://api.run.pivotal.io/
tenant_name: foo@bar.com

View File

@ -0,0 +1 @@
python-cloudfoundryclient