diff --git a/poppy/manager/default/services.py b/poppy/manager/default/services.py index f9f61a09..bb029ff7 100644 --- a/poppy/manager/default/services.py +++ b/poppy/manager/default/services.py @@ -107,6 +107,13 @@ class DefaultServicesController(base.ServicesController): return services_project_ids + def get_domains_by_provider_url(self, provider_url): + + domains = \ + self.storage_controller.get_domains_by_provider_url(provider_url) + + return domains + def _append_defaults(self, service_json, operation='create'): # default origin rule for origin in service_json.get('origins', []): diff --git a/poppy/storage/cassandra/migrations/010_provider_url_domain.cql b/poppy/storage/cassandra/migrations/010_provider_url_domain.cql new file mode 100644 index 00000000..81b363c2 --- /dev/null +++ b/poppy/storage/cassandra/migrations/010_provider_url_domain.cql @@ -0,0 +1,9 @@ +CREATE TABLE provider_url_domain ( + provider_url VARCHAR, + domain_name VARCHAR, + PRIMARY KEY (provider_url, domain_name)); + + +--//@UNDO + +DROP TABLE provider_url_domain; \ No newline at end of file diff --git a/poppy/storage/cassandra/services.py b/poppy/storage/cassandra/services.py index 188d2b07..1e878206 100644 --- a/poppy/storage/cassandra/services.py +++ b/poppy/storage/cassandra/services.py @@ -278,6 +278,23 @@ CQL_SET_SERVICE_STATUS = ''' %(status)s) ''' +CQL_SET_PROVIDER_URL = ''' + INSERT INTO provider_url_domain(provider_url, + domain_name) + VALUES (%(provider_url)s, %(domain_name)s) +''' + +CQL_DELETE_PROVIDER_URL = ''' + DELETE FROM provider_url_domain + WHERE provider_url = %(provider_url)s + AND domain_name = %(domain_name)s +''' + +CQL_GET_BY_PROVIDER_URL = ''' + SELECT domain_name FROM provider_url_domain + WHERE provider_url = %(provider_url)s +''' + CQL_GET_SERVICE_STATUS = ''' SELECT project_id, service_id @@ -489,6 +506,39 @@ class ServicesController(base.ServicesController): self.session.execute(stmt, args) + def get_domains_by_provider_url(self, provider_url): + + LOG.info("Getting domains by provider_url: {0}".format(provider_url)) + + get_domain_provider_url_args = { + 'provider_url': provider_url, + } + + stmt = query.SimpleStatement( + CQL_GET_BY_PROVIDER_URL, + consistency_level=self._driver.consistency_level) + + resultset = self.session.execute(stmt, get_domain_provider_url_args) + + return list(resultset) + + def delete_provider_url(self, provider_url, domain_name): + + LOG.info("Deleting provider_url: {0} and " + "domain_name: {1} from provider_url_domain " + "column family".format(provider_url, domain_name)) + + del_provider_url_args = { + 'provider_url': provider_url, + 'domain_name': domain_name + } + + stmt = query.SimpleStatement( + CQL_DELETE_PROVIDER_URL, + consistency_level=self._driver.consistency_level) + + self.session.execute(stmt, del_provider_url_args) + def get_service_limit(self, project_id): """get_service_limit @@ -577,12 +627,22 @@ class ServicesController(base.ServicesController): """ - LOG.info("Setting service" - "status for" + LOG.info("Setting service " + "status for " "service_id : {0}, " "project_id: {1} to be {2}".format(service_id, project_id, status)) + status_args = { + 'service_id': uuid.UUID(str(service_id)), + 'project_id': project_id, + 'status': status + } + + stmt = query.SimpleStatement( + CQL_SET_SERVICE_STATUS, + consistency_level=self._driver.consistency_level) + self.session.execute(stmt, status_args) provider_details_dict = self.get_provider_details( project_id=project_id, @@ -839,6 +899,7 @@ class ServicesController(base.ServicesController): consistency_level=self._driver.consistency_level) self.session.execute(stmt, args) + self.set_service_provider_details(project_id, service_id, status) # claim new domains batch_claim = query.BatchStatement( consistency_level=self._driver.consistency_level) @@ -875,17 +936,6 @@ class ServicesController(base.ServicesController): consistency_level=self._driver.consistency_level) self.session.execute(stmt, args) - status_args = { - 'service_id': uuid.UUID(str(service_id)), - 'project_id': project_id, - 'status': status - } - - stmt = query.SimpleStatement( - CQL_SET_SERVICE_STATUS, - consistency_level=self._driver.consistency_level) - self.session.execute(stmt, status_args) - def update_state(self, project_id, service_id, state): """update_state @@ -931,11 +981,24 @@ class ServicesController(base.ServicesController): pds = result.get('provider_details', {}) or {} pds = {key: value for key, value in pds.items()} status = None + provider_urls_domain = [] + for provider in pds: pds_provider_dict = json.loads(pds.get(provider, {})) status = pds_provider_dict.get('status', '') + access_urls = pds_provider_dict.get('access_urls', []) + for access_url in access_urls: + provider_url = access_url.get('provider_url', None) + domain = access_url.get('domain', None) + if provider_url and domain: + provider_urls_domain.append((provider_url, domain)) + self.delete_services_by_status(project_id, service_id, status) + for provider_url_domain in provider_urls_domain: + provider_url, domain = provider_url_domain + self.delete_provider_url(provider_url, domain) + if self._driver.archive_on_delete: archive_args = { 'project_id': result.get('project_id'), @@ -1082,12 +1145,19 @@ class ServicesController(base.ServicesController): provider_detail_dict = {} status = None + domain_names_provider_urls = [] for provider_name in sorted(provider_details.keys()): the_provider_detail_dict = collections.OrderedDict() the_provider_detail_dict["id"] = ( provider_details[provider_name].provider_service_id) the_provider_detail_dict["access_urls"] = ( provider_details[provider_name].access_urls) + for access_url in the_provider_detail_dict["access_urls"]: + domain_name = access_url.get("domain", None) + provider_url = access_url.get("provider_url", None) + if domain_name and provider_url: + domain_names_provider_urls.append((domain_name, + provider_url)) the_provider_detail_dict["status"] = ( provider_details[provider_name].status) status = the_provider_detail_dict["status"] @@ -1118,7 +1188,7 @@ class ServicesController(base.ServicesController): consistency_level=self._driver.consistency_level) self.session.execute(stmt, args) - args = { + service_args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), 'status': status @@ -1127,7 +1197,19 @@ class ServicesController(base.ServicesController): stmt = query.SimpleStatement( CQL_SET_SERVICE_STATUS, consistency_level=self._driver.consistency_level) - self.session.execute(stmt, args) + self.session.execute(stmt, service_args) + + if domain_names_provider_urls: + for domain_name, provider_url in domain_names_provider_urls: + provider_url_args = { + 'domain_name': domain_name, + 'provider_url': provider_url + } + + stmt = query.SimpleStatement( + CQL_SET_PROVIDER_URL, + consistency_level=self._driver.consistency_level) + self.session.execute(stmt, provider_url_args) def update_cert_info(self, domain_name, cert_type, flavor_id, cert_details): diff --git a/poppy/transport/pecan/controllers/v1/admin.py b/poppy/transport/pecan/controllers/v1/admin.py index 3eb0ede7..e2a34dc7 100644 --- a/poppy/transport/pecan/controllers/v1/admin.py +++ b/poppy/transport/pecan/controllers/v1/admin.py @@ -413,7 +413,6 @@ class AdminCertController(base.Controller, hooks.HookController): def __init__(self, driver): super(AdminCertController, self).__init__(driver) - @pecan.expose('json') @pecan.expose('json') @decorators.validate( request=rule.Rule( @@ -443,7 +442,6 @@ class AdminServiceController(base.Controller, hooks.HookController): self.__class__.action = OperatorServiceActionController(driver) self.__class__.status = ServiceStatusController(driver) - @pecan.expose('json') @pecan.expose('json') @decorators.validate( request=rule.Rule( @@ -490,6 +488,25 @@ class DomainController(base.Controller, hooks.HookController): # convert a service model into a response service model return resp_service_model.Model(service_obj, self) + @pecan.expose('json') + @decorators.validate( + request=rule.Rule( + helpers.is_valid_provider_url(), + helpers.abort_with_message, + stoplight_helpers.pecan_getter) + ) + def get(self): + services_controller = self._driver.manager.services_controller + + call_args = getattr(pecan.request.context, + "call_args") + provider_url = call_args.pop('provider_url') + domains = services_controller.get_domains_by_provider_url( + provider_url) + + return pecan.Response(json_body=domains, + status=200) + class AdminController(base.Controller, hooks.HookController): def __init__(self, driver): diff --git a/poppy/transport/validators/helpers.py b/poppy/transport/validators/helpers.py index c3c96ba3..634277b1 100644 --- a/poppy/transport/validators/helpers.py +++ b/poppy/transport/validators/helpers.py @@ -467,6 +467,30 @@ def is_valid_domain_by_name(domain_name): u'Domain {0} is not valid'.format(domain_name)) +@decorators.validation_function +def is_valid_provider_url(request): + + provider_url = request.GET.get("provider_url", None) + if not provider_url: + raise exceptions.ValidationFailed('provider_url needs to be ' + 'provided as a query parameter') + provider_url_regex_1 = ('^([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.' + '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.' + '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}$') + provider_url_regex_2 = ('^([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.' + '([A-Za-z0-9-]){1,255}\.([A-Za-z0-9-]){1,255}\.' + '([A-Za-z0-9-]){1,255}$') + if not re.match(provider_url_regex_1, provider_url): + if not re.match(provider_url_regex_2, provider_url): + raise exceptions.ValidationFailed( + u'Provider url {0} is not valid'.format(provider_url)) + + # Update context so the decorated function can get all this parameters + request.context.call_args = { + 'provider_url': provider_url, + } + + def is_valid_flavor_configuration(flavor, schema): if schema is not None: errors_list = list( diff --git a/tests/api/utils/schema/admin.py b/tests/api/utils/schema/admin.py index 1aa7a02a..02dc0296 100644 --- a/tests/api/utils/schema/admin.py +++ b/tests/api/utils/schema/admin.py @@ -19,7 +19,6 @@ from tests.api.utils.schema.services import project_id from tests.api.utils.schema.services import service_id get_service_project_status = { - 'type': 'array', 'items': [ { 'type': 'object', diff --git a/tests/functional/transport/pecan/controllers/test_get_domain_by_provider_url.py b/tests/functional/transport/pecan/controllers/test_get_domain_by_provider_url.py new file mode 100644 index 00000000..15b5c345 --- /dev/null +++ b/tests/functional/transport/pecan/controllers/test_get_domain_by_provider_url.py @@ -0,0 +1,83 @@ +# Copyright (c) 2015 Rackspace, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import ddt +from hypothesis import given +from hypothesis import strategies +import mock +import six + +from poppy.manager.default.services import DefaultServicesController + +from tests.functional.transport.pecan import base + + +@ddt.ddt +class TestGetDomainsbyProviderurl(base.FunctionalTest): + + def test_get_domains_provider_url_no_queryparam(self): + # no provider_url field + url = '/v1.0/admin/domains' + response = self.app.get(url, + headers={'Content-Type': + 'application/json', + 'X-Project-ID': + str(uuid.uuid4())}, + expect_errors=True) + self.assertEqual(response.status_code, 400) + + @given(strategies.text()) + def test_get_domains_provider_url_invalid_queryparam(self, + provider_url): + # invalid provider_url field + try: + # NOTE(TheSriram): Py3k Hack + if six.PY3 and type(provider_url) == str: + provider_url = provider_url.encode('utf-8') + url = '/v1.0/admin/domains?' \ + 'provider_url={0}'.format(provider_url) + + else: + url = '/v1.0/admin/domains?provider_url=%s' \ + % provider_url.decode('utf-8') + except (UnicodeDecodeError, UnicodeEncodeError): + pass + else: + response = self.app.get(url, + headers={'Content-Type': + 'application/json', + 'X-Project-ID': + str(uuid.uuid4())}, + expect_errors=True) + + self.assertEqual(response.status_code, 400) + + @ddt.data('provider.com.extension.provideredge.net', + 'secure.shard.domain.com.provideredge.net', + 'www.domain.com.provideredge.net') + def test_get_domains_provider_url_valid_queryparam(self, provider_url): + # valid provider_url + with mock.patch.object(DefaultServicesController, + 'get_domains_by_provider_url'): + response = self.app.get('/v1.0/admin/domains' + '?provider_url={0}'.format(provider_url), + headers={'Content-Type': + 'application/json', + 'X-Project-ID': + str(uuid.uuid4())}) + + self.assertEqual(response.status_code, 200) diff --git a/tests/unit/storage/cassandra/data_provider_details.json b/tests/unit/storage/cassandra/data_provider_details.json index 457f73b1..cf7a1893 100644 --- a/tests/unit/storage/cassandra/data_provider_details.json +++ b/tests/unit/storage/cassandra/data_provider_details.json @@ -1,9 +1,9 @@ { "provider_details": { - "MaxCDN": "{\"id\": 11942, \"access_urls\": [\"mypullzone.netdata.com\"], \"domains_certificate_status\":{\"mypullzone.com\": \"failed\"} }", - "Mock": "{\"id\": 73242, \"access_urls\": [\"mycdn.mock.com\"], \"domains_certificate_status\":{\"mycdn.mock.com\": \"deployed\"} }", - "CloudFront": "{\"id\": \"5ABC892\", \"access_urls\": [\"cf123.cloudcf.com\"]}", - "Fastly": "{\"id\": 3488, \"access_urls\": [\"mockcf123.fastly.prod.com\"], \"domains_certificate_status\":{\"mockcf123.com\": \"create_in_progress\"}}" + "MaxCDN": "{\"id\": 11942, \"access_urls\": [{\"provider_url\": \"maxcdn.provider.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mypullzone.com\": \"failed\"} }", + "Mock": "{\"id\": 73242, \"access_urls\": [{\"provider_url\": \"mycdn.mock.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mycdn.mock.com\": \"deployed\"} }", + "CloudFront": "{\"id\": \"5ABC892\", \"access_urls\": [{\"provider_url\": \"cloudfront.provider.com\", \"domain\": \"xk.cd\"}]}", + "Fastly": "{\"id\": 3488, \"access_urls\": [{\"provider_url\": \"fastly.provider.com\", \"domain\": \"xk.cd\"}], \"domains_certificate_status\":{\"mockcf123.com\": \"create_in_progress\"}}" } } \ No newline at end of file diff --git a/tests/unit/storage/cassandra/test_services.py b/tests/unit/storage/cassandra/test_services.py index ecafec4a..764c46ab 100644 --- a/tests/unit/storage/cassandra/test_services.py +++ b/tests/unit/storage/cassandra/test_services.py @@ -154,18 +154,32 @@ class CassandraStorageServiceTests(base.TestCase): return_value=False) @mock.patch.object(services.ServicesController, 'session') @mock.patch.object(cassandra.cluster.Session, 'execute') + @mock.patch.object(services.ServicesController, + 'set_service_provider_details') def test_update_service(self, service_json, + mock_set_service_provider_details, mock_execute, mock_session, mock_check): - mock_check.return_value = False - mock_session.execute.return_value = iter([{}]) - service_obj = req_service.load_from_json(service_json) - actual_response = self.sc.update(self.project_id, - self.service_id, - service_obj) + with mock.patch.object( + services.ServicesController, + 'get_provider_details') as mock_provider_det: - # Expect the response to be None as there are no providers passed - # into the driver to respond to this call - self.assertEqual(actual_response, None) + mock_provider_det.return_value = { + "MaxCDN": "{\"id\": 11942, \"access_urls\": " + "[{\"provider_url\": \"maxcdn.provider.com\", " + "\"domain\": \"xk.cd\"}], " + "\"domains_certificate_status\":" + "{\"mypullzone.com\": " + "\"failed\"} }", + } + mock_session.execute.return_value = iter([{}]) + service_obj = req_service.load_from_json(service_json) + actual_response = self.sc.update(self.project_id, + self.service_id, + service_obj) + + # Expect the response to be None as there are no + # providers passed into the driver to respond to this call + self.assertEqual(actual_response, None) @ddt.file_data('data_provider_details.json') @mock.patch.object(services.ServicesController, 'session') @@ -237,6 +251,7 @@ class CassandraStorageServiceTests(base.TestCase): provider_details_dict = {} for k, v in provider_details_json.items(): provider_detail_dict = json.loads(v) + provider_details_dict[k] = provider_details.ProviderDetail( provider_service_id=( provider_detail_dict["id"]),