Check shard is within records limit
Before adding a new record to a domain, check to see if it's at maximum capacity and skip to the next cert if necessary. Change-Id: I7e4c56e27cebc45a9cdddf9659762f29f9c6227c
This commit is contained in:
parent
5232069ad4
commit
0458de585d
|
@ -120,6 +120,7 @@ username = "<operator_username>"
|
||||||
api_key = "<operator_api_key>"
|
api_key = "<operator_api_key>"
|
||||||
use_shards = True
|
use_shards = True
|
||||||
num_shards = 400
|
num_shards = 400
|
||||||
|
;records_limit = 400
|
||||||
shard_prefix = "cdn"
|
shard_prefix = "cdn"
|
||||||
shared_ssl_num_shards = 5
|
shared_ssl_num_shards = 5
|
||||||
shared_ssl_shard_prefix = "scdn"
|
shared_ssl_shard_prefix = "scdn"
|
||||||
|
|
|
@ -69,3 +69,11 @@ class ServicesControllerBase(controller.DNSControllerBase):
|
||||||
"""
|
"""
|
||||||
|
|
||||||
raise NotImplementedError
|
raise NotImplementedError
|
||||||
|
|
||||||
|
def is_shard_full(self, shard_name):
|
||||||
|
"""Check if shard can accept new records
|
||||||
|
|
||||||
|
:raises NotImplementedError
|
||||||
|
"""
|
||||||
|
|
||||||
|
raise NotImplementedError
|
||||||
|
|
|
@ -90,3 +90,6 @@ class ServicesController(base.ServicesBase):
|
||||||
random.shuffle(shard_ids)
|
random.shuffle(shard_ids)
|
||||||
for shard in shard_ids:
|
for shard in shard_ids:
|
||||||
yield 'scdn{0}.secure.defaultcdn.com'.format(shard)
|
yield 'scdn{0}.secure.defaultcdn.com'.format(shard)
|
||||||
|
|
||||||
|
def is_shard_full(self, shard_name):
|
||||||
|
return False
|
||||||
|
|
|
@ -32,6 +32,8 @@ RACKSPACE_OPTIONS = [
|
||||||
cfg.BoolOpt('sharding_enabled', default=True,
|
cfg.BoolOpt('sharding_enabled', default=True,
|
||||||
help='Enable Sharding?'),
|
help='Enable Sharding?'),
|
||||||
cfg.IntOpt('num_shards', default=10, help='Number of Shards to use'),
|
cfg.IntOpt('num_shards', default=10, help='Number of Shards to use'),
|
||||||
|
cfg.IntOpt('records_limit', default=400,
|
||||||
|
help='Number of records per domain.'),
|
||||||
cfg.StrOpt('shard_prefix', default='cdn',
|
cfg.StrOpt('shard_prefix', default='cdn',
|
||||||
help='The shard prefix to use'),
|
help='The shard prefix to use'),
|
||||||
cfg.IntOpt('shared_ssl_num_shards', default=5, help='Number of Shards '
|
cfg.IntOpt('shared_ssl_num_shards', default=5, help='Number of Shards '
|
||||||
|
|
|
@ -799,3 +799,33 @@ class ServicesController(base.ServicesBase):
|
||||||
def modify_cname(self, access_url, new_cert):
|
def modify_cname(self, access_url, new_cert):
|
||||||
self._change_cname_record(access_url=access_url,
|
self._change_cname_record(access_url=access_url,
|
||||||
target_url=new_cert, shared_ssl_flag=False)
|
target_url=new_cert, shared_ssl_flag=False)
|
||||||
|
|
||||||
|
def is_shard_full(self, shard_name):
|
||||||
|
count = 0
|
||||||
|
try:
|
||||||
|
shard_domain = self.client.find(name=shard_name)
|
||||||
|
except exc.NotFound:
|
||||||
|
LOG.error("Shards not configured properly, could not find {0}.")
|
||||||
|
return True
|
||||||
|
|
||||||
|
records = shard_domain.list_records(limit=100)
|
||||||
|
count += len(records)
|
||||||
|
|
||||||
|
# Loop until all records are printed
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
records = self.client.list_records_next_page()
|
||||||
|
count += len(records)
|
||||||
|
except exc.NoMoreResults:
|
||||||
|
break
|
||||||
|
|
||||||
|
LOG.info(
|
||||||
|
"There were a total of {0} record(s) for {1}.".format(
|
||||||
|
count,
|
||||||
|
shard_name
|
||||||
|
))
|
||||||
|
|
||||||
|
if count >= self._driver.rackdns_conf.records_limit:
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
|
@ -685,14 +685,19 @@ class DefaultServicesController(base.ServicesController):
|
||||||
uuid_store[domain_name] = \
|
uuid_store[domain_name] = \
|
||||||
self.dns_controller.generate_shared_ssl_domain_suffix()
|
self.dns_controller.generate_shared_ssl_domain_suffix()
|
||||||
setattr(self, store, uuid_store)
|
setattr(self, store, uuid_store)
|
||||||
return '.'.join([domain_name,
|
|
||||||
next(uuid_store[domain_name])])
|
shard = next(uuid_store[domain_name])
|
||||||
|
while self.dns_controller.is_shard_full(shard):
|
||||||
|
LOG.info(
|
||||||
|
"Skipped shard {0} because it's at maximum capacity."
|
||||||
|
.format(shard))
|
||||||
|
shard = next(uuid_store[domain_name])
|
||||||
|
|
||||||
|
return '.'.join([domain_name, shard])
|
||||||
except StopIteration:
|
except StopIteration:
|
||||||
delattr(self, store)
|
delattr(self, store)
|
||||||
raise errors.SharedShardsExhausted('Domain {0} '
|
raise errors.SharedShardsExhausted(
|
||||||
'has already '
|
'Domain {0} has already been taken'.format(domain_name))
|
||||||
'been '
|
|
||||||
'taken'.format(domain_name))
|
|
||||||
|
|
||||||
def _pick_shared_ssl_domain(self, domain, service_id, store):
|
def _pick_shared_ssl_domain(self, domain, service_id, store):
|
||||||
shared_ssl_domain = self._generate_shared_ssl_domain(
|
shared_ssl_domain = self._generate_shared_ssl_domain(
|
||||||
|
@ -707,7 +712,7 @@ class DefaultServicesController(base.ServicesController):
|
||||||
return shared_ssl_domain
|
return shared_ssl_domain
|
||||||
|
|
||||||
def _shard_retry(self, project_id, service_obj, store=None):
|
def _shard_retry(self, project_id, service_obj, store=None):
|
||||||
# deal with shared ssl domains
|
# deal with shared ssl domains
|
||||||
try:
|
try:
|
||||||
for domain in service_obj.domains:
|
for domain in service_obj.domains:
|
||||||
if domain.protocol == 'https' \
|
if domain.protocol == 'https' \
|
||||||
|
|
|
@ -28,12 +28,12 @@ import uuid
|
||||||
|
|
||||||
import jsonschema
|
import jsonschema
|
||||||
import pecan
|
import pecan
|
||||||
|
import tld as tld_helper
|
||||||
|
|
||||||
from poppy.common import util
|
from poppy.common import util
|
||||||
from poppy.transport.validators import root_domain_regexes as regexes
|
from poppy.transport.validators import root_domain_regexes as regexes
|
||||||
from poppy.transport.validators.stoplight import decorators
|
from poppy.transport.validators.stoplight import decorators
|
||||||
from poppy.transport.validators.stoplight import exceptions
|
from poppy.transport.validators.stoplight import exceptions
|
||||||
from tld import get_tld
|
|
||||||
|
|
||||||
|
|
||||||
def req_accepts_json_pecan(request, desired_content_type='application/json'):
|
def req_accepts_json_pecan(request, desired_content_type='application/json'):
|
||||||
|
@ -143,8 +143,10 @@ def is_valid_tld(domain_name):
|
||||||
status = whois.whois(domain_name)['status']
|
status = whois.whois(domain_name)['status']
|
||||||
if status is not None or status != '':
|
if status is not None or status != '':
|
||||||
url = 'https://{domain}'
|
url = 'https://{domain}'
|
||||||
tld_obj = get_tld(url.format(domain=domain_name),
|
tld_obj = tld_helper.get_tld(
|
||||||
as_object=True)
|
url.format(domain=domain_name),
|
||||||
|
as_object=True
|
||||||
|
)
|
||||||
tld = tld_obj.suffix
|
tld = tld_obj.suffix
|
||||||
try:
|
try:
|
||||||
dns.resolver.query(tld + '.', 'SOA')
|
dns.resolver.query(tld + '.', 'SOA')
|
||||||
|
|
|
@ -15,6 +15,8 @@
|
||||||
|
|
||||||
"""Unittests for TaskFlow distributed_task driver implementation."""
|
"""Unittests for TaskFlow distributed_task driver implementation."""
|
||||||
|
|
||||||
|
import mock
|
||||||
|
|
||||||
from poppy.distributed_task.utils import memoized_controllers
|
from poppy.distributed_task.utils import memoized_controllers
|
||||||
from tests.unit import base
|
from tests.unit import base
|
||||||
|
|
||||||
|
@ -24,6 +26,14 @@ class TestMemoizeUtils(base.TestCase):
|
||||||
def setUp(self):
|
def setUp(self):
|
||||||
super(TestMemoizeUtils, self).setUp()
|
super(TestMemoizeUtils, self).setUp()
|
||||||
|
|
||||||
|
rax_dns_set_credentials = mock.patch('pyrax.set_credentials')
|
||||||
|
rax_dns_set_credentials.start()
|
||||||
|
self.addCleanup(rax_dns_set_credentials.stop)
|
||||||
|
|
||||||
|
rax_dns = mock.patch('pyrax.cloud_dns')
|
||||||
|
rax_dns.start()
|
||||||
|
self.addCleanup(rax_dns.stop)
|
||||||
|
|
||||||
def test_memoization_service_controller(self):
|
def test_memoization_service_controller(self):
|
||||||
service_controller_first = \
|
service_controller_first = \
|
||||||
memoized_controllers.task_controllers('poppy')
|
memoized_controllers.task_controllers('poppy')
|
||||||
|
|
|
@ -34,6 +34,8 @@ RACKSPACE_OPTIONS = [
|
||||||
cfg.BoolOpt('sharding_enabled', default=True,
|
cfg.BoolOpt('sharding_enabled', default=True,
|
||||||
help='Enable Sharding?'),
|
help='Enable Sharding?'),
|
||||||
cfg.IntOpt('num_shards', default=500, help='Number of Shards to use'),
|
cfg.IntOpt('num_shards', default=500, help='Number of Shards to use'),
|
||||||
|
cfg.IntOpt('records_limit', default=400,
|
||||||
|
help='Number of records per domain.'),
|
||||||
cfg.StrOpt('shard_prefix', default='cdn',
|
cfg.StrOpt('shard_prefix', default='cdn',
|
||||||
help='The shard prefix to use'),
|
help='The shard prefix to use'),
|
||||||
cfg.StrOpt('url', default='mycdn.com',
|
cfg.StrOpt('url', default='mycdn.com',
|
||||||
|
@ -1047,3 +1049,38 @@ class TestServicesUpdate(base.TestCase):
|
||||||
responder_disable = self.controller.disable(self.service_old)
|
responder_disable = self.controller.disable(self.service_old)
|
||||||
# TODO(isaacm): Add assertions on the returned object
|
# TODO(isaacm): Add assertions on the returned object
|
||||||
self.assertIsNotNone(responder_disable)
|
self.assertIsNotNone(responder_disable)
|
||||||
|
|
||||||
|
def test_is_shard_full_shard_not_found(self):
|
||||||
|
self.client.find.side_effect = exc.NotFound(404)
|
||||||
|
|
||||||
|
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||||
|
|
||||||
|
def test_is_shard_full_false(self):
|
||||||
|
find_mock = mock.Mock()
|
||||||
|
find_mock.list_records.return_value = range(100)
|
||||||
|
self.client.find.return_value = find_mock
|
||||||
|
|
||||||
|
self.client.list_records_next_page.side_effect = exc.NoMoreResults
|
||||||
|
|
||||||
|
self.assertFalse(self.controller.is_shard_full('shard_name'))
|
||||||
|
|
||||||
|
def test_is_shard_full_true(self):
|
||||||
|
find_mock = mock.Mock()
|
||||||
|
find_mock.list_records.return_value = range(600)
|
||||||
|
self.client.find.return_value = find_mock
|
||||||
|
|
||||||
|
self.client.list_records_next_page.side_effect = exc.NoMoreResults
|
||||||
|
|
||||||
|
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||||
|
|
||||||
|
def test_is_shard_full_paginate_true(self):
|
||||||
|
find_mock = mock.Mock()
|
||||||
|
find_mock.list_records.return_value = range(300)
|
||||||
|
self.client.find.return_value = find_mock
|
||||||
|
|
||||||
|
self.client.list_records_next_page.side_effect = [
|
||||||
|
range(300),
|
||||||
|
exc.NoMoreResults,
|
||||||
|
]
|
||||||
|
|
||||||
|
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||||
|
|
|
@ -116,6 +116,14 @@ class DefaultManagerServiceTests(base.TestCase):
|
||||||
# in the reverse order of the arguments present
|
# in the reverse order of the arguments present
|
||||||
super(DefaultManagerServiceTests, self).setUp()
|
super(DefaultManagerServiceTests, self).setUp()
|
||||||
|
|
||||||
|
tld_patcher = mock.patch('tld.get_tld')
|
||||||
|
tld_patcher.start()
|
||||||
|
self.addCleanup(tld_patcher.stop)
|
||||||
|
|
||||||
|
dns_resolver_patcher = mock.patch('dns.resolver')
|
||||||
|
dns_resolver_patcher.start()
|
||||||
|
self.addCleanup(dns_resolver_patcher.stop)
|
||||||
|
|
||||||
self.context = context.RequestContext()
|
self.context = context.RequestContext()
|
||||||
# create mocked config and driver
|
# create mocked config and driver
|
||||||
conf = cfg.ConfigOpts()
|
conf = cfg.ConfigOpts()
|
||||||
|
|
|
@ -40,6 +40,14 @@ class DefaultSSLCertificateControllerTests(base.TestCase):
|
||||||
|
|
||||||
super(DefaultSSLCertificateControllerTests, self).setUp()
|
super(DefaultSSLCertificateControllerTests, self).setUp()
|
||||||
|
|
||||||
|
tld_patcher = mock.patch('tld.get_tld')
|
||||||
|
tld_patcher.start()
|
||||||
|
self.addCleanup(tld_patcher.stop)
|
||||||
|
|
||||||
|
dns_resolver_patcher = mock.patch('dns.resolver')
|
||||||
|
dns_resolver_patcher.start()
|
||||||
|
self.addCleanup(dns_resolver_patcher.stop)
|
||||||
|
|
||||||
conf = cfg.ConfigOpts()
|
conf = cfg.ConfigOpts()
|
||||||
|
|
||||||
self.provider_mocks = {
|
self.provider_mocks = {
|
||||||
|
|
Loading…
Reference in New Issue