Check shard is within records limit
Before adding a new record to a domain, check to see if it's at maximum capacity and skip to the next cert if necessary. Change-Id: I7e4c56e27cebc45a9cdddf9659762f29f9c6227c
This commit is contained in:
parent
5232069ad4
commit
0458de585d
|
@ -120,6 +120,7 @@ username = "<operator_username>"
|
|||
api_key = "<operator_api_key>"
|
||||
use_shards = True
|
||||
num_shards = 400
|
||||
;records_limit = 400
|
||||
shard_prefix = "cdn"
|
||||
shared_ssl_num_shards = 5
|
||||
shared_ssl_shard_prefix = "scdn"
|
||||
|
|
|
@ -69,3 +69,11 @@ class ServicesControllerBase(controller.DNSControllerBase):
|
|||
"""
|
||||
|
||||
raise NotImplementedError
|
||||
|
||||
def is_shard_full(self, shard_name):
|
||||
"""Check if shard can accept new records
|
||||
|
||||
:raises NotImplementedError
|
||||
"""
|
||||
|
||||
raise NotImplementedError
|
||||
|
|
|
@ -90,3 +90,6 @@ class ServicesController(base.ServicesBase):
|
|||
random.shuffle(shard_ids)
|
||||
for shard in shard_ids:
|
||||
yield 'scdn{0}.secure.defaultcdn.com'.format(shard)
|
||||
|
||||
def is_shard_full(self, shard_name):
|
||||
return False
|
||||
|
|
|
@ -32,6 +32,8 @@ RACKSPACE_OPTIONS = [
|
|||
cfg.BoolOpt('sharding_enabled', default=True,
|
||||
help='Enable Sharding?'),
|
||||
cfg.IntOpt('num_shards', default=10, help='Number of Shards to use'),
|
||||
cfg.IntOpt('records_limit', default=400,
|
||||
help='Number of records per domain.'),
|
||||
cfg.StrOpt('shard_prefix', default='cdn',
|
||||
help='The shard prefix to use'),
|
||||
cfg.IntOpt('shared_ssl_num_shards', default=5, help='Number of Shards '
|
||||
|
|
|
@ -799,3 +799,33 @@ class ServicesController(base.ServicesBase):
|
|||
def modify_cname(self, access_url, new_cert):
|
||||
self._change_cname_record(access_url=access_url,
|
||||
target_url=new_cert, shared_ssl_flag=False)
|
||||
|
||||
def is_shard_full(self, shard_name):
|
||||
count = 0
|
||||
try:
|
||||
shard_domain = self.client.find(name=shard_name)
|
||||
except exc.NotFound:
|
||||
LOG.error("Shards not configured properly, could not find {0}.")
|
||||
return True
|
||||
|
||||
records = shard_domain.list_records(limit=100)
|
||||
count += len(records)
|
||||
|
||||
# Loop until all records are printed
|
||||
while True:
|
||||
try:
|
||||
records = self.client.list_records_next_page()
|
||||
count += len(records)
|
||||
except exc.NoMoreResults:
|
||||
break
|
||||
|
||||
LOG.info(
|
||||
"There were a total of {0} record(s) for {1}.".format(
|
||||
count,
|
||||
shard_name
|
||||
))
|
||||
|
||||
if count >= self._driver.rackdns_conf.records_limit:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
|
|
@ -685,14 +685,19 @@ class DefaultServicesController(base.ServicesController):
|
|||
uuid_store[domain_name] = \
|
||||
self.dns_controller.generate_shared_ssl_domain_suffix()
|
||||
setattr(self, store, uuid_store)
|
||||
return '.'.join([domain_name,
|
||||
next(uuid_store[domain_name])])
|
||||
|
||||
shard = next(uuid_store[domain_name])
|
||||
while self.dns_controller.is_shard_full(shard):
|
||||
LOG.info(
|
||||
"Skipped shard {0} because it's at maximum capacity."
|
||||
.format(shard))
|
||||
shard = next(uuid_store[domain_name])
|
||||
|
||||
return '.'.join([domain_name, shard])
|
||||
except StopIteration:
|
||||
delattr(self, store)
|
||||
raise errors.SharedShardsExhausted('Domain {0} '
|
||||
'has already '
|
||||
'been '
|
||||
'taken'.format(domain_name))
|
||||
raise errors.SharedShardsExhausted(
|
||||
'Domain {0} has already been taken'.format(domain_name))
|
||||
|
||||
def _pick_shared_ssl_domain(self, domain, service_id, store):
|
||||
shared_ssl_domain = self._generate_shared_ssl_domain(
|
||||
|
@ -707,7 +712,7 @@ class DefaultServicesController(base.ServicesController):
|
|||
return shared_ssl_domain
|
||||
|
||||
def _shard_retry(self, project_id, service_obj, store=None):
|
||||
# deal with shared ssl domains
|
||||
# deal with shared ssl domains
|
||||
try:
|
||||
for domain in service_obj.domains:
|
||||
if domain.protocol == 'https' \
|
||||
|
|
|
@ -28,12 +28,12 @@ import uuid
|
|||
|
||||
import jsonschema
|
||||
import pecan
|
||||
import tld as tld_helper
|
||||
|
||||
from poppy.common import util
|
||||
from poppy.transport.validators import root_domain_regexes as regexes
|
||||
from poppy.transport.validators.stoplight import decorators
|
||||
from poppy.transport.validators.stoplight import exceptions
|
||||
from tld import get_tld
|
||||
|
||||
|
||||
def req_accepts_json_pecan(request, desired_content_type='application/json'):
|
||||
|
@ -143,8 +143,10 @@ def is_valid_tld(domain_name):
|
|||
status = whois.whois(domain_name)['status']
|
||||
if status is not None or status != '':
|
||||
url = 'https://{domain}'
|
||||
tld_obj = get_tld(url.format(domain=domain_name),
|
||||
as_object=True)
|
||||
tld_obj = tld_helper.get_tld(
|
||||
url.format(domain=domain_name),
|
||||
as_object=True
|
||||
)
|
||||
tld = tld_obj.suffix
|
||||
try:
|
||||
dns.resolver.query(tld + '.', 'SOA')
|
||||
|
|
|
@ -15,6 +15,8 @@
|
|||
|
||||
"""Unittests for TaskFlow distributed_task driver implementation."""
|
||||
|
||||
import mock
|
||||
|
||||
from poppy.distributed_task.utils import memoized_controllers
|
||||
from tests.unit import base
|
||||
|
||||
|
@ -24,6 +26,14 @@ class TestMemoizeUtils(base.TestCase):
|
|||
def setUp(self):
|
||||
super(TestMemoizeUtils, self).setUp()
|
||||
|
||||
rax_dns_set_credentials = mock.patch('pyrax.set_credentials')
|
||||
rax_dns_set_credentials.start()
|
||||
self.addCleanup(rax_dns_set_credentials.stop)
|
||||
|
||||
rax_dns = mock.patch('pyrax.cloud_dns')
|
||||
rax_dns.start()
|
||||
self.addCleanup(rax_dns.stop)
|
||||
|
||||
def test_memoization_service_controller(self):
|
||||
service_controller_first = \
|
||||
memoized_controllers.task_controllers('poppy')
|
||||
|
|
|
@ -34,6 +34,8 @@ RACKSPACE_OPTIONS = [
|
|||
cfg.BoolOpt('sharding_enabled', default=True,
|
||||
help='Enable Sharding?'),
|
||||
cfg.IntOpt('num_shards', default=500, help='Number of Shards to use'),
|
||||
cfg.IntOpt('records_limit', default=400,
|
||||
help='Number of records per domain.'),
|
||||
cfg.StrOpt('shard_prefix', default='cdn',
|
||||
help='The shard prefix to use'),
|
||||
cfg.StrOpt('url', default='mycdn.com',
|
||||
|
@ -1047,3 +1049,38 @@ class TestServicesUpdate(base.TestCase):
|
|||
responder_disable = self.controller.disable(self.service_old)
|
||||
# TODO(isaacm): Add assertions on the returned object
|
||||
self.assertIsNotNone(responder_disable)
|
||||
|
||||
def test_is_shard_full_shard_not_found(self):
|
||||
self.client.find.side_effect = exc.NotFound(404)
|
||||
|
||||
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||
|
||||
def test_is_shard_full_false(self):
|
||||
find_mock = mock.Mock()
|
||||
find_mock.list_records.return_value = range(100)
|
||||
self.client.find.return_value = find_mock
|
||||
|
||||
self.client.list_records_next_page.side_effect = exc.NoMoreResults
|
||||
|
||||
self.assertFalse(self.controller.is_shard_full('shard_name'))
|
||||
|
||||
def test_is_shard_full_true(self):
|
||||
find_mock = mock.Mock()
|
||||
find_mock.list_records.return_value = range(600)
|
||||
self.client.find.return_value = find_mock
|
||||
|
||||
self.client.list_records_next_page.side_effect = exc.NoMoreResults
|
||||
|
||||
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||
|
||||
def test_is_shard_full_paginate_true(self):
|
||||
find_mock = mock.Mock()
|
||||
find_mock.list_records.return_value = range(300)
|
||||
self.client.find.return_value = find_mock
|
||||
|
||||
self.client.list_records_next_page.side_effect = [
|
||||
range(300),
|
||||
exc.NoMoreResults,
|
||||
]
|
||||
|
||||
self.assertTrue(self.controller.is_shard_full('shard_name'))
|
||||
|
|
|
@ -116,6 +116,14 @@ class DefaultManagerServiceTests(base.TestCase):
|
|||
# in the reverse order of the arguments present
|
||||
super(DefaultManagerServiceTests, self).setUp()
|
||||
|
||||
tld_patcher = mock.patch('tld.get_tld')
|
||||
tld_patcher.start()
|
||||
self.addCleanup(tld_patcher.stop)
|
||||
|
||||
dns_resolver_patcher = mock.patch('dns.resolver')
|
||||
dns_resolver_patcher.start()
|
||||
self.addCleanup(dns_resolver_patcher.stop)
|
||||
|
||||
self.context = context.RequestContext()
|
||||
# create mocked config and driver
|
||||
conf = cfg.ConfigOpts()
|
||||
|
|
|
@ -40,6 +40,14 @@ class DefaultSSLCertificateControllerTests(base.TestCase):
|
|||
|
||||
super(DefaultSSLCertificateControllerTests, self).setUp()
|
||||
|
||||
tld_patcher = mock.patch('tld.get_tld')
|
||||
tld_patcher.start()
|
||||
self.addCleanup(tld_patcher.stop)
|
||||
|
||||
dns_resolver_patcher = mock.patch('dns.resolver')
|
||||
dns_resolver_patcher.start()
|
||||
self.addCleanup(dns_resolver_patcher.stop)
|
||||
|
||||
conf = cfg.ConfigOpts()
|
||||
|
||||
self.provider_mocks = {
|
||||
|
|
Loading…
Reference in New Issue