openstack
/
charm-ceph-radosgw
Code Issues Proposed changes

Implement a new cloud-sync multi-site relation 

The new relation is based on the Ceph cloud sync module:
https://docs.ceph.com/en/latest/radosgw/cloud-sync-module/

The cloud sync module leverages the same multi-site replication
framework, but instead of writing to a secondary Ceph cluster, it
writes to an S3 (or S3-compatible) target.

The S3 target connection credentials are obtained through a new
relation with the s3-integrator charm.

This new relation behaves exactly as a secondary multi-site relation
(it implements the same interface), but it is used to configure the
cloud sync module for the primary RGW zone. The secondary Ceph RGW is
related to the Ceph Mons in the same cluster as the primary Ceph RGW.

Change-Id: Ia9b69d2f48e77f73b55a2036ae7845c0cbba8045
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1176
Signed-off-by: Ionut Balutoiu <ibalutoiu@cloudbasesolutions.com>
This commit is contained in:
Ionut Balutoiu 2023-10-17 09:23:58 +03:00
parent 940be7fdfc
commit e03ad8451e
15 changed files with 1154 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
config.yaml
View File

@ -429,6 +429,30 @@ options:
description: |
Name of RADOS Gateway Zone to create for multi-site replication. This
option must be specific to the local site e.g. us-west or us-east.
### Cloud sync options
cloud-sync-default-s3-target:
type: string
default: minio
description: |
The default S3 target to use for cloud sync. This is the name of the
s3-integrator Juju application that relates to the ceph-radosgw
application configured with cloud-sync.
cloud-sync-target-path:
type: string
default: rgwx-${zonegroup}-${zone}-${sid}/${bucket}
description: |
A string that defines how the target path is created. The target path
specifies a prefix to which the source object name is appended.
The target path configurable can include any of the following variables:
* sid: unique string that represents the sync instance ID
* zonegroup: the zonegroup name
* zonegroup_id: the zonegroup ID
* zone: the zone name
* zone_id: the zone id
* bucket: source bucket name
* owner: source bucket owner ID
###
namespace-tenants:
type: boolean
default: False

54
hooks/ceph_radosgw_context.py
View File

@ -40,6 +40,10 @@ from charmhelpers.core.hookenv import (
relation_ids,
unit_public_ip,
leader_get,
remote_service_name,
)
from charmhelpers.core.services.helpers import (
RelationContext,
)
from charmhelpers.contrib.network.ip import (
format_ipv6_addr,
@ -353,3 +357,53 @@ class MonContext(context.CephContext):
if 'fsid' not in ctxt:
return False
return context.OSContextGenerator.context_complete(self, ctxt)
class SecondaryContext(context.OSContextGenerator):
interfaces = ['secondary']
def __call__(self):
ctxt = {}
for rid in relation_ids(self.interfaces[0]):
self.related = True
for unit in related_units(rid):
rel_data = relation_get(rid=rid, unit=unit)
ctxt = {
'realm': rel_data.get('realm'),
'zonegroup': rel_data.get('zonegroup'),
'access_key': rel_data.get('access_key'),
'secret': rel_data.get('secret'),
'url': rel_data.get('url'),
}
if self.context_complete(ctxt):
return ctxt
return {}
class CloudSyncContext(SecondaryContext):
interfaces = ['cloud-sync']
class S3CredentialsRelationContext(RelationContext):
name = 's3-credentials'
interface = 's3'
def __init__(self, *args, **kwargs):
self.required_keys = [
'access-key', 'secret-key', 'region', 'endpoint',
]
RelationContext.__init__(self, *args, **kwargs)
def get_data(self):
if not relation_ids(self.name):
return
for rid in sorted(relation_ids(self.name)):
app = remote_service_name(rid)
if not app:
continue
if self.get(app, None):
continue
reldata = relation_get(rid=rid, app=app)
if reldata and self._is_ready(reldata):
self.setdefault(app, reldata)

1
hooks/cloud-sync-relation-changed Symbolic link
View File

@ -0,0 +1 @@
hooks.py

150
hooks/hooks.py
View File

@ -49,6 +49,7 @@ from charmhelpers.core.hookenv import (
leader_set,
leader_get,
remote_service_name,
application_name,
WORKLOAD_STATES,
)
from charmhelpers.core.strutils import bool_from_string
@ -119,6 +120,10 @@ from charmhelpers.contrib.openstack.cert_utils import (
get_certificate_request,
process_certificates,
)
from ceph_radosgw_context import (
CloudSyncContext,
S3CredentialsRelationContext,
)
hooks = Hooks()
CONFIGS = register_configs()
@ -980,6 +985,148 @@ def secondary_relation_changed(relation_id=None, unit=None):
log('No mutation detected.', 'INFO')
@hooks.hook('cloud-sync-relation-changed')
def cloud_sync_relation_changed(relation_id=None, unit=None):
if not is_leader():
log('Cannot setup multisite configuration, this unit is not the '
'leader')
return
if not ready_for_service(legacy=False):
log('Unit not ready, deferring multisite configuration')
return
primary_data = CloudSyncContext()()
if not primary_data:
log("Defer processing until primary RGW has provided required data")
return
public_url = '{}:{}'.format(
canonical_url(CONFIGS, PUBLIC),
listen_port(),
)
endpoints = [public_url]
realm = config('realm')
zonegroup = config('zonegroup')
zone = config('zone')
if (realm, zonegroup) != (primary_data['realm'],
primary_data['zonegroup']):
log("Mismatched configuration so stop multi-site configuration now")
return
s3_rel_ctxt = S3CredentialsRelationContext()
default_profile = config('cloud-sync-default-s3-target')
if default_profile not in s3_rel_ctxt:
log("Defer cloud-sync relation until default profile {} is set by an "
"s3-integrator".format(default_profile))
return
mutation = False
if realm not in multisite.list_realms():
log('Realm {} not found, pulling now'.format(realm))
multisite.pull_realm(url=primary_data['url'],
access_key=primary_data['access_key'],
secret=primary_data['secret'])
multisite.pull_period(url=primary_data['url'],
access_key=primary_data['access_key'],
secret=primary_data['secret'])
multisite.set_default_realm(realm)
mutation = True
cloud_sync_zone_kwargs = {
'zonegroup': zonegroup,
'endpoints': endpoints,
'default': False,
'master': False,
'readonly': True,
'access_key': primary_data['access_key'],
'secret': primary_data['secret'],
'tier_type': 'cloud',
}
if zone not in multisite.list_zones():
log('cloud-sync zone {} not found, creating now'.format(zone))
multisite.pull_period(url=primary_data['url'],
access_key=primary_data['access_key'],
secret=primary_data['secret'])
multisite.create_zone(zone, **cloud_sync_zone_kwargs)
mutation = True
zone_info = multisite.get_zone_info(zone, zonegroup=zonegroup)
zone_system_key = zone_info.get('system_key', {})
zonegroup_info = multisite.get_zonegroup_info(zonegroup)
extra_zone_info = None
for z in zonegroup_info['zones']:
if z['id'] == zone_info['id']:
extra_zone_info = z
break
if not extra_zone_info:
raise RuntimeError("Could not find zone {} in zonegroup {}".format(
zone, zonegroup))
current_tier_config = zone_info.get('tier_config', {})
tier_config = multisite.get_cloud_sync_tier_config(
s3_rel_context=s3_rel_ctxt,
default_profile=default_profile,
target_path=config('cloud-sync-target-path'))
mutation = (
mutation or
extra_zone_info.get('tier_type') != 'cloud' or
extra_zone_info.get('endpoints') != endpoints or
zonegroup_info.get('master_zone') == zone_info['id'] or
zone_system_key.get('access_key') != primary_data['access_key'] or
zone_system_key.get('secret_key') != primary_data['secret'] or
not extra_zone_info.get('read_only') or
not multisite.equal_tier_config(actual=current_tier_config,
expected=tier_config)
)
if mutation:
flatten_tier_config = multisite.flatten_zone_tier_config(tier_config)
multisite.modify_zone(
zone,
zonegroup=zonegroup,
tier_config_rm='.')
multisite.modify_zone(
zone,
tier_config=','.join(flatten_tier_config),
**cloud_sync_zone_kwargs)
log(
'Mutation detected. Restarting {}.'.format(service_name()),
'INFO')
multisite.update_period(zonegroup=zonegroup, zone=zone)
CONFIGS.write_all()
service_restart(service_name())
leader_set(restart_nonce=str(uuid.uuid4()))
else:
log('No mutation detected.', 'INFO')
@hooks.hook('s3-credentials-relation-joined')
def s3_credentials_relation_joined(relation_id=None):
if is_leader():
# Unless we set the bucket on the app relation data, the s3-integrator
# won't set the full s3 connection info on the relation. This is a
# NOOP value, since the s3-integrator will set the bucket based on its
# config. The bucket value is echoed back on the relation only if the
# s3-integrator doesn't have the 'bucket' config set.
relation_set(relation_id=relation_id,
app=True,
bucket=application_name())
@hooks.hook('s3-credentials-relation-changed')
@hooks.hook('s3-credentials-relation-broken')
def s3_credentials_relation_changed(relation_id=None, unit=None):
# we handle the s3 credentials in the cloud-sync relation
for r_id in relation_ids('cloud-sync'):
for unit in related_units(r_id):
cloud_sync_relation_changed(r_id, unit)
@hooks.hook('master-relation-departed')
@hooks.hook('slave-relation-departed')
def master_slave_relation_departed():
@ -1034,6 +1181,9 @@ def process_multisite_relations():
for r_id in relation_ids('secondary'):
for unit in related_units(r_id):
secondary_relation_changed(r_id, unit)
for r_id in relation_ids('cloud-sync'):
for unit in related_units(r_id):
cloud_sync_relation_changed(r_id, unit)
def cert_rel_ca():

219
hooks/multisite.py
View File

@ -274,7 +274,7 @@ def modify_zonegroup(name, endpoints=None, default=False,
def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
access_key=None, secret=None, readonly=False):
access_key=None, secret=None, readonly=False, tier_type=None):
"""
Create a new RADOS Gateway zone
@ -294,6 +294,8 @@ def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
:type secret: str
:param readonly: set zone as read only
:type: readonly: boolean
:param tier_type: tier type to use for the zone
:type tier_type: str
:return: dict of zone configuration
:rtype: dict
"""
@ -313,6 +315,8 @@ def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
cmd.append('--access-key={}'.format(access_key))
cmd.append('--secret={}'.format(secret))
cmd.append('--read-only={}'.format(1 if readonly else 0))
if tier_type:
cmd.append('--tier-type={}'.format(tier_type))
try:
return json.loads(_check_output(cmd))
except TypeError:
@ -321,7 +325,8 @@ def create_zone(name, endpoints, default=False, master=False, zonegroup=None,
def modify_zone(name, endpoints=None, default=False, master=False,
access_key=None, secret=None, readonly=False,
realm=None, zonegroup=None):
realm=None, zonegroup=None, tier_type=None, tier_config=None,
tier_config_rm=None):
"""Modify an existing RADOS Gateway zone
:param name: name of zone to create
@ -342,6 +347,12 @@ def modify_zone(name, endpoints=None, default=False, master=False,
:type realm: str
:param zonegroup: zonegroup to use for zone
:type zonegroup: str
:param tier_type: tier type to use for the zone
:type tier_type: str
:param tier_config: tier config to use for the zone
:type tier_config: str
:param tier_config_rm: tier config to remove from the zone
:type tier_config_rm: str
:return: zone configuration
:rtype: dict
"""
@ -363,6 +374,12 @@ def modify_zone(name, endpoints=None, default=False, master=False,
cmd.append('--master')
if default:
cmd.append('--default')
if tier_type:
cmd.append('--tier-type={}'.format(tier_type))
if tier_config:
cmd.append('--tier-config={}'.format(tier_config))
if tier_config_rm:
cmd.append('--tier-config-rm={}'.format(tier_config_rm))
cmd.append('--read-only={}'.format(1 if readonly else 0))
try:
return json.loads(_check_output(cmd))
@ -370,6 +387,28 @@ def modify_zone(name, endpoints=None, default=False, master=False,
return None
def get_zone_info(name, zonegroup=None):
"""Fetch detailed info for the provided zone
:param name: zone name
:type name: str
:param zonegroup: parent zonegroup name
:type zonegroup: str
:rtype: dict
"""
cmd = [
RGW_ADMIN, '--id={}'.format(_key_name()),
'zone', 'get',
'--rgw-zone={}'.format(name),
]
if zonegroup:
cmd.append('--rgw-zonegroup={}'.format(zonegroup))
try:
return json.loads(_check_output(cmd))
except TypeError:
return None
def remove_zone_from_zonegroup(zone, zonegroup):
"""Remove RADOS Gateway zone from provided parent zonegroup
@ -888,3 +927,179 @@ def check_cluster_has_buckets():
if check_zonegroup_has_buckets(zonegroup):
return True
return False
def get_cloud_sync_tier_config(s3_rel_context, default_profile, target_path):
"""Create a tier config for the zone configured with cloud sync.
This is a helper function used to create a tier config dict with all the
values from 's3-credentials' relation (given as 's3_rel_context'), and the
charm config values (given as 'default_profile' and 'target_path').
:param s3_rel_context: relation data with all the remote applications in
the 's3-credentials' relation. This value is returned by the
'S3CredentialsRelationContext' relation context.
:type s3_rel_context: dict
:param default_profile: default S3 target to use for cloud sync.
:type default_profile: str
:param target_path: prefix added to the synced objects on the S3 target.
:type target_path: str
:return: tier config formed from the relation data and charm configs.
:rtype: dict
"""
tier_config = {
'connections': [],
'profiles': [],
'connection_id': default_profile,
'target_path': target_path,
}
for profile in s3_rel_context:
s3_info = s3_rel_context[profile]
conn = {
'id': profile,
'region': s3_info['region'],
'endpoint': s3_info['endpoint'],
'access_key': s3_info['access-key'],
'secret': s3_info['secret-key'],
}
if s3_info.get('s3-uri-style'):
conn['host_style'] = s3_info['s3-uri-style']
tier_config['connections'].append(conn)
if profile == default_profile:
# we don't need to add default profile to profiles list
continue
if not s3_info.get('bucket'):
hookenv.log(
"S3 credentials for profile {} does not contain bucket "
"information".format(profile), level=hookenv.WARNING)
continue
buckets = s3_info['bucket'].split(',')
for bucket in buckets:
tier_config['profiles'].append({
'connection_id': profile,
'source_bucket': bucket,
'target_path': target_path,
})
# remove empty lists from the tier config before returning
for k in ['profiles', 'connections']:
if len(tier_config[k]) == 0:
tier_config.pop(k)
return tier_config
def equal_tier_config(actual, expected):
"""Compares two tier configs and returns whether they are equal.
The expected tier config format is:
{
"connection_id": "<id>",
"target_path": "<target_path>",
"connections": [
{
"id": "<id>",
"access_key": <access>,
"secret": <secret>,
"region": "<region>",
"endpoint": "<endpoint>",
...
}
...
],
profiles: [
{
"connection_id": "<id>",
"source_bucket": "<bucket>",
"target_path": "<target_path>",
...
}
...
]
}
The tier config can get more complex than this, but the above format is
what the charm is currently using. We only care to check if the charm
configs updated the previously applied tier config. Order of the items in
the nested lists and dicts is not important.
:param actual: tier config for comparison.
:type actual: dict
:param expected: expected tier config (usually the value returned by
'get_cloud_sync_tier_config' function).
:type expected: dict
:rtype: Boolean
"""
if type(actual) is list and type(expected) is list:
if len(actual) != len(expected):
return False
for actual_item in actual:
found = False
for expected_item in expected:
if equal_tier_config(actual_item, expected_item):
found = True
break
if not found:
return False
return True
elif type(actual) is dict and type(expected) is dict:
if len(actual) != len(expected):
return False
for k in actual:
if not equal_tier_config(actual.get(k), expected.get(k)):
return False
return True
return actual == expected
def flatten_zone_tier_config(config, root_key_name=''):
"""Returns a flatten list of zone tier config.
This function is used to convert a dict tier config to a flatten list of
key=value pairs that can be passed as '--tier-config' parameter to
'radosgw-admin zone modify' command.
For example, the following dict:
{
"connection": {
"access_key": "<access>",
"secret": "<secret>",
"endpoint": "<endpoint>"
},
"acls": [
{
"type": "<id>",
"source_id": "<source>",
"dest_id": "<dest>"
}
],
"target_path": "<target_path>"
}
is flattened to:
[
'connection.access_key=<access>',
'connection.secret=<secret>',
'connection.endpoint=<endpoint>',
'acls[0].type=<id>',
'acls[0].source_id=<source>',
'acls[0].dest_id=<dest>',
'target_path=<target_path>'
]
:param config: Dict with the zone tier config.
:type config: dict
:param root_key_name: Root key name for the config (optional).
:type root_key_name: str
:return: List of key=value pairs for the tier config.
:rtype: list
"""
flatten_config = []
def flatten(elem, key_name=''):
if type(elem) is dict:
for i in elem:
flatten(elem[i], '{}.{}'.format(key_name, i))
elif type(elem) is list:
for index, item in enumerate(elem):
flatten(item, '{}[{}]'.format(key_name, index))
else:
flatten_config.append('{}={}'.format(key_name[1:], elem))
flatten(config, root_key_name)
return flatten_config

1
hooks/s3-credentials-relation-broken Symbolic link
View File

@ -0,0 +1 @@
hooks.py

1
hooks/s3-credentials-relation-changed Symbolic link
View File

@ -0,0 +1 @@
hooks.py

1
hooks/s3-credentials-relation-joined Symbolic link
View File

@ -0,0 +1 @@
hooks.py

12
hooks/utils.py
View File

@ -224,7 +224,10 @@ def check_optional_config_and_relations(configs):
# An operator may have deployed both relations
primary_rids = relation_ids('master') + relation_ids('primary')
secondary_rids = relation_ids('slave') + relation_ids('secondary')
cloud_sync_rids = relation_ids('cloud-sync')
secondary_rids = (relation_ids('slave') +
relation_ids('secondary') +
cloud_sync_rids)
multisite_rids = primary_rids + secondary_rids
# Any realm or zonegroup config is present, multisite checks can be done.
@ -286,6 +289,13 @@ def check_optional_config_and_relations(configs):
if not multisite_ready:
return ('waiting',
'multi-site master relation incomplete')
if cloud_sync_rids:
default_profile = config('cloud-sync-default-s3-target')
s3_ctxt = ceph_radosgw_context.S3CredentialsRelationContext()
if default_profile not in s3_ctxt:
return ('blocked',
"cloud-sync default s3 target creds not found in "
"any s3-credentials relation")
# Check that provided Ceph BlueStoe configuration is valid.
try:

4
metadata.yaml
View File

@ -36,6 +36,10 @@ requires:
interface: radosgw-multisite
secondary:
interface: radosgw-multisite
cloud-sync:
interface: radosgw-multisite
s3-credentials:
interface: s3
provides:
nrpe-external-master:
interface: nrpe-external-master

137
tests/bundles/jammy-bobcat-multisite-cloud-sync.yaml Normal file
View File

@ -0,0 +1,137 @@
options:
source: &source cloud:jammy-bobcat
series: jammy
comment:
- 'machines section to decide order of deployment. database sooner = faster'
machines:
'0':
'1':
'2':
'3':
'4':
'5':
'6':
'7':
'8':
'9':
'10':
'11':
'12':
applications:
ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '0'
secondary-ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
to:
- '1'
cloud-sync-ceph-radosgw:
charm: ../../ceph-radosgw.charm
num_units: 1
options:
source: *source
realm: zaza_realm
zonegroup: zaza_zg
zone: zaza_cloud_sync
cloud-sync-default-s3-target: s3-default
cloud-sync-target-path: ${bucket}
to:
- '2'
ceph-osd:
charm: ch:ceph-osd
num_units: 3
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '3'
- '4'
- '5'
channel: latest/edge
secondary-ceph-osd:
charm: ch:ceph-osd
num_units: 3
storage:
osd-devices: 'cinder,10G'
options:
source: *source
osd-devices: '/srv/ceph /dev/test-non-existent'
to:
- '6'
- '7'
- '8'
channel: latest/edge
ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '9'
channel: latest/edge
secondary-ceph-mon:
charm: ch:ceph-mon
num_units: 1
options:
monitor-count: 1
source: *source
to:
- '10'
channel: latest/edge
s3-default:
charm: ch:minio-test
num_units: 1
to:
- '11'
channel: latest/edge
s3-dev:
charm: ch:minio-test
num_units: 1
options:
bucket: dev*
to:
- '12'
channel: latest/edge
relations:
- - 'ceph-osd:mon'
- 'ceph-mon:osd'
- - 'ceph-radosgw:mon'
- 'ceph-mon:radosgw'
- - 'secondary-ceph-osd:mon'
- 'secondary-ceph-mon:osd'
- - 'secondary-ceph-radosgw:mon'
- 'secondary-ceph-mon:radosgw'
- - 'cloud-sync-ceph-radosgw:mon'
- 'ceph-mon:radosgw'
- - 'cloud-sync-ceph-radosgw:s3-credentials'
- 's3-default:s3-credentials'
- - 'cloud-sync-ceph-radosgw:s3-credentials'
- 's3-dev:s3-credentials'

4
tests/tests.yaml
View File

@ -14,6 +14,7 @@ dev_bundles:
- mantic-bobcat-multisite
- jammy-antelope-multisite
- jammy-bobcat-multisite
- jammy-bobcat-multisite-cloud-sync
- vault: lunar-antelope
- vault: mantic-bobcat
- vault: lunar-antelope-namespaced
@ -24,6 +25,9 @@ dev_bundles:
- vault: jammy-bobcat-namespaced
target_deploy_status:
cloud-sync-ceph-radosgw:
workload-status: blocked
workload-status-message-prefix: multi-site configuration but primary/secondary relation missing
vault:
workload-status: blocked
workload-status-message-prefix: Vault needs to be initialized

119
unit_tests/test_ceph_radosgw_context.py
View File

@ -26,6 +26,7 @@ TO_PATCH = [
'relation_get',
'relation_ids',
'related_units',
'remote_service_name',
'cmp_pkgrevno',
'arch',
'socket',
@ -586,3 +587,121 @@ class ApacheContextTest(CharmTestCase):
def setUp(self):
super(ApacheContextTest, self).setUp(context, TO_PATCH)
self.config.side_effect = self.test_config.get
class SecondaryContextTest(CharmTestCase):
def setUp(self):
super(SecondaryContextTest, self).setUp(context, TO_PATCH)
self.relation_get.side_effect = self.test_relation.get
self.relation_ids.return_value = ['secondary:6']
self.related_units.return_value = ['primary-ceph-radosgw/0']
def test_complete_ctxt(self):
test_ctxt = {
'realm': 'realmX',
'zonegroup': 'zonegroup1',
'access_key': 's3_access_key',
'secret': 's3_secret',
'url': 'http://10.9.3.3:80',
}
self.test_relation.set(test_ctxt)
ctxt = context.SecondaryContext()
self.assertEqual(test_ctxt, ctxt())
def test_incomplete_ctxt(self):
self.test_relation.set({
'realm': 'realmX',
'zonegroup': 'zonegroup1',
'url': 'http://10.9.3.3:80',
'access_key': None,
'secret': None,
})
ctxt = context.SecondaryContext()
self.assertEqual({}, ctxt())
class S3CredentialsRelationContextTest(CharmTestCase):
def setUp(self):
super(S3CredentialsRelationContextTest, self).setUp(context, TO_PATCH)
def test_get_data(self):
self.relation_ids.return_value = [
's3-credentials:6',
's3-credentials:7',
's3-credentials:8',
's3-credentials:9',
]
def _remote_service_name(name):
if name == 's3-credentials:6':
return 'minio-default'
elif name == 's3-credentials:7':
return 'minio-dev'
elif name == 's3-credentials:8':
return 'minio-prod'
elif name == 's3-credentials:9':
return 'minio-local'
def _relation_get(rid, app):
if app == 'minio-default':
return {
'access-key': 'default-access-key',
'secret-key': 'default-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.2:9000',
'bucket': 'default',
}
elif app == 'minio-dev':
return {
'access-key': 'dev-access-key',
'secret-key': 'dev-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.5:9000',
'bucket': 'staging,test*,dev',
}
elif app == 'minio-prod':
return {
'access-key': 'prod-access-key',
'secret-key': 'prod-secret-key',
'region': 'us-east-2',
'endpoint': 'http://10.13.1.10:9000',
'bucket': 'prod',
}
elif app == 'minio-local':
# This returns incomplete relation app data. It will not be
# included in the relation context.
return {
'region': 'local',
'endpoint': 'http://192.168.1.100:9000',
}
self.remote_service_name.side_effect = _remote_service_name
self.relation_get.side_effect = _relation_get
expected = {
'minio-default': {
'access-key': 'default-access-key',
'secret-key': 'default-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.2:9000',
'bucket': 'default',
},
'minio-dev': {
'access-key': 'dev-access-key',
'secret-key': 'dev-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.5:9000',
'bucket': 'staging,test*,dev',
},
'minio-prod': {
'access-key': 'prod-access-key',
'secret-key': 'prod-secret-key',
'region': 'us-east-2',
'endpoint': 'http://10.13.1.10:9000',
'bucket': 'prod',
}
}
s3_rel_ctxt = context.S3CredentialsRelationContext()
self.assertEqual(expected, s3_rel_ctxt)

186
unit_tests/test_hooks.py
View File

@ -681,13 +681,16 @@ class CephRadosGWTests(CharmTestCase):
class MiscMultisiteTests(CharmTestCase):
TO_PATCH = [
'application_name',
'restart_nonce_changed',
'relation_ids',
'related_units',
'relation_set',
'leader_get',
'is_leader',
'primary_relation_joined',
'secondary_relation_changed',
'cloud_sync_relation_changed',
'service_restart',
'service_name',
'multisite'
@ -696,11 +699,13 @@ class MiscMultisiteTests(CharmTestCase):
_relation_ids = {
'primary': ['primary:1'],
'secondary': ['secondary:1'],
'cloud-sync': ['cloud-sync:1'],
}
_related_units = {
'primary:1': ['rgw/0', 'rgw/1'],
'secondary:1': ['rgw-s/0', 'rgw-s/1'],
'cloud-sync:1': ['rgw-c-s/0', 'rgw-c-s/1'],
}
def setUp(self):
@ -724,10 +729,37 @@ class MiscMultisiteTests(CharmTestCase):
def test_process_multisite_relations(self):
ceph_hooks.process_multisite_relations()
self.primary_relation_joined.assert_called_once_with('primary:1')
self.assertEqual(self.secondary_relation_changed.call_count, 2)
self.secondary_relation_changed.assert_has_calls([
call('secondary:1', 'rgw-s/0'),
call('secondary:1', 'rgw-s/1'),
])
self.assertEqual(self.cloud_sync_relation_changed.call_count, 2)
self.cloud_sync_relation_changed.assert_has_calls([
call('cloud-sync:1', 'rgw-c-s/0'),
call('cloud-sync:1', 'rgw-c-s/1'),
])
def test_s3_credentials_relation_joined(self):
self.is_leader.return_value = True
self.application_name.return_value = 'ceph-radosgw'
ceph_hooks.s3_credentials_relation_joined('s3-credentials:1')
self.is_leader.assert_called_once_with()
self.application_name.assert_called_once_with()
self.relation_set.assert_called_once_with(
relation_id='s3-credentials:1',
app=True,
bucket='ceph-radosgw')
def test_s3_credentials_relation_changed(self):
ceph_hooks.s3_credentials_relation_changed('s3-credentials:1')
self.assertEqual(self.cloud_sync_relation_changed.call_count, 2)
self.cloud_sync_relation_changed.assert_has_calls([
call('cloud-sync:1', 'rgw-c-s/0'),
call('cloud-sync:1', 'rgw-c-s/1'),
])
class CephRadosMultisiteTests(CharmTestCase):
@ -1005,6 +1037,160 @@ class SecondaryMultisiteTests(CephRadosMultisiteTests):
ceph_hooks.secondary_relation_changed('secondary:1', 'rgw/0')
self.relation_get.assert_not_called()
@patch.object(ceph_hooks, "S3CredentialsRelationContext")
@patch.object(ceph_hooks, "CloudSyncContext")
def test_cloud_sync_relation_changed(self, cloud_sync_ctxt, s3_rel_ctxt):
for k, v in self._complete_config.items():
self.test_config.set(k, v)
self.test_config.set('cloud-sync-default-s3-target', 'default')
self.test_config.set('cloud-sync-target-path', 'test_target_path')
self.is_leader.return_value = True
ctxt_mock = MagicMock()
ctxt_mock.side_effect = lambda: self._test_relation
cloud_sync_ctxt.side_effect = lambda: ctxt_mock
self.canonical_url.return_value = 'http://rgw'
self.listen_port.return_value = 80
self.multisite.list_realms.return_value = []
self.multisite.list_zones.return_value = []
test_s3_rel_ctxt = {
'default': {
'access-key': 'default-access-key',
'secret-key': 'default-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.2:9000',
'bucket': 'default',
}
}
s3_rel_ctxt.return_value = test_s3_rel_ctxt
self.multisite.get_cloud_sync_tier_config.return_value = {
'connection_id': 'default',
'target_path': 'test_target_path',
}
self.multisite.get_zonegroup_info.return_value = get_zonegroup_stub()
self.multisite.get_zone_info.return_value = {
'id': 'test_zone_id_one',
'tier_config': {
'connection_id': 'default',
'target_path': 'test_target_path',
},
}
self.multisite.flatten_zone_tier_config.return_value = [
'connection_id=default',
'target_path=test_target_path',
]
ceph_hooks.cloud_sync_relation_changed('cloud-sync:1', 'rgw/0')
cloud_sync_ctxt.assert_called_once_with()
ctxt_mock.assert_called_once_with()
self.canonical_url.assert_called_once_with(ANY, 'public')
self.listen_port.assert_called_once_with()
self.assertEqual(self.config.call_count, 5)
self.config.assert_has_calls([
call('realm'),
call('zonegroup'),
call('zone'),
call('cloud-sync-default-s3-target'),
call('cloud-sync-target-path'),
])
self.multisite.pull_realm.assert_called_once_with(
url=self._test_relation['url'],
access_key=self._test_relation['access_key'],
secret=self._test_relation['secret'])
self.assertEqual(self.multisite.pull_period.call_count, 2)
self.multisite.pull_period.assert_has_calls([
call(url=self._test_relation['url'],
access_key=self._test_relation['access_key'],
secret=self._test_relation['secret']),
call(
url=self._test_relation['url'],
access_key=self._test_relation['access_key'],
secret=self._test_relation['secret'],
),
])
self.multisite.set_default_realm.assert_called_once_with(
self._complete_config['realm'])
self.multisite.create_zone.assert_called_once_with(
self._complete_config['zone'],
endpoints=['http://rgw:80'],
default=False, master=False, readonly=True,
zonegroup=self._complete_config['zonegroup'],
access_key=self._test_relation['access_key'],
secret=self._test_relation['secret'],
tier_type='cloud',
)
self.multisite.get_cloud_sync_tier_config.assert_called_once_with(
s3_rel_context=test_s3_rel_ctxt,
default_profile='default',
target_path='test_target_path')
self.multisite.get_zone_info.assert_called_once_with(
self._complete_config['zone'],
zonegroup=self._complete_config['zonegroup'])
self.multisite.flatten_zone_tier_config.assert_called_once_with(
{'connection_id': 'default',
'target_path': 'test_target_path'})
self.assertEqual(self.multisite.modify_zone.call_count, 2)
self.multisite.modify_zone.assert_has_calls([
call(self._complete_config['zone'],
zonegroup=self._complete_config['zonegroup'],
tier_config_rm='.'),
call(self._complete_config['zone'],
zonegroup=self._complete_config['zonegroup'],
endpoints=['http://rgw:80'],
default=False, master=False, readonly=True,
access_key=self._test_relation['access_key'],
secret=self._test_relation['secret'],
tier_type='cloud',
tier_config=('connection_id=default,'
'target_path=test_target_path')),
])
self.multisite.update_period.assert_called_once_with(
zonegroup=self._complete_config['zonegroup'],
zone=self._complete_config['zone'])
self.service_restart.assert_called_once_with('rgw@hostname')
self.leader_set.assert_called_once_with(restart_nonce=ANY)
@patch.object(ceph_hooks, "CloudSyncContext")
def test_cloud_sync_relation_changed_incomplete_relation(self,
cloud_sync_ctxt):
self.is_leader.return_value = True
ctxt_mock = MagicMock()
ctxt_mock.side_effect = lambda: {}
cloud_sync_ctxt.side_effect = lambda: ctxt_mock
ceph_hooks.cloud_sync_relation_changed('cloud-sync:1', 'rgw/0')
cloud_sync_ctxt.assert_called_once_with()
ctxt_mock.assert_called_once_with()
self.config.assert_not_called()
@patch.object(ceph_hooks, "CloudSyncContext")
def test_cloud_sync_relation_changed_mismatching_config(self,
cloud_sync_ctxt):
for k, v in self._complete_config.items():
self.test_config.set(k, v)
self.is_leader.return_value = True
ctxt_mock = MagicMock()
ctxt_mock.side_effect = lambda: self._test_bad_relation
cloud_sync_ctxt.side_effect = lambda: ctxt_mock
ceph_hooks.cloud_sync_relation_changed('cloud-sync:1', 'rgw/0')
cloud_sync_ctxt.assert_called_once_with()
ctxt_mock.assert_called_once_with()
self.assertEqual(self.config.call_count, 3)
self.config.assert_has_calls([
call('realm'),
call('zonegroup'),
call('zone'),
])
self.multisite.list_realms.assert_not_called()
def test_cloud_sync_relation_changed_not_leader(self):
self.is_leader.return_value = False
ceph_hooks.cloud_sync_relation_changed('cloud-sync:1', 'rgw/0')
self.ready_for_service.assert_not_called()
@patch.object(ceph_hooks, 'apt_install')
@patch.object(ceph_hooks, 'services')
@patch.object(ceph_hooks, 'nrpe')

244
unit_tests/test_multisite.py
View File

@ -41,6 +41,7 @@ def get_zonegroup_stub():
class TestMultisiteHelpers(CharmTestCase):
maxDiff = None
TO_PATCH = [
'subprocess',
@ -170,6 +171,7 @@ class TestMultisiteHelpers(CharmTestCase):
zonegroup='brundall',
access_key='mykey',
secret='mypassword',
tier_type='cloud',
)
self.assertEqual(result['name'], 'brundall-east')
self.subprocess.check_output.assert_called_with([
@ -182,6 +184,7 @@ class TestMultisiteHelpers(CharmTestCase):
'--access-key=mykey',
'--secret=mypassword',
'--read-only=0',
'--tier-type=cloud',
])
def test_modify_zone(self):
@ -190,6 +193,8 @@ class TestMultisiteHelpers(CharmTestCase):
endpoints=['http://localhost:80', 'https://localhost:443'],
access_key='mykey',
secret='secret',
tier_config='connection.access_key=my-secret-s3-access-key',
tier_config_rm='connection.host_style',
readonly=True
)
self.subprocess.check_output.assert_called_with([
@ -198,6 +203,8 @@ class TestMultisiteHelpers(CharmTestCase):
'--rgw-zone=brundall-east',
'--endpoints=http://localhost:80,https://localhost:443',
'--access-key=mykey', '--secret=secret',
'--tier-config=connection.access_key=my-secret-s3-access-key',
'--tier-config-rm=connection.host_style',
'--read-only=1',
])
@ -484,3 +491,240 @@ class TestMultisiteHelpers(CharmTestCase):
multisite.check_cluster_has_buckets(),
True
)
def test_get_zone_info(self):
multisite.get_zone_info('test_zone', 'test_zonegroup')
self.subprocess.check_output.assert_called_with([
'radosgw-admin', '--id=rgw.testhost',
'zone', 'get',
'--rgw-zone=test_zone', '--rgw-zonegroup=test_zonegroup',
])
def test_get_cloud_sync_tier_config(self):
s3_rel_context = {
'minio-default': {
'access-key': 'default-access-key',
'secret-key': 'default-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.2:9000',
's3-uri-style': 'path',
},
'minio-dev': {
'access-key': 'dev-access-key',
'secret-key': 'dev-secret-key',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.5:9000',
'bucket': 'staging,test*,dev',
},
'minio-prod': {
'access-key': 'prod-access-key',
'secret-key': 'prod-secret-key',
'region': 'us-east-2',
'endpoint': 'http://10.13.1.10:9000',
'bucket': 'prod',
's3-uri-style': 'virtual',
}
}
default_profile = 'minio-default'
target_path = 'rgwx-${zonegroup}-${zone}-${sid}/${bucket}'
tier_config = multisite.get_cloud_sync_tier_config(
s3_rel_context=s3_rel_context,
default_profile=default_profile,
target_path=target_path)
expected = {
'connection_id': default_profile,
'target_path': target_path,
'connections': [
{
'id': 'minio-default',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.2:9000',
'access_key': 'default-access-key',
'secret': 'default-secret-key',
'host_style': 'path',
},
{
'id': 'minio-dev',
'region': 'us-east-1',
'endpoint': 'http://10.13.1.5:9000',
'access_key': 'dev-access-key',
'secret': 'dev-secret-key',
},
{
'id': 'minio-prod',
'region': 'us-east-2',
'endpoint': 'http://10.13.1.10:9000',
'access_key': 'prod-access-key',
'secret': 'prod-secret-key',
'host_style': 'virtual',
},
],
'profiles': [
{
'connection_id': 'minio-dev',
'source_bucket': 'staging',
'target_path': target_path,
},
{
'connection_id': 'minio-dev',
'source_bucket': 'test*',
'target_path': target_path,
},
{
'connection_id': 'minio-dev',
'source_bucket': 'dev',
'target_path': target_path,
},
{
'connection_id': 'minio-prod',
'source_bucket': 'prod',
'target_path': target_path,
},
],
}
self.assertEqual(tier_config, expected)
def test_equal_tier_config(self):
actual = {
"connection_id": "<id>",
"target_path": "<target_path>",
"connections": [
{
"id": "<id2>",
"endpoint": "<endpoint2>",
},
{
"id": "<id1>",
"endpoint": "<endpoint1>",
},
],
"profiles": [
{
"connection_id": "<id>",
"source_bucket": "<bucket>",
"target_path": "<target_path>",
}
],
}
expected = {
"connection_id": "<id>",
"profiles": [
{
"connection_id": "<id>",
"source_bucket": "<bucket>",
"target_path": "<target_path>",
}
],
"connections": [
{
"id": "<id1>",
"endpoint": "<endpoint1>",
},
{
"id": "<id2>",
"endpoint": "<endpoint2>",
},
],
"target_path": "<target_path>",
}
is_equal = multisite.equal_tier_config(actual, expected)
self.assertTrue(is_equal)
def test_non_equal_tier_config(self):
actual = {
"connection_id": "<id>",
"target_path": "<target_path>",
"connections": [
{
"id": "<id>",
"endpoint": "<endpoint>",
},
],
}
expected = {
"connection_id": "<id>",
"profiles": [
{
"connection_id": "<id>",
"source_bucket": "<bucket>",
"target_path": "<target_path>",
}
],
"connections": [
{
"id": "<id>",
"endpoint": "<endpoint>",
},
],
"target_path": "<target_path>",
}
is_equal = multisite.equal_tier_config(actual, expected)
self.assertFalse(is_equal)
def test_flatten_zone_tier_config(self):
tier_config = {
'connection': {
'access_key': 's3_access_key',
'secret': 's3_secret',
'endpoint': 's3_endpoint',
},
'acls': [
{
'type': 'acl1',
'source_id': 'source1',
'dest_id': 'dest1',
},
{
'type': 'acl2',
'source_id': 'source2',
'dest_id': 'dest2',
},
],
'connections': [
{
'id': 'conn1',
'access_key': 'conn1_s3_access_key',
'secret': 'conn1_s3_secret',
'endpoint': 'conn1_s3_endpoint',
},
],
'profiles': [
{
'source_bucket': 'bucket1',
'connection_id': 'conn1',
'acls_id': 'acl1',
},
{
'source_bucket': 'bucket2',
'connection_id': 'conn1',
'acls_id': 'acl2',
},
],
'target_path': 'rgwx-${zonegroup}-${zone}-${sid}/${bucket}'
}
flatten_config = multisite.flatten_zone_tier_config(tier_config)
expected = [
'connection.access_key=s3_access_key',
'connection.secret=s3_secret',
'connection.endpoint=s3_endpoint',
'acls[0].type=acl1',
'acls[0].source_id=source1',
'acls[0].dest_id=dest1',
'acls[1].type=acl2',
'acls[1].source_id=source2',
'acls[1].dest_id=dest2',
'connections[0].id=conn1',
'connections[0].access_key=conn1_s3_access_key',
'connections[0].secret=conn1_s3_secret',
'connections[0].endpoint=conn1_s3_endpoint',
'profiles[0].source_bucket=bucket1',
'profiles[0].connection_id=conn1',
'profiles[0].acls_id=acl1',
'profiles[1].source_bucket=bucket2',
'profiles[1].connection_id=conn1',
'profiles[1].acls_id=acl2',
'target_path=rgwx-${zonegroup}-${zone}-${sid}/${bucket}',
]
self.assertEqual(flatten_config, expected)