openstack
/
charm-ceph-mon
Code Issues Proposed changes

Guard cluster operations until sufficient OSD's booted 

Ensure that broker requests are not processed and that client
access keys are not issued until the expected number of OSD's
have been bootstrapped into the cluster.

This depends on presentation of the number of bootstrapped
OSD's from the ceph-osd charm (see Depends-On).

For upgraders, keys will have already been issued so there
should be no impact on existing access to the Ceph cluster;
The ceph-osd units will present the required relation data
post upgrade at which point the charm will mark the cluster
as ready for service and continue to process and pending
requests.

Change-Id: Id67e13c176fc8fd4953ba7c2cf7e33252810940c
Depends-On: I1370524f0f31120e3cb7305c5bc509a6494c5586
Closes-Bug: 1794878
This commit is contained in:
James Page 2018-10-16 09:44:48 +01:00
parent 61e20c2be3
commit ed99bd2b57
2 changed files with 55 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

69
hooks/ceph_hooks.py
View File

@ -506,6 +506,46 @@ def related_osds(num_units=3):
return False
def sufficient_osds(minimum_osds=3):
'''
Determine if the minimum number of OSD's have been
bootstrapped into the cluster.
@param expected_osds: The minimum number of OSD's required
@return: boolean indicating whether the required number of
OSD's where detected.
'''
bootstrapped_osds = 0
for r_id in relation_ids('osd'):
for unit in related_units(r_id):
unit_osds = relation_get(
attribute='bootstrapped-osds',
unit=unit, rid=r_id
)
if unit_osds is not None:
bootstrapped_osds += int(unit_osds)
if bootstrapped_osds >= minimum_osds:
return True
return False
def ready_for_service():
'''
Determine whether the Ceph cluster is ready to service
storage traffic from clients
@return: boolean indicating whether the Ceph cluster is
ready for pool creation/client usage.
'''
if not ceph.is_quorum():
log('mon cluster is not in quorum', level=DEBUG)
return False
if not sufficient_osds(config('expected-osd-count') or 3):
log('insufficient osds bootstrapped', level=DEBUG)
return False
return True
@hooks.hook('radosgw-relation-changed')
@hooks.hook('radosgw-relation-joined')
def radosgw_relation(relid=None, unit=None):
@ -516,8 +556,8 @@ def radosgw_relation(relid=None, unit=None):
# NOTE: radosgw needs some usage OSD storage, so defer key
# provision until OSD units are detected.
if ceph.is_quorum() and related_osds():
log('mon cluster in quorum and osds related '
if ready_for_service():
log('mon cluster in quorum and osds bootstrapped '
'- providing radosgw with keys')
public_addr = get_public_addr()
data = {
@ -539,15 +579,13 @@ def radosgw_relation(relid=None, unit=None):
log("Not leader - ignoring broker request", level=DEBUG)
relation_set(relation_id=relid, relation_settings=data)
else:
log('mon cluster not in quorum or no osds - deferring key provision')
@hooks.hook('mds-relation-changed')
@hooks.hook('mds-relation-joined')
def mds_relation_joined(relid=None, unit=None):
if ceph.is_quorum() and related_osds():
log('mon cluster in quorum and OSDs related'
if ready_for_service():
log('mon cluster in quorum and osds bootstrapped '
'- providing mds client with keys')
mds_name = relation_get(attribute='mds-name',
rid=relid, unit=unit)
@ -571,8 +609,6 @@ def mds_relation_joined(relid=None, unit=None):
log("Not leader - ignoring mds broker request", level=DEBUG)
relation_set(relation_id=relid, relation_settings=data)
else:
log('Waiting on mon quorum or min osds before provisioning mds keys')
@hooks.hook('admin-relation-changed')
@ -582,7 +618,7 @@ def admin_relation_joined(relid=None):
name = relation_get('keyring-name')
if name is None:
name = 'admin'
log('mon cluster in quorum - providing client with keys')
log('mon cluster in quorum - providing admin client with keys')
mon_hosts = config('monitor-hosts') or ' '.join(get_mon_hosts())
data = {'key': ceph.get_named_key(name=name, caps=ceph.admin_caps),
'fsid': leader_get('fsid'),
@ -591,14 +627,13 @@ def admin_relation_joined(relid=None):
}
relation_set(relation_id=relid,
relation_settings=data)
else:
log('mon cluster not in quorum - deferring key provision')
@hooks.hook('client-relation-joined')
def client_relation_joined(relid=None):
if ceph.is_quorum():
log('mon cluster in quorum - providing client with keys')
if ready_for_service():
log('mon cluster in quorum and osds bootstrapped '
'- providing client with keys')
service_name = None
if relid is None:
units = [remote_unit()]
@ -617,14 +652,14 @@ def client_relation_joined(relid=None):
data['rbd-features'] = config('default-rbd-features')
relation_set(relation_id=relid,
relation_settings=data)
else:
log('mon cluster not in quorum - deferring key provision')
@hooks.hook('client-relation-changed')
def client_relation_changed(relid=None, unit=None):
"""Process broker requests from ceph client relations."""
if ceph.is_quorum():
if ready_for_service():
log('mon cluster in quorum and osds bootstrapped '
'- processing client broker requests')
if not unit:
unit = remote_unit()
settings = relation_get(rid=relid, unit=unit)
@ -643,8 +678,6 @@ def client_relation_changed(relid=None, unit=None):
}
relation_set(relation_id=relid,
relation_settings=data)
else:
log('mon cluster not in quorum', level=DEBUG)
@hooks.hook('upgrade-charm.real')

5
unit_tests/test_ceph_hooks.py
View File

@ -257,6 +257,7 @@ class RelatedUnitsTestCase(unittest.TestCase):
call('osd:23')
])
@patch.object(ceph_hooks, 'ready_for_service')
@patch.object(ceph_hooks.ceph, 'is_quorum')
@patch.object(ceph_hooks, 'remote_unit')
@patch.object(ceph_hooks, 'relation_get')
@ -268,8 +269,10 @@ class RelatedUnitsTestCase(unittest.TestCase):
is_leader,
relation_get,
remote_unit,
is_quorum):
is_quorum,
ready_for_service):
# Check for LP #1738154
ready_for_service.return_value = True
process_requests.return_value = 'AOK'
is_leader.return_value = True
relation_get.return_value = {'broker_req': 'req'}