openstack
/
charm-nova-cloud-controller
Code Issues Proposed changes

[hopem,r=] 

Fix upgrade breakage whereby if upgrading from
version of charm that did not support
dbsync_state peer setting db ops get stuck
waiting infinitely for db to be intialised.

Closes-Bug: 1519035
This commit is contained in:
Edward Hope-Morley 2016-01-08 12:30:31 +00:00
parent 0d4f70e9ea
commit 8c875c4009
3 changed files with 90 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
hooks/nova_cc_hooks.py
View File

@ -19,6 +19,7 @@ from charmhelpers.core.hookenv import (
is_relation_made,
log,
local_unit,
DEBUG,
ERROR,
relation_get,
relation_ids,
@ -105,6 +106,7 @@ from nova_cc_utils import (
setup_ipv6,
REQUIRED_INTERFACES,
check_optional_relations,
is_db_initialised,
)
from charmhelpers.contrib.hahelpers.cluster import (
@ -148,6 +150,43 @@ AGENT_CA_PARAMS = 'op monitor interval="5s"'
NOVA_CONSOLEAUTH_OVERRIDE = '/etc/init/nova-consoleauth.override'
def leader_init_db_if_ready(skip_acl_check=False, skip_cells_restarts=False,
db_rid=None, unit=None):
"""Initialise db if leader and db not yet intialised.
NOTE: must be called from database context.
"""
if not is_elected_leader(CLUSTER_RES):
log("Not leader - skipping db init", level=DEBUG)
return
if is_db_initialised():
log("Database already initialised - skipping db init", level=DEBUG)
return
# Bugs 1353135 & 1187508. Dbs can appear to be ready before the units
# acl entry has been added. So, if the db supports passing a list of
# permitted units then check if we're in the list.
allowed_units = relation_get('nova_allowed_units', rid=db_rid, unit=unit)
if skip_acl_check or (allowed_units and local_unit() in
allowed_units.split()):
status_set('maintenance', 'Running nova db migration')
migrate_nova_database()
log('Triggering remote cloud-compute restarts.')
[compute_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cloud-compute')]
if not skip_cells_restarts:
log('Triggering remote cell restarts.')
[nova_cell_relation_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cell')]
conditional_neutron_migration()
else:
log('allowed_units either not presented, or local unit '
'not in acl list: %s' % repr(allowed_units))
@hooks.hook('install.real')
def install():
status_set('maintenance', 'Executing pre-install')
@ -340,26 +379,9 @@ def db_changed():
if 'shared-db' not in CONFIGS.complete_contexts():
log('shared-db relation incomplete. Peer not ready?')
return
CONFIGS.write_all()
if is_elected_leader(CLUSTER_RES):
# Bugs 1353135 & 1187508. Dbs can appear to be ready before the units
# acl entry has been added. So, if the db supports passing a list of
# permitted units then check if we're in the list.
allowed_units = relation_get('nova_allowed_units')
if allowed_units and local_unit() in allowed_units.split():
status_set('maintenance', 'Running nova db migration')
migrate_nova_database()
log('Triggering remote cloud-compute restarts.')
[compute_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cloud-compute')]
log('Triggering remote cell restarts.')
[nova_cell_relation_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cell')]
conditional_neutron_migration()
else:
log('allowed_units either not presented, or local unit '
'not in acl list: %s' % repr(allowed_units))
CONFIGS.write_all()
leader_init_db_if_ready()
@hooks.hook('pgsql-nova-db-relation-changed')
@ -370,15 +392,9 @@ def postgresql_nova_db_changed():
if 'pgsql-nova-db' not in CONFIGS.complete_contexts():
log('pgsql-nova-db relation incomplete. Peer not ready?')
return
CONFIGS.write_all()
if is_elected_leader(CLUSTER_RES):
status_set('maintenance', 'Running nova db migration')
migrate_nova_database()
log('Triggering remote cloud-compute restarts.')
[compute_joined(rid=rid, remote_restart=True)
for rid in relation_ids('cloud-compute')]
conditional_neutron_migration()
CONFIGS.write_all()
leader_init_db_if_ready(skip_acl_check=True, skip_cells_restarts=True)
@hooks.hook('pgsql-neutron-db-relation-changed')
@ -880,6 +896,18 @@ def upgrade_charm():
for r_id in relation_ids('cloud-compute'):
for unit in related_units(r_id):
compute_changed(r_id, unit)
rels = ['shared-db', 'pgsql-nova-db']
for rname in rels:
for rid in relation_ids(rname):
for unit in related_units(rid):
if rname == 'pgsql-nova-db':
leader_init_db_if_ready(skip_acl_check=True,
skip_cells_restarts=True,
db_rid=rid, unit=unit)
else:
leader_init_db_if_ready(db_rid=rid)
update_nrpe_config()
update_nova_consoleauth_config()

19
hooks/nova_cc_utils.py
View File

@ -16,7 +16,10 @@ from charmhelpers.contrib.hahelpers.cluster import (
get_hacluster_config,
)
from charmhelpers.contrib.peerstorage import peer_store
from charmhelpers.contrib.peerstorage import (
peer_retrieve,
peer_store,
)
from charmhelpers.contrib.python.packages import (
pip_install,
@ -53,9 +56,11 @@ from charmhelpers.core.hookenv import (
relation_ids,
remote_unit,
is_relation_made,
DEBUG,
INFO,
ERROR,
status_get,
status_set,
)
from charmhelpers.core.host import (
@ -591,6 +596,17 @@ def ml2_migration():
subprocess.check_call(cmd)
def is_db_initialised():
if relation_ids('cluster'):
dbsync_state = peer_retrieve('dbsync_state')
if dbsync_state == 'complete':
log("Database is initialised", level=DEBUG)
return True
log("Database is NOT initialised", level=DEBUG)
return False
def _do_openstack_upgrade(new_src):
enable_policy_rcd()
cur_os_rel = os_release('nova-common')
@ -634,6 +650,7 @@ def _do_openstack_upgrade(new_src):
ml2_migration()
if is_elected_leader(CLUSTER_RES):
status_set('maintenance', 'Running nova db migration')
migrate_nova_database()
[service_start(s) for s in services()]

22
unit_tests/test_nova_cc_hooks.py
View File

@ -441,17 +441,22 @@ class NovaCCHooksTests(CharmTestCase):
configs.write = MagicMock()
hooks.postgresql_nova_db_changed()
@patch.object(hooks, 'is_db_initialised')
@patch.object(hooks, 'conditional_neutron_migration')
@patch.object(hooks, 'CONFIGS')
def test_db_changed(self, configs, cond_neutron_mig):
def test_db_changed(self, configs, cond_neutron_mig,
mock_is_db_initialised):
mock_is_db_initialised.return_value = False
'No database migration is attempted when ACL list is not present'
self._shared_db_test(configs)
self.assertTrue(configs.write_all.called)
self.assertFalse(self.migrate_nova_database.called)
self.assertFalse(cond_neutron_mig.called)
@patch.object(hooks, 'is_db_initialised')
@patch.object(hooks, 'CONFIGS')
def test_db_changed_allowed(self, configs):
def test_db_changed_allowed(self, configs, mock_is_db_initialised):
mock_is_db_initialised.return_value = False
allowed_units = 'nova-cloud-controller/0 nova-cloud-controller/3'
self.test_relation.set({
'nova_allowed_units': allowed_units,
@ -461,8 +466,10 @@ class NovaCCHooksTests(CharmTestCase):
self.assertTrue(configs.write_all.called)
self.migrate_nova_database.assert_called_with()
@patch.object(hooks, 'is_db_initialised')
@patch.object(hooks, 'CONFIGS')
def test_db_changed_not_allowed(self, configs):
def test_db_changed_not_allowed(self, configs, mock_is_db_initialised):
mock_is_db_initialised.return_value = False
allowed_units = 'nova-cloud-controller/0 nova-cloud-controller/3'
self.test_relation.set({
'nova_allowed_units': allowed_units,
@ -472,17 +479,22 @@ class NovaCCHooksTests(CharmTestCase):
self.assertTrue(configs.write_all.called)
self.assertFalse(self.migrate_nova_database.called)
@patch.object(hooks, 'is_db_initialised')
@patch.object(hooks, 'CONFIGS')
def test_postgresql_db_changed(self, configs):
def test_postgresql_db_changed(self, configs, mock_is_db_initialised):
mock_is_db_initialised.return_value = False
self._postgresql_db_test(configs)
self.assertTrue(configs.write_all.called)
self.migrate_nova_database.assert_called_with()
@patch.object(hooks, 'is_db_initialised')
@patch.object(hooks, 'nova_cell_relation_joined')
@patch.object(hooks, 'compute_joined')
@patch.object(hooks, 'CONFIGS')
def test_db_changed_remote_restarts(self, configs, comp_joined,
cell_joined):
cell_joined, mock_is_db_initialised):
mock_is_db_initialised.return_value = False
def _relation_ids(rel):
relid = {
'cloud-compute': ['nova-compute/0'],