diff --git a/hooks/hooks.py b/hooks/hooks.py index 0dd65e1..be89450 100755 --- a/hooks/hooks.py +++ b/hooks/hooks.py @@ -70,6 +70,8 @@ from utils import ( setup_maas_api, setup_ocf_files, set_unit_status, + ocf_file_exists, + kill_legacy_ocf_daemon_process, ) from charmhelpers.contrib.charmsupport import nrpe @@ -266,10 +268,17 @@ def ha_relation_changed(): log('Deleting Resources' % (delete_resources), level=DEBUG) for res_name in delete_resources: if pcmk.crm_opt_exists(res_name): - log('Stopping and deleting resource %s' % res_name, - level=DEBUG) - if pcmk.crm_res_running(res_name): - pcmk.commit('crm -w -F resource stop %s' % res_name) + if ocf_file_exists(res_name, resources): + log('Stopping and deleting resource %s' % res_name, + level=DEBUG) + if pcmk.crm_res_running(res_name): + pcmk.commit('crm -w -F resource stop %s' % res_name) + else: + log('Cleanuping and deleting resource %s' % res_name, + level=DEBUG) + pcmk.commit('crm resource cleanup %s' % res_name) + # Daemon process may still be running after the upgrade. + kill_legacy_ocf_daemon_process(res_name) pcmk.commit('crm -w -F configure delete %s' % res_name) log('Configuring Resources: %s' % (resources), level=DEBUG) diff --git a/hooks/utils.py b/hooks/utils.py index e8d4eb1..9064207 100644 --- a/hooks/utils.py +++ b/hooks/utils.py @@ -18,6 +18,7 @@ import ast import pcmk import maas import os +import re import subprocess import socket import fcntl @@ -720,3 +721,39 @@ def assess_status_helper(): message = ("Insufficient peer units for ha cluster " "(require {})".format(node_count)) return status, message + + +def ocf_file_exists(res_name, resources, + RES_ROOT='/usr/lib/ocf/resource.d'): + """To determine whether the ocf file exists, allow multiple ocf + files with the same name in different directories + + @param res_name: The name of the ocf resource to check + @param resources: ocf resources + @return: boolean - True if the ocf resource exists + """ + res_type = None + for key, val in resources.iteritems(): + if res_name == key: + if len(val.split(':')) > 2: + res_type = val.split(':')[1] + ocf_name = res_name.replace('res_', '').replace('_', '-') + ocf_file = os.path.join(RES_ROOT, res_type, ocf_name) + if os.path.isfile(ocf_file): + return True + return False + + +def kill_legacy_ocf_daemon_process(res_name): + """Kill legacy ocf daemon process + + @param res_name: The name of the ocf process to kill + """ + ocf_name = res_name.replace('res_', '').replace('_', '-') + reg_expr = '([0-9]+)\s+[^0-9]+{}'.format(ocf_name) + cmd = ['ps', '-eo', 'pid,cmd'] + ps = subprocess.check_output(cmd) + res = re.search(reg_expr, ps, re.MULTILINE) + if res: + pid = res.group(1) + subprocess.call(['sudo', 'kill', '-9', pid]) diff --git a/unit_tests/test_hacluster_utils.py b/unit_tests/test_hacluster_utils.py index 817d083..19fa740 100644 --- a/unit_tests/test_hacluster_utils.py +++ b/unit_tests/test_hacluster_utils.py @@ -16,6 +16,7 @@ import mock import os import re import shutil +import subprocess import tempfile import unittest @@ -196,3 +197,24 @@ class UtilsTestCase(unittest.TestCase): utils.setup_maas_api() add_source.assert_called_with(cfg['maas_source']) self.assertTrue(apt_install.called) + + @mock.patch('os.path.isfile') + def test_ocf_file_exists(self, isfile_mock): + RES_NAME = 'res_ceilometer_agent_central' + resources = {RES_NAME: ('ocf:openstack:ceilometer-agent-central')} + utils.ocf_file_exists(RES_NAME, resources) + wish = '/usr/lib/ocf/resource.d/openstack/ceilometer-agent-central' + isfile_mock.assert_called_once_with(wish) + + @mock.patch.object(subprocess, 'check_output') + @mock.patch.object(subprocess, 'call') + def test_kill_legacy_ocf_daemon_process(self, call_mock, + check_output_mock): + ps_output = ''' + PID CMD + 6863 sshd: ubuntu@pts/7 + 11109 /usr/bin/python /usr/bin/ceilometer-agent-central --config + ''' + check_output_mock.return_value = ps_output + utils.kill_legacy_ocf_daemon_process('res_ceilometer_agent_central') + call_mock.assert_called_once_with(['sudo', 'kill', '-9', '11109'])