Failed to delete ocf resources via delete_resources option

Now the charm is using 'crm -w -F resource stop <RES>'
and 'crm -w -F configure delete <RES>' to delete legacy ocf
resource via delete_resources option, but the 'stop' step will
fail when ocf file no longer exists (e.g. due to packages no
longer being available after upgrade). In addition, the daemon
process may still be running after the upgrade in this case.

Change-Id: I7fdac7894366a88a154da5a0510acc535ee776d8
Closes-Bug: #1631858

hooks/hooks.py

@@ -70,6 +70,8 @@ from utils import (
     setup_maas_api,
     setup_ocf_files,
     set_unit_status,
+    ocf_file_exists,
+    kill_legacy_ocf_daemon_process,
 )
 
 from charmhelpers.contrib.charmsupport import nrpe
@@ -266,10 +268,17 @@ def ha_relation_changed():
         log('Deleting Resources' % (delete_resources), level=DEBUG)
         for res_name in delete_resources:
             if pcmk.crm_opt_exists(res_name):
-                log('Stopping and deleting resource %s' % res_name,
-                    level=DEBUG)
-                if pcmk.crm_res_running(res_name):
-                    pcmk.commit('crm -w -F resource stop %s' % res_name)
+                if ocf_file_exists(res_name, resources):
+                    log('Stopping and deleting resource %s' % res_name,
+                        level=DEBUG)
+                    if pcmk.crm_res_running(res_name):
+                        pcmk.commit('crm -w -F resource stop %s' % res_name)
+                else:
+                    log('Cleanuping and deleting resource %s' % res_name,
+                        level=DEBUG)
+                    pcmk.commit('crm resource cleanup %s' % res_name)
+                # Daemon process may still be running after the upgrade.
+                kill_legacy_ocf_daemon_process(res_name)
                 pcmk.commit('crm -w -F configure delete %s' % res_name)
 
         log('Configuring Resources: %s' % (resources), level=DEBUG)

hooks/utils.py

@@ -18,6 +18,7 @@ import ast
 import pcmk
 import maas
 import os
+import re
 import subprocess
 import socket
 import fcntl
@@ -720,3 +721,39 @@ def assess_status_helper():
             message = ("Insufficient peer units for ha cluster "
                        "(require {})".format(node_count))
     return status, message
+
+
+def ocf_file_exists(res_name, resources,
+                    RES_ROOT='/usr/lib/ocf/resource.d'):
+    """To determine whether the ocf file exists, allow multiple ocf
+       files with the same name in different directories
+
+    @param res_name: The name of the ocf resource to check
+    @param resources: ocf resources
+    @return: boolean - True if the ocf resource exists
+    """
+    res_type = None
+    for key, val in resources.iteritems():
+        if res_name == key:
+            if len(val.split(':')) > 2:
+                res_type = val.split(':')[1]
+                ocf_name = res_name.replace('res_', '').replace('_', '-')
+                ocf_file = os.path.join(RES_ROOT, res_type, ocf_name)
+                if os.path.isfile(ocf_file):
+                    return True
+    return False
+
+
+def kill_legacy_ocf_daemon_process(res_name):
+    """Kill legacy ocf daemon process
+
+    @param res_name: The name of the ocf process to kill
+    """
+    ocf_name = res_name.replace('res_', '').replace('_', '-')
+    reg_expr = '([0-9]+)\s+[^0-9]+{}'.format(ocf_name)
+    cmd = ['ps', '-eo', 'pid,cmd']
+    ps = subprocess.check_output(cmd)
+    res = re.search(reg_expr, ps, re.MULTILINE)
+    if res:
+        pid = res.group(1)
+        subprocess.call(['sudo', 'kill', '-9', pid])

unit_tests/test_hacluster_utils.py

@@ -16,6 +16,7 @@ import mock
 import os
 import re
 import shutil
+import subprocess
 import tempfile
 import unittest
 
@@ -196,3 +197,24 @@ class UtilsTestCase(unittest.TestCase):
         utils.setup_maas_api()
         add_source.assert_called_with(cfg['maas_source'])
         self.assertTrue(apt_install.called)
+
+    @mock.patch('os.path.isfile')
+    def test_ocf_file_exists(self, isfile_mock):
+        RES_NAME = 'res_ceilometer_agent_central'
+        resources = {RES_NAME: ('ocf:openstack:ceilometer-agent-central')}
+        utils.ocf_file_exists(RES_NAME, resources)
+        wish = '/usr/lib/ocf/resource.d/openstack/ceilometer-agent-central'
+        isfile_mock.assert_called_once_with(wish)
+
+    @mock.patch.object(subprocess, 'check_output')
+    @mock.patch.object(subprocess, 'call')
+    def test_kill_legacy_ocf_daemon_process(self, call_mock,
+                                            check_output_mock):
+        ps_output = '''
+          PID CMD
+          6863 sshd: ubuntu@pts/7
+          11109 /usr/bin/python /usr/bin/ceilometer-agent-central --config
+        '''
+        check_output_mock.return_value = ps_output
+        utils.kill_legacy_ocf_daemon_process('res_ceilometer_agent_central')
+        call_mock.assert_called_once_with(['sudo', 'kill', '-9', '11109'])