Merge "Add detailed healthcheck response body and fix messaging config"

contrib/tempest/tempest/api/keyvalue/stable/rest/test_healthcheck.py

@@ -0,0 +1,32 @@
+# Copyright 2015 Mirantis Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the 'License'); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an 'AS IS' BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from tempest.api.keyvalue.rest_base.base import MagnetoDBTestCase
+
+
+class MagnetoDBHealthcheckTest(MagnetoDBTestCase):
+
+    def test_healthcheck(self):
+        headers, body = self.client.healthcheck()
+        self.assertEqual('200', headers['status'])
+        self.assertEqual('{"API": "OK"}', body)
+
+    def test_healthcheck_fullcheck(self):
+        headers, body = self.client.healthcheck(fullcheck=True)
+        self.assertEqual('200', headers['status'])
+        results = body.split(',')
+        self.assertEqual(4, len(results))
+        for result in results:
+            self.assertIn('OK', result)

contrib/tempest/tempest/services/keyvalue/json/magnetodb_client.py

@@ -174,5 +174,13 @@ class MagnetoDBClientJSON(service_client.ServiceClient):
         resp, body = self.get(url)
         return resp, self._parse_resp(body)
 
+    def healthcheck(self, fullcheck=False):
+        self.get('tables')
+        base_url = '/'.join(self.base_url.split('/')[0:-3])
+        url = '/'.join([base_url, 'healthcheck'])
+        if fullcheck:
+            url = url + '?fullcheck=true'
+        return self.raw_request(url, 'GET')
+
     def _parse_resp(self, body):
         return json.loads(body)

doc/source/health_check.rst

@@ -8,8 +8,8 @@ HealthCheck
 **Request Parameters**
 
     **fullcheck**
-       | If fullcheck is 'true' keystone and back-end availability will be checked
-       | Syntax: healthcheck?fullcheck={true, false}
+       | If fullcheck is 'true' subsystems availability will be checked
+       | Syntax: healthcheck?fullcheck={true,false}
        | default: false
 
 **Request Syntax**
@@ -22,5 +22,8 @@ HealthCheck
         | 200 or 503
 
     **Response Body**
-        | Content-Type: text/plain
-        | "OK" or "Cassanra: ERROR" or "Keystone: ERROR" or "Keystone: ERROR, Cassandra: ERROR"
+        | Content-Type: application/json
+        | {"API": "string", "Identity": "string", "Messaging": "string", "Storage": "string"}
+
+    **Sample Response Body**
+        | {"API": "OK", "Identity": "ERROR", "Messaging": "OK", "Storage": "OK"}

etc/magnetodb-api.conf

@@ -57,62 +57,8 @@ jolokia_endpoint_list = http://127.0.0.1:8778/jolokia/, http://127.0.0.2:8778/jo
 # AMQP exchange to connect to if using RabbitMQ or QPID
 control_exchange = magnetodb
 
-# If passed, use a fake RabbitMQ provider
-# fake_rabbit = False
-
-# Configuration options if sending notifications via kombu rpc (these are
-# the defaults)
-# SSL version to use (valid only if SSL enabled)
-# kombu_ssl_version =
-# SSL key file (valid only if SSL enabled)
-# kombu_ssl_keyfile =
-# SSL cert file (valid only if SSL enabled)
-# kombu_ssl_certfile =
-# SSL certification authority file (valid only if SSL enabled)
-# kombu_ssl_ca_certs =
-# IP address of the RabbitMQ installation
-# rabbit_host = localhost
-# Password of the RabbitMQ server
-# rabbit_password = guest
-# Port where RabbitMQ server is running/listening
-# rabbit_port = 5672
-# RabbitMQ single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
-# rabbit_hosts is defaulted to '$rabbit_host:$rabbit_port'
-# rabbit_hosts = localhost:5672
-# User ID used for RabbitMQ connections
-rabbit_userid = stackrabbit
-# Location of a virtual RabbitMQ installation.
-# rabbit_virtual_host = /
-# Maximum retries with trying to connect to RabbitMQ
-# (the default of 0 implies an infinite retry count)
-# rabbit_max_retries = 0
-# RabbitMQ connection retry interval
-# rabbit_retry_interval = 1
-# Use HA queues in RabbitMQ (x-ha-policy: all). You need to
-# wipe RabbitMQ database when changing this option. (boolean value)
-# rabbit_ha_queues = false
-
 # QPID
 # rpc_backend=qpid
-# Qpid broker hostname
-# qpid_hostname = localhost
-# Qpid broker port
-# qpid_port = 5672
-# Qpid single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
-# qpid_hosts is defaulted to '$qpid_hostname:$qpid_port'
-# qpid_hosts = localhost:5672
-# Username for qpid connection
-# qpid_username = ''
-# Password for qpid connection
-# qpid_password = ''
-# Space separated list of SASL mechanisms to use for auth
-# qpid_sasl_mechanisms = ''
-# Seconds between connection keepalive heartbeats
-# qpid_heartbeat = 60
-# Transport to use, either 'tcp' or 'ssl'
-# qpid_protocol = tcp
-# Disable Nagle algorithm
-# qpid_tcp_nodelay = True
 
 # ZMQ
 # rpc_backend=zmq
@@ -235,5 +181,62 @@ storage_manager_config =
         }
     }
 
+[oslo_messaging_rabbit]
+# If passed, use a fake RabbitMQ provider
+# fake_rabbit = False
+
+# Configuration options if sending notifications via kombu rpc (these are
+# the defaults)
+# SSL version to use (valid only if SSL enabled)
+# kombu_ssl_version =
+# SSL key file (valid only if SSL enabled)
+# kombu_ssl_keyfile =
+# SSL cert file (valid only if SSL enabled)
+# kombu_ssl_certfile =
+# SSL certification authority file (valid only if SSL enabled)
+# kombu_ssl_ca_certs =
+# IP address of the RabbitMQ installation
+# rabbit_host = localhost
+# Password of the RabbitMQ server
+# rabbit_password = guest
+# Port where RabbitMQ server is running/listening
+# rabbit_port = 5672
+# RabbitMQ single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
+# rabbit_hosts is defaulted to '$rabbit_host:$rabbit_port'
+# rabbit_hosts = localhost:5672
+# User ID used for RabbitMQ connections
+rabbit_userid = stackrabbit
+# Location of a virtual RabbitMQ installation.
+# rabbit_virtual_host = /
+# Maximum retries with trying to connect to RabbitMQ
+# (the default of 0 implies an infinite retry count)
+# rabbit_max_retries = 0
+# RabbitMQ connection retry interval
+# rabbit_retry_interval = 1
+# Use HA queues in RabbitMQ (x-ha-policy: all). You need to
+# wipe RabbitMQ database when changing this option. (boolean value)
+# rabbit_ha_queues = false
+
+[oslo_messaging_qpid]
+# Qpid broker hostname
+# qpid_hostname = localhost
+# Qpid broker port
+# qpid_port = 5672
+# Qpid single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
+# qpid_hosts is defaulted to '$qpid_hostname:$qpid_port'
+# qpid_hosts = localhost:5672
+# Username for qpid connection
+# qpid_username = ''
+# Password for qpid connection
+# qpid_password = ''
+# Space separated list of SASL mechanisms to use for auth
+# qpid_sasl_mechanisms = ''
+# Seconds between connection keepalive heartbeats
+# qpid_heartbeat = 60
+# Transport to use, either 'tcp' or 'ssl'
+# qpid_protocol = tcp
+# Disable Nagle algorithm
+# qpid_tcp_nodelay = True
+
 [PROBE]
 enabled = True

etc/magnetodb-async-task-executor.conf

@@ -37,63 +37,6 @@ logging_exception_prefix = '%(instance)s'
 # AMQP exchange to connect to if using RabbitMQ or QPID
 control_exchange = magnetodb
 
-# If passed, use a fake RabbitMQ provider
-# fake_rabbit = False
-
-# Configuration options if sending notifications via kombu rpc (these are
-# the defaults)
-# SSL version to use (valid only if SSL enabled)
-# kombu_ssl_version =
-# SSL key file (valid only if SSL enabled)
-# kombu_ssl_keyfile =
-# SSL cert file (valid only if SSL enabled)
-# kombu_ssl_certfile =
-# SSL certification authority file (valid only if SSL enabled)
-# kombu_ssl_ca_certs =
-# IP address of the RabbitMQ installation
-# rabbit_host = localhost
-# Password of the RabbitMQ server
-# rabbit_password = guest
-# Port where RabbitMQ server is running/listening
-# rabbit_port = 5672
-# RabbitMQ single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
-# rabbit_hosts is defaulted to '$rabbit_host:$rabbit_port'
-# rabbit_hosts = localhost:5672
-# User ID used for RabbitMQ connections
-rabbit_userid = stackrabbit
-# Location of a virtual RabbitMQ installation.
-# rabbit_virtual_host = /
-# Maximum retries with trying to connect to RabbitMQ
-# (the default of 0 implies an infinite retry count)
-# rabbit_max_retries = 0
-# RabbitMQ connection retry interval
-# rabbit_retry_interval = 1
-# Use HA queues in RabbitMQ (x-ha-policy: all). You need to
-# wipe RabbitMQ database when changing this option. (boolean value)
-# rabbit_ha_queues = false
-
-# QPID
-# rpc_backend=qpid
-# Qpid broker hostname
-# qpid_hostname = localhost
-# Qpid broker port
-# qpid_port = 5672
-# Qpid single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
-# qpid_hosts is defaulted to '$qpid_hostname:$qpid_port'
-# qpid_hosts = localhost:5672
-# Username for qpid connection
-# qpid_username = ''
-# Password for qpid connection
-# qpid_password = ''
-# Space separated list of SASL mechanisms to use for auth
-# qpid_sasl_mechanisms = ''
-# Seconds between connection keepalive heartbeats
-# qpid_heartbeat = 60
-# Transport to use, either 'tcp' or 'ssl'
-# qpid_protocol = tcp
-# Disable Nagle algorithm
-# qpid_tcp_nodelay = True
-
 # ZMQ
 # rpc_backend=zmq
 # ZeroMQ bind address. Should be a wildcard (*), an ethernet interface, or IP.
@@ -182,5 +125,64 @@ storage_manager_config =
         }
     }
 
+[oslo_messaging_rabbit]
+# If passed, use a fake RabbitMQ provider
+# fake_rabbit = False
+
+# Configuration options if sending notifications via kombu rpc (these are
+# the defaults)
+# SSL version to use (valid only if SSL enabled)
+# kombu_ssl_version =
+# SSL key file (valid only if SSL enabled)
+# kombu_ssl_keyfile =
+# SSL cert file (valid only if SSL enabled)
+# kombu_ssl_certfile =
+# SSL certification authority file (valid only if SSL enabled)
+# kombu_ssl_ca_certs =
+# IP address of the RabbitMQ installation
+# rabbit_host = localhost
+# Password of the RabbitMQ server
+# rabbit_password = guest
+# Port where RabbitMQ server is running/listening
+# rabbit_port = 5672
+# RabbitMQ single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
+# rabbit_hosts is defaulted to '$rabbit_host:$rabbit_port'
+# rabbit_hosts = localhost:5672
+# User ID used for RabbitMQ connections
+rabbit_userid = stackrabbit
+# Location of a virtual RabbitMQ installation.
+# rabbit_virtual_host = /
+# Maximum retries with trying to connect to RabbitMQ
+# (the default of 0 implies an infinite retry count)
+# rabbit_max_retries = 0
+# RabbitMQ connection retry interval
+# rabbit_retry_interval = 1
+# Use HA queues in RabbitMQ (x-ha-policy: all). You need to
+# wipe RabbitMQ database when changing this option. (boolean value)
+# rabbit_ha_queues = false
+
+[oslo_messaging_qpid]
+# QPID
+# rpc_backend=qpid
+# Qpid broker hostname
+# qpid_hostname = localhost
+# Qpid broker port
+# qpid_port = 5672
+# Qpid single or HA cluster (host:port pairs i.e: host1:5672, host2:5672)
+# qpid_hosts is defaulted to '$qpid_hostname:$qpid_port'
+# qpid_hosts = localhost:5672
+# Username for qpid connection
+# qpid_username = ''
+# Password for qpid connection
+# qpid_password = ''
+# Space separated list of SASL mechanisms to use for auth
+# qpid_sasl_mechanisms = ''
+# Seconds between connection keepalive heartbeats
+# qpid_heartbeat = 60
+# Transport to use, either 'tcp' or 'ssl'
+# qpid_protocol = tcp
+# Disable Nagle algorithm
+# qpid_tcp_nodelay = True
+
 [PROBE]
 enabled = True

magnetodb/api/openstack/health_check.py

@@ -18,11 +18,11 @@ Health check request is a lightweight request that allows to check availability
 of magnetodb-api service and its subsystems (Keystone, DB back-end, MQ).
 """
 
-import urlparse
+import json
 import kombu
+import urlparse
 
 from keystoneclient.generic import client
-
 from magnetodb import api
 from magnetodb.common import config
 from magnetodb.common import exception
@@ -30,20 +30,178 @@ from magnetodb.openstack.common import log as logging
 from magnetodb import storage
 
 LOG = logging.getLogger(__name__)
-
-STATUS_OK = '200'
-STATUS_ERROR = '503'
-
 keystoneclient = client.Client()
 CONF = config.CONF
 
 
+def _check_keystone(auth_uri):
+    return keystoneclient.discover(auth_uri)
+
+
+def _test_messaging(url):
+    with kombu.Connection(url, connect_timeout=1) as conn:
+        simple_queue = conn.SimpleQueue('simple_queue')
+        message = 'Healthcheck sent'
+        simple_queue.put(message)
+        LOG.debug('Sent: %s' % message)
+
+        message = simple_queue.get(block=True, timeout=1)
+        LOG.debug("Received: %s" % message.payload)
+        message.ack()
+        simple_queue.close()
+
+
+def _check_rabbit():
+    url_pattern = 'amqp://{}:{}@{}:{}//'
+
+    rabbit_hosts = CONF.oslo_messaging_rabbit.rabbit_hosts
+    urls = []
+    if rabbit_hosts:
+        for host in rabbit_hosts:
+            hostname_port_pair = host.split(':')
+            url = (
+                url_pattern
+            ).format(
+                CONF.oslo_messaging_rabbit.rabbit_userid,
+                CONF.oslo_messaging_rabbit.rabbit_password,
+                hostname_port_pair[0],
+                hostname_port_pair[1]
+            )
+            urls.append(url)
+    else:
+        url = (
+            url_pattern
+        ).format(
+            CONF.oslo_messaging_rabbit.rabbit_userid,
+            CONF.oslo_messaging_rabbit.rabbit_password,
+            CONF.oslo_messaging_rabbit.rabbit_host,
+            CONF.oslo_messaging_rabbit.rabbit_port
+        )
+        urls.append(url)
+
+    healthy = True
+    for url in urls:
+        LOG.debug('Check messaging server at %s' % url)
+        try:
+            _test_messaging(url)
+        except Exception:
+            healthy = False
+            break
+
+    return healthy
+
+
+def _check_qpid():
+    url_pattern = 'amqp://{}:{}@{}:{}//'
+
+    qpid_hosts = CONF.oslo_messaging_qpid.qpid_hosts
+    urls = []
+    if qpid_hosts:
+        for host in qpid_hosts:
+            hostname_port_pair = host.split(':')
+            url = (
+                url_pattern
+            ).format(
+                CONF.oslo_messaging_qpid.qpid_username,
+                CONF.oslo_messaging_qpid.qpid_password,
+                hostname_port_pair[0],
+                hostname_port_pair[1]
+            )
+            urls.append(url)
+    else:
+        url = (
+            url_pattern
+        ).format(
+            CONF.oslo_messaging_qpid.qpid_username,
+            CONF.oslo_messaging_qpid.qpid_password,
+            CONF.oslo_messaging_qpid.qpid_hostname,
+            CONF.oslo_messaging_qpid.qpid_port
+        )
+        urls.append(url)
+
+    healthy = True
+    for url in urls:
+        LOG.debug('Check messaging server at %s' % url)
+        try:
+            _test_messaging(url)
+        except Exception:
+            healthy = False
+            break
+
+    return healthy
+
+
+class SubsystemCheck(object):
+
+    def check(self):
+        return NotImplemented
+
+
+class IdentityCheck(SubsystemCheck):
+
+    name = "Identity"
+
+    def __init__(self, auth_uri=''):
+        super(IdentityCheck, self).__init__()
+        self.auth_uri = auth_uri
+
+    def check(self):
+        return _check_keystone(self.auth_uri)
+
+
+class StorageCheck(SubsystemCheck):
+
+    name = "Storage"
+
+    def check(self):
+        try:
+            storage.health_check()
+        except exception.BackendInteractionError:
+            return False
+        return True
+
+
+class MessagingCheck(SubsystemCheck):
+
+    name = "Messaging"
+
+    RPC_CHECK_MAP = {
+        "rabbit": _check_rabbit,
+        "qpid": _check_qpid
+    }
+
+    def __init__(self):
+        super(MessagingCheck, self).__init__()
+        self._rpc_backend = CONF.rpc_backend
+
+    def check(self):
+        check = self.RPC_CHECK_MAP.get(self._rpc_backend, None)
+        if check:
+            return check()
+        else:
+            raise NotImplementedError()
+
+
 class HealthCheckApp(object):
     """Controller for health check request."""
 
+    STATUS_OK = '200'
+    STATUS_NOTFOUND = '404'
+    STATUS_ERROR = '503'
+    error_msg = 'ERROR'
+    success_msg = 'OK'
+    not_implemented_msg = 'Not Implemented'
+    not_found_msg = 'Not found'
+
+    TRUE_VALUES = set(('true', '1', 'yes', 'on', 't', 'y'))
+
     def __init__(self, auth_uri=''):
         super(HealthCheckApp, self).__init__()
-        self.auth_uri = auth_uri
+        self.subsystem_check_tuple = (
+            IdentityCheck(auth_uri),
+            StorageCheck(),
+            MessagingCheck(),
+        )
 
     def __call__(self, environ, start_response):
         path = environ['PATH_INFO']
@@ -51,72 +209,46 @@ class HealthCheckApp(object):
         LOG.debug('Request received: %s', path)
 
         if path and path != '/':
-            start_response('404 Not found', [('Content-Type', 'text/plain')])
+            start_response(
+                ' '.join((self.STATUS_NOTFOUND, self.not_found_msg)),
+                [('Content-Type', 'text/plain')])
             return 'Incorrect url. Please check it and try again\n'
 
-        resp = ''
+        resp = dict(API=self.success_msg)
+        error_status = self.STATUS_OK
+
         query_string = environ.get('QUERY_STRING', '')
         fullcheck = urlparse.parse_qs(query_string).get('fullcheck')
+        subsystems_status = {}
+        if fullcheck and fullcheck[0] in self.TRUE_VALUES:
+            subsystems_status = self._check_subsystems()
 
-        if isinstance(fullcheck, list) and 'true' in fullcheck:
-            resp = self._check_subsystems()
+        if subsystems_status:
+            resp.update(subsystems_status)
+        resp = self._format_resp(resp)
+        if self.error_msg in subsystems_status.values():
+            error_status = self.STATUS_ERROR
 
-        if not resp:
-            error_status = STATUS_OK
-            resp = 'OK'
-        else:
-            error_status = STATUS_ERROR
-
-        resp += '\n'
-        start_response(error_status, [('Content-Type', 'text/plain')])
+        start_response(error_status, [('Content-Type', 'application/json')])
         return resp
 
+    def _format_resp(self, body):
+        return json.dumps(body)
+
     def _check_subsystems(self):
-        cas_error_msg = 'Cassandra: ERROR'
-        key_error_msg = 'Keystone: ERROR'
-        que_error_msg = 'RabbitMQ: ERROR'
-        resp = ''
-        try:
-            storage.health_check()
-        except exception.BackendInteractionError as ex:
-            LOG.debug(ex)
-            resp = cas_error_msg
-
-        if not keystoneclient.discover(self.auth_uri):
-            if resp:
-                resp = '. '.join([resp, key_error_msg])
+        results = {}
+        for subsystem in self.subsystem_check_tuple:
+            try:
+                res = subsystem.check()
+            except NotImplementedError:
+                msg = self.not_implemented_msg
             else:
-                resp = key_error_msg
-
-        try:
-            self._check_queue()
-        except Exception:
-            if resp:
-                resp = '. '.join([resp, que_error_msg])
-            else:
-                resp = que_error_msg
-
-        return resp
-
-    def _check_queue(self):
-        url = (
-            'amqp://{}:{}@{}:{}//'
-        ).format(
-            CONF.rabbit_userid,
-            CONF.rabbit_password,
-            CONF.rabbit_host,
-            CONF.rabbit_port
-        )
-        with kombu.Connection(url) as conn:
-            simple_queue = conn.SimpleQueue('simple_queue')
-            message = 'Healthcheck sent'
-            simple_queue.put(message)
-            LOG.debug('Sent: %s' % message)
-
-            message = simple_queue.get(block=True, timeout=1)
-            LOG.debug("Received: %s" % message.payload)
-            message.ack()
-            simple_queue.close()
+                if res:
+                    msg = self.success_msg
+                else:
+                    msg = self.error_msg
+            results[subsystem.name] = msg
+        return results
 
 
 @api.with_global_env(default_program='magnetodb-api')