Add statsd metrics support to the swift-storage charm

By default, statsd metrics can be sent by the swift account,
container and object storage services for diagnostic and
monitoring purposes, but are disabled by default. This change exposes
charm config settings that allow it to be enabled by setting
'statsd_host' to a non-empty value. 'statsd_port' and
'statsd_sample_rate' are also supported for changing the destination
port and rate at which metrics are collected.

Closes-Bug: #1729770

Change-Id: If1bf3ced8a9ed07af81f352eb0263659d147e3aa

config.yaml

@@ -152,3 +152,22 @@ options:
     description: |
       YAML formatted associative array of sysctl values, e.g.:
       '{ kernel.pid_max : 4194303 }'
+  statsd-host:
+    default: ''
+    type: string
+    description: |
+      Enable statsd metrics to be sent to the specified host.
+      If this value is empty, statsd logging will be disabled.
+  statsd-port:
+    default: 3125
+    type: int
+    description: |
+      Destination port on the provided statsd host to send samples to.
+      Only takes effect if statsd-host is set.
+  statsd-sample-rate:
+    default: 1.0
+    type: float
+    description: |
+      Sample rate determines what percentage of the metric points a
+      client should send to the server.
+      Only takes effect if statsd-host is set.

lib/swift_storage_context.py

@@ -93,5 +93,8 @@ class SwiftStorageServerContext(OSContextGenerator):
             'object_replicator_concurrency': config(
                 'object-replicator-concurrency'),
             'object_rsync_timeout': config('object-rsync-timeout'),
+            'statsd_host': config('statsd-host'),
+            'statsd_port': config('statsd-port'),
+            'statsd_sample_rate': config('statsd-sample-rate'),
         }
         return ctxt

templates/account-server.conf

@@ -3,6 +3,12 @@ bind_ip = {{ bind_host }}
 bind_port = {{ account_server_port }}
 workers = {{ workers }}
 
+{% if statsd_host %}
+log_statsd_host = {{ statsd_host }}
+log_statsd_port = {{ statsd_port }}
+log_statsd_default_sample_rate = {{ statsd_sample_rate }}
+
+{% endif %}
 [pipeline:main]
 pipeline = recon account-server
 
@@ -18,4 +24,3 @@ use = egg:swift#account
 [account-auditor]
 
 [account-reaper]
-

templates/container-server.conf

@@ -3,6 +3,12 @@ bind_ip = {{ bind_host }}
 bind_port = {{ container_server_port }}
 workers = {{ workers }}
 
+{% if statsd_host %}
+log_statsd_host = {{ statsd_host }}
+log_statsd_port = {{ statsd_port }}
+log_statsd_default_sample_rate = {{ statsd_sample_rate }}
+
+{% endif %}
 [pipeline:main]
 pipeline = recon container-server
 
@@ -21,4 +27,3 @@ allow_versions = true
 [container-auditor]
 
 [container-sync]
-

templates/object-server.conf

@@ -3,6 +3,12 @@ bind_ip = {{ bind_host }}
 bind_port = {{ object_server_port }}
 workers = {{ workers }}
 
+{% if statsd_host %}
+log_statsd_host = {{ statsd_host }}
+log_statsd_port = {{ statsd_port }}
+log_statsd_default_sample_rate = {{ statsd_sample_rate }}
+
+{% endif %}
 [pipeline:main]
 pipeline = recon object-server
 
@@ -23,4 +29,3 @@ rsync_timeout = {{ object_rsync_timeout }}
 [object-auditor]
 
 [object-sync]
-

unit_tests/test_swift_storage_context.py

@@ -104,5 +104,8 @@ class SwiftStorageContextTests(CharmTestCase):
             'container_max_connections': '10',
             'object_max_connections': '10',
             'object_rsync_timeout': '950',
+            'statsd_host': '',
+            'statsd_port': 3125,
+            'statsd_sample_rate': 1.0
         }
         self.assertEqual(ex, result)

unit_tests/test_templates.py

@@ -0,0 +1,87 @@
+# Copyright 2016 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import unittest
+
+from jinja2 import Environment
+
+from charmhelpers.contrib.openstack.templating import get_loader
+
+
+class StorageServerTemplateTestCase(unittest.TestCase):
+
+    @mock.patch('charmhelpers.contrib.openstack.templating.log')
+    def get_template_for_release_and_server(
+            self,
+            os_release,
+            server,
+            mock_log):
+
+        if not server:
+            server = 'object'
+
+        loader = get_loader('./templates', os_release)
+        env = Environment(loader=loader)
+
+        return env.get_template('{}-server.conf'.format(server))
+
+    def test_os_release_not_in_templates(self):
+        """Regression test for bug 1251551.
+
+        The os_release is no longer provided as context to the templates.
+        """
+        for release in ('essex', 'grizzly', 'havana', 'icehouse'):
+            for server in ('object', 'container', 'account'):
+                template = self.get_template_for_release_and_server(
+                    release,
+                    server)
+                with open(template.filename, 'r') as template_orig:
+                    self.assertNotIn(
+                        'os_release', template_orig.read(),
+                        "The template '{}' contains os_release which is "
+                        "no longer provided in the context.".format(
+                            template.filename))
+
+    def test_config_renders_for_all_releases_and_servers(self):
+        """The configs render without syntax error."""
+        for release in ('essex', 'grizzly', 'havana', 'icehouse'):
+            for server in ('object', 'container', 'account'):
+                template = self.get_template_for_release_and_server(
+                    release,
+                    server)
+
+                result = template.render()
+
+                self.assertTrue(result.startswith("[DEFAULT]"))
+
+    def test_statsd_config_for_all_releases_and_servers(self):
+        """The configs contain statsd settings if statsd-host is set."""
+        for release in ('grizzly', 'havana', 'icehouse', 'mitaka'):
+            for server in ('object', 'container', 'account'):
+                template = self.get_template_for_release_and_server(
+                    release,
+                    server)
+
+                result = template.render(statsd_host='127.0.0.1')
+
+                self.assertIn("log_statsd_host", result)
+                self.assertIn("log_statsd_port", result)
+                self.assertIn("log_statsd_default_sample_rate", result)
+
+                result = template.render()
+
+                self.assertNotIn("log_statsd_host", result)
+                self.assertNotIn("log_statsd_port", result)
+                self.assertNotIn("log_statsd_default_sample_rate", result)