Dispersion report

This patchset adds a template for the 'dispersion.conf' file which is
used for swift cluster health monitoring.

Change-Id: I348ded9f94f2bcb7a680b2c2280ff163cde65c46
Closes-Bug: 1328064

actions.yaml

@@ -83,3 +83,7 @@ set-weight:
     - ring
     - search-value
     - weight
+dispersion-populate:
+  description: Run swift-dispersion-populate command on the specified unit.
+dispersion-report:
+  description: Run swift-dispersion-report command on the specified unit.

actions/actions.py

@@ -174,10 +174,42 @@ def set_weight(args):
     balance_rings()
 
 
+def dispersion_populate(args):
+    """Runs swift-dispersion-populate command and returns the output
+    @raises CalledProcessError
+    """
+    cmd = 'swift-dispersion-populate'
+    try:
+        output = check_output(cmd)
+        action_set({'output': output})
+    except CalledProcessError as e:
+        action_set({'output': e.output})
+        action_fail("Failed to run {}".format(cmd))
+
+
+def dispersion_report(args):
+    """Runs swift-dispersion-report command and returns the output
+    @raises CalledProcessError
+    """
+    cmd = 'swift-dispersion-report'
+    try:
+        output = check_output(cmd)
+        action_set({'output': output})
+    except CalledProcessError as e:
+        action_set({'output': e.output})
+        action_fail("Failed to run {}".format(cmd))
+
+
 # A dictionary of all the defined actions to callables (which take
 # parsed arguments).
-ACTIONS = {"pause": pause, "resume": resume, 'diskusage': diskusage,
-           'remove-devices': remove_devices, 'set-weight': set_weight}
+ACTIONS = {
+    "pause": pause,
+    "resume": resume,
+    'diskusage': diskusage,
+    'remove-devices': remove_devices,
+    'set-weight': set_weight,
+    "dispersion-populate": dispersion_populate,
+    "dispersion-report": dispersion_report}
 
 
 def main(argv):

actions/dispersion-populate

@@ -0,0 +1 @@
+actions.py

actions/dispersion-report

@@ -0,0 +1 @@
+actions.py

lib/swift_utils.py

@@ -82,6 +82,7 @@ from charmhelpers.core.decorators import (
 SWIFT_CONF_DIR = '/etc/swift'
 SWIFT_RING_EXT = 'ring.gz'
 SWIFT_CONF = os.path.join(SWIFT_CONF_DIR, 'swift.conf')
+SWIFT_DISPERSION_CONF = os.path.join(SWIFT_CONF_DIR, 'dispersion.conf')
 SWIFT_PROXY_CONF = os.path.join(SWIFT_CONF_DIR, 'proxy-server.conf')
 MEMCACHED_CONF = '/etc/memcached.conf'
 SWIFT_RINGS_CONF = '/etc/apache2/conf.d/swift-rings'
@@ -124,6 +125,7 @@ BASE_PACKAGES = [
     'memcached',
     'apache2',
     'python-keystone',
+    'python-swiftclient',
 ]
 # > Folsom specific packages
 FOLSOM_PACKAGES = ['swift-plugin-s3', 'swauth']
@@ -143,6 +145,10 @@ CONFIG_FILES = OrderedDict([
         'hook_contexts': [SwiftHashContext()],
         'services': ['swift-proxy'],
     }),
+    (SWIFT_DISPERSION_CONF, {
+        'hook_contexts': [SwiftIdentityContext()],
+        'services': [],
+    }),
     (SWIFT_PROXY_CONF, {
         'hook_contexts': [SwiftIdentityContext(),
                           SwiftS3Context(),
@@ -397,6 +403,7 @@ def register_configs():
                                           openstack_release=release)
 
     confs = [SWIFT_CONF,
+             SWIFT_DISPERSION_CONF,
              SWIFT_PROXY_CONF,
              HAPROXY_CONF,
              MEMCACHED_CONF]

templates/icehouse/dispersion.conf

@@ -0,0 +1,6 @@
+[dispersion]
+auth_url = {{ auth_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0
+auth_user = {{ service_tenant }}:{{ service_user }}
+auth_key = {{ service_password }}
+endpoint_type = internalURL
+auth_version = 2

templates/queens/dispersion.conf

@@ -0,0 +1,9 @@
+[dispersion]
+auth_url = {{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }}
+auth_user = {{ service_user }}
+auth_key = {{ service_password }}
+endpoint_type = internalURL
+auth_version = 3
+project_domain_name = {{ admin_domain_name }}
+user_domain_name = {{ admin_domain_name }}
+project_name = {{ admin_tenant_name }}

tests/basic_deployment.py

@@ -672,3 +672,21 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment):
         assert u.wait_on_action(action_id), "diskusage action failed."
 
         u.log.info('OK')
+
+    def test_904_dispersion_populate_action(self):
+        """dispersion-populate action can be run"""
+        u.log.info("Testing dispersion-populate action")
+        action_id = u.run_action(self.swift_proxy_sentry,
+                                 "dispersion-populate")
+        assert u.wait_on_action(action_id), \
+            "dispersion-populate action failed."
+
+        u.log.info('OK')
+
+    def test_905_dispersion_report_action(self):
+        """dispersion-report action can be run"""
+        u.log.info("Testing dispersion-report action")
+        action_id = u.run_action(self.swift_proxy_sentry, "dispersion-report")
+        assert u.wait_on_action(action_id), "dispersion-report action failed."
+
+        u.log.info('OK')

unit_tests/test_actions.py

@@ -359,3 +359,62 @@ class SetWeightTestCase(CharmTestCase):
         self.action_fail.assert_called()
         self.set_weight_in_ring.assert_not_called()
         self.balance_rings.assert_not_called()
+
+
+class DispersionPopulateTestCase(CharmTestCase):
+
+    TEST_OUTPUT = (
+        b'Using storage policy: Policy-0 \n'
+        b'[KCreated 2 containers for dispersion reporting, 1s, 0 retries\n'
+        b'[KCreated 2 objects for dispersion reporting, 1s, 0 retries\n')
+
+    def setUp(self):
+        super(DispersionPopulateTestCase, self).setUp(
+            actions.actions, ['check_output', 'action_set', 'action_fail'])
+
+    def test_success(self):
+        self.check_output.return_value = self.TEST_OUTPUT
+        actions.actions.dispersion_populate([])
+        self.check_output.assert_called_once_with('swift-dispersion-populate')
+        self.action_set.assert_called_once_with({'output': self.TEST_OUTPUT})
+
+    def test_failure(self):
+        self.check_output.side_effect = actions.actions.CalledProcessError(
+            1, 'swift-dispersion-populate', output='Failure')
+        actions.actions.dispersion_populate([])
+        self.check_output.assert_called_once_with('swift-dispersion-populate')
+        self.action_set.assert_called_once_with({'output': 'Failure'})
+        self.action_fail.assert_called_once_with(
+            "Failed to run swift-dispersion-populate")
+
+
+class DispersionReportTestCase(CharmTestCase):
+
+    TEST_OUTPUT = (
+        b'Using storage policy: Policy-0 \n'
+        b'[KQueried 2 containers for dispersion reporting, 0s, 0 retries\n'
+        b'100.00% of container copies found (2 of2)\n'
+        b'Sample represents 0.78% of the container partition space\n'
+        b'[KQueried 2 objects for dispersion reporting, 0s, 0 retries\n'
+        b'! There were 2 partitions missing 0 copies.\n'
+        b'100.00% of object copies found (2 of 2)\n'
+        b'Sample represents 0.78% of the object partition space')
+
+    def setUp(self):
+        super(DispersionReportTestCase, self).setUp(
+            actions.actions, ['check_output', 'action_set', 'action_fail'])
+
+    def test_success(self):
+        self.check_output.return_value = self.TEST_OUTPUT
+        actions.actions.dispersion_report([])
+        self.check_output.assert_called_once_with('swift-dispersion-report')
+        self.action_set.assert_called_once_with({'output': self.TEST_OUTPUT})
+
+    def test_failure(self):
+        self.check_output.side_effect = actions.actions.CalledProcessError(
+            1, 'swift-dispersion-report', output='Failure')
+        actions.actions.dispersion_report([])
+        self.check_output.assert_called_once_with('swift-dispersion-report')
+        self.action_set.assert_called_once_with({'output': 'Failure'})
+        self.action_fail.assert_called_once_with(
+            "Failed to run swift-dispersion-report")

unit_tests/test_swift_utils.py

@@ -576,7 +576,8 @@ class SwiftUtilsTestCase(unittest.TestCase):
              'swift-proxy',
              'memcached',
              'apache2',
-             'python-keystone'],
+             'python-keystone',
+             'python-swiftclient'],
             swift_utils.determine_packages('essex')
         )
 
@@ -586,6 +587,7 @@ class SwiftUtilsTestCase(unittest.TestCase):
              'memcached',
              'apache2',
              'python-keystone',
+             'python-swiftclient',
              'swift-plugin-s3',
              'swauth'],
             swift_utils.determine_packages('folsom')
@@ -596,6 +598,7 @@ class SwiftUtilsTestCase(unittest.TestCase):
              'swift-proxy',
              'memcached',
              'apache2',
+             'python-swiftclient',
              'swift-plugin-s3',
              'swauth',
              'python-ceilometermiddleware',
@@ -608,6 +611,7 @@ class SwiftUtilsTestCase(unittest.TestCase):
              'swift-proxy',
              'memcached',
              'apache2',
+             'python-swiftclient',
              'swauth',
              'python-ceilometermiddleware',
              'python-keystonemiddleware'],