From f9a02a3f584dd3ec6ad70493ad671818a42822ef Mon Sep 17 00:00:00 2001 From: Tytus Kurek Date: Wed, 27 Mar 2019 12:38:51 +0000 Subject: [PATCH] Dispersion report This patchset adds a template for the 'dispersion.conf' file which is used for swift cluster health monitoring. Change-Id: I348ded9f94f2bcb7a680b2c2280ff163cde65c46 Closes-Bug: 1328064 --- actions.yaml | 4 ++ actions/actions.py | 36 +++++++++++++++++- actions/dispersion-populate | 1 + actions/dispersion-report | 1 + lib/swift_utils.py | 7 ++++ templates/icehouse/dispersion.conf | 6 +++ templates/queens/dispersion.conf | 9 +++++ tests/basic_deployment.py | 18 +++++++++ unit_tests/test_actions.py | 59 ++++++++++++++++++++++++++++++ unit_tests/test_swift_utils.py | 6 ++- 10 files changed, 144 insertions(+), 3 deletions(-) create mode 120000 actions/dispersion-populate create mode 120000 actions/dispersion-report create mode 100644 templates/icehouse/dispersion.conf create mode 100644 templates/queens/dispersion.conf diff --git a/actions.yaml b/actions.yaml index c1b8571..a5e8053 100644 --- a/actions.yaml +++ b/actions.yaml @@ -83,3 +83,7 @@ set-weight: - ring - search-value - weight +dispersion-populate: + description: Run swift-dispersion-populate command on the specified unit. +dispersion-report: + description: Run swift-dispersion-report command on the specified unit. diff --git a/actions/actions.py b/actions/actions.py index 27d7906..b03c421 100755 --- a/actions/actions.py +++ b/actions/actions.py @@ -174,10 +174,42 @@ def set_weight(args): balance_rings() +def dispersion_populate(args): + """Runs swift-dispersion-populate command and returns the output + @raises CalledProcessError + """ + cmd = 'swift-dispersion-populate' + try: + output = check_output(cmd) + action_set({'output': output}) + except CalledProcessError as e: + action_set({'output': e.output}) + action_fail("Failed to run {}".format(cmd)) + + +def dispersion_report(args): + """Runs swift-dispersion-report command and returns the output + @raises CalledProcessError + """ + cmd = 'swift-dispersion-report' + try: + output = check_output(cmd) + action_set({'output': output}) + except CalledProcessError as e: + action_set({'output': e.output}) + action_fail("Failed to run {}".format(cmd)) + + # A dictionary of all the defined actions to callables (which take # parsed arguments). -ACTIONS = {"pause": pause, "resume": resume, 'diskusage': diskusage, - 'remove-devices': remove_devices, 'set-weight': set_weight} +ACTIONS = { + "pause": pause, + "resume": resume, + 'diskusage': diskusage, + 'remove-devices': remove_devices, + 'set-weight': set_weight, + "dispersion-populate": dispersion_populate, + "dispersion-report": dispersion_report} def main(argv): diff --git a/actions/dispersion-populate b/actions/dispersion-populate new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/actions/dispersion-populate @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/actions/dispersion-report b/actions/dispersion-report new file mode 120000 index 0000000..405a394 --- /dev/null +++ b/actions/dispersion-report @@ -0,0 +1 @@ +actions.py \ No newline at end of file diff --git a/lib/swift_utils.py b/lib/swift_utils.py index 512ece5..49e1ea8 100644 --- a/lib/swift_utils.py +++ b/lib/swift_utils.py @@ -82,6 +82,7 @@ from charmhelpers.core.decorators import ( SWIFT_CONF_DIR = '/etc/swift' SWIFT_RING_EXT = 'ring.gz' SWIFT_CONF = os.path.join(SWIFT_CONF_DIR, 'swift.conf') +SWIFT_DISPERSION_CONF = os.path.join(SWIFT_CONF_DIR, 'dispersion.conf') SWIFT_PROXY_CONF = os.path.join(SWIFT_CONF_DIR, 'proxy-server.conf') MEMCACHED_CONF = '/etc/memcached.conf' SWIFT_RINGS_CONF = '/etc/apache2/conf.d/swift-rings' @@ -124,6 +125,7 @@ BASE_PACKAGES = [ 'memcached', 'apache2', 'python-keystone', + 'python-swiftclient', ] # > Folsom specific packages FOLSOM_PACKAGES = ['swift-plugin-s3', 'swauth'] @@ -143,6 +145,10 @@ CONFIG_FILES = OrderedDict([ 'hook_contexts': [SwiftHashContext()], 'services': ['swift-proxy'], }), + (SWIFT_DISPERSION_CONF, { + 'hook_contexts': [SwiftIdentityContext()], + 'services': [], + }), (SWIFT_PROXY_CONF, { 'hook_contexts': [SwiftIdentityContext(), SwiftS3Context(), @@ -397,6 +403,7 @@ def register_configs(): openstack_release=release) confs = [SWIFT_CONF, + SWIFT_DISPERSION_CONF, SWIFT_PROXY_CONF, HAPROXY_CONF, MEMCACHED_CONF] diff --git a/templates/icehouse/dispersion.conf b/templates/icehouse/dispersion.conf new file mode 100644 index 0000000..9eb8445 --- /dev/null +++ b/templates/icehouse/dispersion.conf @@ -0,0 +1,6 @@ +[dispersion] +auth_url = {{ auth_protocol }}://{{ keystone_host }}:{{ service_port }}/v2.0 +auth_user = {{ service_tenant }}:{{ service_user }} +auth_key = {{ service_password }} +endpoint_type = internalURL +auth_version = 2 diff --git a/templates/queens/dispersion.conf b/templates/queens/dispersion.conf new file mode 100644 index 0000000..9b4871d --- /dev/null +++ b/templates/queens/dispersion.conf @@ -0,0 +1,9 @@ +[dispersion] +auth_url = {{ auth_protocol }}://{{ keystone_host }}:{{ auth_port }} +auth_user = {{ service_user }} +auth_key = {{ service_password }} +endpoint_type = internalURL +auth_version = 3 +project_domain_name = {{ admin_domain_name }} +user_domain_name = {{ admin_domain_name }} +project_name = {{ admin_tenant_name }} diff --git a/tests/basic_deployment.py b/tests/basic_deployment.py index a4a0343..9ea08c2 100644 --- a/tests/basic_deployment.py +++ b/tests/basic_deployment.py @@ -672,3 +672,21 @@ class SwiftProxyBasicDeployment(OpenStackAmuletDeployment): assert u.wait_on_action(action_id), "diskusage action failed." u.log.info('OK') + + def test_904_dispersion_populate_action(self): + """dispersion-populate action can be run""" + u.log.info("Testing dispersion-populate action") + action_id = u.run_action(self.swift_proxy_sentry, + "dispersion-populate") + assert u.wait_on_action(action_id), \ + "dispersion-populate action failed." + + u.log.info('OK') + + def test_905_dispersion_report_action(self): + """dispersion-report action can be run""" + u.log.info("Testing dispersion-report action") + action_id = u.run_action(self.swift_proxy_sentry, "dispersion-report") + assert u.wait_on_action(action_id), "dispersion-report action failed." + + u.log.info('OK') diff --git a/unit_tests/test_actions.py b/unit_tests/test_actions.py index 97c0db7..a126999 100644 --- a/unit_tests/test_actions.py +++ b/unit_tests/test_actions.py @@ -359,3 +359,62 @@ class SetWeightTestCase(CharmTestCase): self.action_fail.assert_called() self.set_weight_in_ring.assert_not_called() self.balance_rings.assert_not_called() + + +class DispersionPopulateTestCase(CharmTestCase): + + TEST_OUTPUT = ( + b'Using storage policy: Policy-0 \n' + b'[KCreated 2 containers for dispersion reporting, 1s, 0 retries\n' + b'[KCreated 2 objects for dispersion reporting, 1s, 0 retries\n') + + def setUp(self): + super(DispersionPopulateTestCase, self).setUp( + actions.actions, ['check_output', 'action_set', 'action_fail']) + + def test_success(self): + self.check_output.return_value = self.TEST_OUTPUT + actions.actions.dispersion_populate([]) + self.check_output.assert_called_once_with('swift-dispersion-populate') + self.action_set.assert_called_once_with({'output': self.TEST_OUTPUT}) + + def test_failure(self): + self.check_output.side_effect = actions.actions.CalledProcessError( + 1, 'swift-dispersion-populate', output='Failure') + actions.actions.dispersion_populate([]) + self.check_output.assert_called_once_with('swift-dispersion-populate') + self.action_set.assert_called_once_with({'output': 'Failure'}) + self.action_fail.assert_called_once_with( + "Failed to run swift-dispersion-populate") + + +class DispersionReportTestCase(CharmTestCase): + + TEST_OUTPUT = ( + b'Using storage policy: Policy-0 \n' + b'[KQueried 2 containers for dispersion reporting, 0s, 0 retries\n' + b'100.00% of container copies found (2 of2)\n' + b'Sample represents 0.78% of the container partition space\n' + b'[KQueried 2 objects for dispersion reporting, 0s, 0 retries\n' + b'! There were 2 partitions missing 0 copies.\n' + b'100.00% of object copies found (2 of 2)\n' + b'Sample represents 0.78% of the object partition space') + + def setUp(self): + super(DispersionReportTestCase, self).setUp( + actions.actions, ['check_output', 'action_set', 'action_fail']) + + def test_success(self): + self.check_output.return_value = self.TEST_OUTPUT + actions.actions.dispersion_report([]) + self.check_output.assert_called_once_with('swift-dispersion-report') + self.action_set.assert_called_once_with({'output': self.TEST_OUTPUT}) + + def test_failure(self): + self.check_output.side_effect = actions.actions.CalledProcessError( + 1, 'swift-dispersion-report', output='Failure') + actions.actions.dispersion_report([]) + self.check_output.assert_called_once_with('swift-dispersion-report') + self.action_set.assert_called_once_with({'output': 'Failure'}) + self.action_fail.assert_called_once_with( + "Failed to run swift-dispersion-report") diff --git a/unit_tests/test_swift_utils.py b/unit_tests/test_swift_utils.py index 3d7880e..7bf08fa 100644 --- a/unit_tests/test_swift_utils.py +++ b/unit_tests/test_swift_utils.py @@ -576,7 +576,8 @@ class SwiftUtilsTestCase(unittest.TestCase): 'swift-proxy', 'memcached', 'apache2', - 'python-keystone'], + 'python-keystone', + 'python-swiftclient'], swift_utils.determine_packages('essex') ) @@ -586,6 +587,7 @@ class SwiftUtilsTestCase(unittest.TestCase): 'memcached', 'apache2', 'python-keystone', + 'python-swiftclient', 'swift-plugin-s3', 'swauth'], swift_utils.determine_packages('folsom') @@ -596,6 +598,7 @@ class SwiftUtilsTestCase(unittest.TestCase): 'swift-proxy', 'memcached', 'apache2', + 'python-swiftclient', 'swift-plugin-s3', 'swauth', 'python-ceilometermiddleware', @@ -608,6 +611,7 @@ class SwiftUtilsTestCase(unittest.TestCase): 'swift-proxy', 'memcached', 'apache2', + 'python-swiftclient', 'swauth', 'python-ceilometermiddleware', 'python-keystonemiddleware'],