From e8831f8229c8d10672911aff1479caaadb1d2fb4 Mon Sep 17 00:00:00 2001 From: Shinya Kawabata Date: Fri, 11 Nov 2016 20:33:27 +0900 Subject: [PATCH] Add multibyte character support for alarm definition It was failed that putting multibyte character into alarm definition name and description by monascaclient. There were some mistakes about treatment of utf8 encoding. And mysql connection had no utf8 option, so mysql could not handle multibyte character. Change-Id: I8743f89fcc5d5efd4e50f440b76d78abc037e8e7 --- devstack/files/monasca-api/api-config.yml | 2 +- etc/api-config.conf | 2 +- java/src/main/resources/api-config.yml | 2 +- monasca_api/tests/test_alarms.py | 41 ++++++++++ monasca_api/v2/reference/alarm_definitions.py | 12 +-- .../tests/api/test_alarm_definitions.py | 79 +++++++++++++++---- .../tests/api/test_metrics.py | 62 ++++++++++++--- 7 files changed, 166 insertions(+), 34 deletions(-) diff --git a/devstack/files/monasca-api/api-config.yml b/devstack/files/monasca-api/api-config.yml index fc1526adf..6228525b3 100644 --- a/devstack/files/monasca-api/api-config.yml +++ b/devstack/files/monasca-api/api-config.yml @@ -42,7 +42,7 @@ kafka: mysql: driverClass: com.mysql.jdbc.Driver - url: "jdbc:mysql://%DATABASE_HOST%:%DATABASE_PORT%/mon?connectTimeout=5000&autoReconnect=true&useSSL=true&useLegacyDatetimeCode=false&serverTimezone=UTC" + url: "jdbc:mysql://%DATABASE_HOST%:%DATABASE_PORT%/mon?connectTimeout=5000&autoReconnect=true&useSSL=true&useLegacyDatetimeCode=false&serverTimezone=UTC&characterEncoding=utf8" user: "%DATABASE_USER%" password: "%DATABASE_PASSWORD%" maxWaitForConnection: 1s diff --git a/etc/api-config.conf b/etc/api-config.conf index 42de7f722..e7737d08d 100755 --- a/etc/api-config.conf +++ b/etc/api-config.conf @@ -117,7 +117,7 @@ keyspace: monasca # Below is configuration for database. [database] -connection = "mysql+pymysql://monapi:password@192.168.10.4/mon" +connection = "mysql+pymysql://monapi:password@192.168.10.4/mon?charset=utf8mb4" # backend = sqlalchemy # host = 192.168.10.4 # username = monapi diff --git a/java/src/main/resources/api-config.yml b/java/src/main/resources/api-config.yml index e706c8f21..e9b0c4f71 100644 --- a/java/src/main/resources/api-config.yml +++ b/java/src/main/resources/api-config.yml @@ -27,7 +27,7 @@ kafka: mysql: driverClass: com.mysql.jdbc.Driver - url: jdbc:mysql://192.168.10.6:3306/mon?connectTimeout=5000&autoReconnect=true&useLegacyDatetimeCode=false + url: jdbc:mysql://192.168.10.6:3306/mon?connectTimeout=5000&autoReconnect=true&useLegacyDatetimeCode=false&characterEncoding=utf8 user: monapi password: password maxWaitForConnection: 1s diff --git a/monasca_api/tests/test_alarms.py b/monasca_api/tests/test_alarms.py index 5eb122507..f183f3b48 100644 --- a/monasca_api/tests/test_alarms.py +++ b/monasca_api/tests/test_alarms.py @@ -32,6 +32,7 @@ from monasca_api.v2.reference import alarms import oslo_config.fixture import oslotest.base as oslotest +import six CONF = oslo_config.cfg.CONF @@ -204,6 +205,7 @@ class TestAlarmsStateHistory(AlarmTestBase): class TestAlarmDefinition(AlarmTestBase): + def setUp(self): super(TestAlarmDefinition, self).setUp() @@ -691,3 +693,42 @@ class TestAlarmDefinition(AlarmTestBase): self.assertEqual(self.srmock.status, falcon.HTTP_200) self.assertThat(response, RESTResponseEquals(expected_data)) + + def test_get_alarm_definitions_with_multibyte_character(self): + def_name = 'ａｌａｒｍ＿ｄｅｆｉｎｉｔｉｏｎ' + if six.PY2: + def_name = def_name.decode('utf8') + + expected_data = { + u'alarm_actions': [], u'ok_actions': [], + u'description': None, u'match_by': [u'hostname'], + u'actions_enabled': True, u'undetermined_actions': [], + u'deterministic': False, + u'expression': u'max(test.metric{hostname=host}) gte 1', + u'id': u'00000001-0001-0001-0001-000000000001', + u'severity': u'LOW', u'name': def_name + } + + self.alarm_def_repo_mock.return_value.get_alarm_definition.return_value = { + 'alarm_actions': None, + 'ok_actions': None, + 'description': None, + 'match_by': u'hostname', + 'name': def_name, + 'actions_enabled': 1, + 'undetermined_actions': None, + 'expression': u'max(test.metric{hostname=host}) gte 1', + 'id': u'00000001-0001-0001-0001-000000000001', + 'severity': u'LOW' + } + + response = self.simulate_request( + '/v2.0/alarm-definitions/%s' % (expected_data[u'id']), + headers={ + 'X-Roles': 'admin', + 'X-Tenant-Id': TENANT_ID, + } + ) + + self.assertEqual(self.srmock.status, falcon.HTTP_200) + self.assertThat(response, RESTResponseEquals(expected_data)) diff --git a/monasca_api/v2/reference/alarm_definitions.py b/monasca_api/v2/reference/alarm_definitions.py index 770ed1a8f..f2db05e35 100644 --- a/monasca_api/v2/reference/alarm_definitions.py +++ b/monasca_api/v2/reference/alarm_definitions.py @@ -266,7 +266,7 @@ class AlarmDefinitions(alarm_definitions_api_v2.AlarmDefinitionsV2API, description = (alarm_definition_row['description'] if alarm_definition_row['description'] is not None else None) - expression = alarm_definition_row['expression'].decode('utf8') + expression = alarm_definition_row['expression'] is_deterministic = is_definition_deterministic(expression) result = { @@ -277,11 +277,11 @@ class AlarmDefinitions(alarm_definitions_api_v2.AlarmDefinitionsV2API, u'description': description, u'expression': expression, u'deterministic': is_deterministic, - u'id': alarm_definition_row['id'].decode('utf8'), + u'id': alarm_definition_row['id'], u'match_by': match_by, - u'name': alarm_definition_row['name'].decode('utf8'), - u'severity': alarm_definition_row['severity'].decode( - 'utf8').upper()} + u'name': alarm_definition_row['name'], + u'severity': alarm_definition_row['severity'].upper() + } return result @@ -629,7 +629,7 @@ def get_query_alarm_definition_expression(alarm_definition, def get_query_alarm_definition_description(alarm_definition, return_none=False): if 'description' in alarm_definition: - return alarm_definition['description'].decode('utf8') + return alarm_definition['description'] else: if return_none: return None diff --git a/monasca_tempest_tests/tests/api/test_alarm_definitions.py b/monasca_tempest_tests/tests/api/test_alarm_definitions.py index 7ff475ba3..84f7db988 100644 --- a/monasca_tempest_tests/tests/api/test_alarm_definitions.py +++ b/monasca_tempest_tests/tests/api/test_alarm_definitions.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -28,15 +29,6 @@ NUM_ALARM_DEFINITIONS = 2 class TestAlarmDefinitions(base.BaseMonascaTest): - @classmethod - def resource_setup(cls): - super(TestAlarmDefinitions, cls).resource_setup() - - @classmethod - def resource_cleanup(cls): - super(TestAlarmDefinitions, cls).resource_cleanup() - - # Create @test.attr(type="gate") def test_create_alarm_definition(self): @@ -345,6 +337,30 @@ class TestAlarmDefinitions(base.BaseMonascaTest): links = response_body['links'] self._verify_list_alarm_definitions_links(links) + @test.attr(type="gate") + def test_list_alarm_definitions_with_multibyte_character(self): + name = data_utils.rand_name('ａｌａｒｍ＿ｄｅｆｉｎｉｔｉｏｎ').decode('utf8') + description = 'ｄｅｓｃｒｉｐｔｉｏｎ'.decode('utf8') + + response_body_list = self._create_alarm_definitions( + name=name, + description=description, + number_of_definitions=1 + ) + alarm_definition = response_body_list[0] + + query_param = '?name=' + urlparse.quote(name.encode('utf8')) + resp, response_body = self.monasca_client.list_alarm_definitions( + query_param) + + self._verify_list_alarm_definitions_response_body(resp, response_body) + + # Test list alarm definition response body + elements = response_body['elements'] + self._verify_alarm_definitions_list(elements, [alarm_definition]) + links = response_body['links'] + self._verify_list_alarm_definitions_links(links) + @test.attr(type="gate") def test_list_alarm_definitions_with_name(self): name = data_utils.rand_name('alarm_definition') @@ -427,7 +443,7 @@ class TestAlarmDefinitions(base.BaseMonascaTest): name=name, description="description", expression=expression, - severity="low") + severity="LOW") resp, res_body_create_alarm_def = self.monasca_client.\ create_alarm_definitions(alarm_definition) self.assertEqual(201, resp.status) @@ -701,6 +717,28 @@ class TestAlarmDefinitions(base.BaseMonascaTest): links = response_body['links'] self._verify_list_alarm_definitions_links(links) + @test.attr(type="gate") + def test_get_alarm_definition_with_multibyte_character(self): + # Create an alarm definition + name = data_utils.rand_name('ａｌａｒｍ＿ｄｅｆｉｎｉｔｉｏｎ').decode('utf8') + description = 'ｄｅｓｃｒｉｐｔｉｏｎ'.decode('utf8') + response_body_list = self._create_alarm_definitions( + name=name, + description=description, + number_of_definitions=1 + ) + alarm_definition = response_body_list[0] + + resp, response_body = self.monasca_client.get_alarm_definition( + alarm_definition['id']) + + self.assertEqual(200, resp.status) + self._verify_element_set(response_body) + self._verify_alarm_definitions_element(response_body, + alarm_definition) + links = response_body['links'] + self._verify_list_alarm_definitions_links(links) + # Update @test.attr(type="gate") @@ -951,17 +989,28 @@ class TestAlarmDefinitions(base.BaseMonascaTest): 'LOW', [], [], []) - def _create_alarm_definitions(self, expression, number_of_definitions): + def _create_alarm_definitions(self, number_of_definitions, **kwargs): self.rule = {'expression': 'mem_total_mb > 0'} + + expression = kwargs.get('expression', None) if expression is None: - expression = "max(cpu.system_perc) > 0" + expression = 'max(cpu.system_perc) > 0' + match_by = kwargs.get('match_by', ['device']) + response_body_list = [] for i in xrange(number_of_definitions): + + name = kwargs.get('name', + data_utils.rand_name('alarm_definition')) + desc = kwargs.get('description', + data_utils.rand_name('description')) + alarm_definition = helpers.create_alarm_definition( - name=data_utils.rand_name('alarm_definition'), - description=data_utils.rand_name('description'), + name=name, + description=desc, expression=expression, - match_by=['device']) + match_by=match_by + ) resp, response_body = self.monasca_client.create_alarm_definitions( alarm_definition) self.assertEqual(201, resp.status) diff --git a/monasca_tempest_tests/tests/api/test_metrics.py b/monasca_tempest_tests/tests/api/test_metrics.py index 41bee496b..5179ae808 100644 --- a/monasca_tempest_tests/tests/api/test_metrics.py +++ b/monasca_tempest_tests/tests/api/test_metrics.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- # (C) Copyright 2014-2016 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -17,6 +18,7 @@ import time from six.moves import range as xrange +from six.moves import urllib_parse as urlparse from tempest.lib.common.utils import data_utils from tempest.lib import exceptions @@ -29,14 +31,6 @@ from monasca_tempest_tests.tests.api import helpers class TestMetrics(base.BaseMonascaTest): - @classmethod - def resource_setup(cls): - super(TestMetrics, cls).resource_setup() - - @classmethod - def resource_cleanup(cls): - super(TestMetrics, cls).resource_cleanup() - @test.attr(type='gate') def test_create_metric(self): name = data_utils.rand_name('name') @@ -79,6 +73,48 @@ class TestMetrics(base.BaseMonascaTest): "metrics = 0" self.fail(error_msg) + @test.attr(type='gate') + def test_create_metric_with_multibyte_character(self): + name = data_utils.rand_name('ｎａｍｅ').decode('utf8') + key = data_utils.rand_name('ｋｅｙ').decode('utf8') + value = data_utils.rand_name('ｖａｌｕｅ').decode('utf8') + timestamp = int(round(time.time() * 1000)) + time_iso = helpers.timestamp_to_iso(timestamp) + end_timestamp = int(round((time.time() + 3600 * 24) * 1000)) + end_time_iso = helpers.timestamp_to_iso(end_timestamp) + value_meta_key = data_utils.rand_name('value_meta_ｋｅｙ').decode('utf8') + value_meta_value = data_utils.rand_name('value_meta_ｖａｌｕｅ').decode('utf8') + metric = helpers.create_metric(name=name, + dimensions={key: value}, + timestamp=timestamp, + value=1.23, + value_meta={ + value_meta_key: value_meta_value + }) + resp, response_body = self.monasca_client.create_metrics(metric) + self.assertEqual(204, resp.status) + query_param = '?name=' + urlparse.quote(name.encode('utf8')) + \ + '&start_time=' + time_iso + '&end_time=' + end_time_iso + for i in xrange(constants.MAX_RETRIES): + resp, response_body = self.monasca_client.\ + list_measurements(query_param) + self.assertEqual(200, resp.status) + elements = response_body['elements'] + for element in elements: + if element['name'] == name: + self._verify_list_measurements_element(element, key, value) + measurement = element['measurements'][0] + self._verify_list_measurements_measurement( + measurement, metric, value_meta_key, value_meta_value) + return + time.sleep(constants.RETRY_WAIT_SECS) + if i == constants.MAX_RETRIES - 1: + error_msg = "Failed test_create_metric: " \ + "timeout on waiting for metrics: at least " \ + "one metric is needed. Current number of " \ + "metrics = 0" + self.fail(error_msg) + @test.attr(type='gate') def test_create_metrics(self): name = data_utils.rand_name('name') @@ -547,7 +583,10 @@ class TestMetrics(base.BaseMonascaTest): set(['timestamp', 'value', 'value_meta'])) self.assertTrue(str(element['id']) is not None) if test_key is not None and test_value is not None: - self.assertEqual(str(element['dimensions'][test_key]), test_value) + self.assertEqual( + element['dimensions'][test_key].encode('utf-8'), + test_value.encode('utf-8') + ) def _verify_list_measurements_measurement(self, measurement, test_metric, test_vm_key, @@ -570,7 +609,10 @@ class TestMetrics(base.BaseMonascaTest): self.fail(error_msg) self.assertEqual(measurement[1], test_metric['value']) if test_vm_key is not None and test_vm_value is not None: - self.assertEqual(str(measurement[2][test_vm_key]), test_vm_value) + self.assertEqual( + measurement[2][test_vm_key].encode('utf-8'), + test_vm_value.encode('utf-8') + ) def _verify_list_metrics_element(self, element, test_key=None, test_value=None, test_name=None):