# # Copyright 2016 Hewlett Packard # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import collections import os import mock from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslo_utils import timeutils from oslotest import base from oslotest import mockpatch import six import yaml from ceilometer.ceilosca_mapping import ceilosca_mapping from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinition) from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader from ceilometer import storage from ceilometer.storage import impl_monasca from ceilometer.storage import models as storage_models MONASCA_MEASUREMENT = { "id": "fef26f9d27f8027ea44b940cf3626fc398f7edfb", "name": "fake_metric", "dimensions": { "resource_id": "2fe6e3a9-9bdf-4c98-882c-a826cf0107a1", "cloud_name": "helion-poc-hlm-003", "component": "vm", "control_plane": "control-plane-1", "service": "compute", "device": "tap3356676e-a5", "tenant_id": "50ce24dd577c43879cede72b77224e2f", "hostname": "hlm003-cp1-comp0003-mgmt", "cluster": "compute", "zone": "nova" }, "columns": ["timestamp", "value", "value_meta"], "measurements": [["2016-05-23T22:22:42.000Z", 54.0, { "audit_period_ending": "None", "audit_period_beginning": "None", "host": "network.hlm003-cp1-c1-m2-mgmt", "availability_zone": "None", "event_type": "subnet.create.end", "enable_dhcp": "true", "gateway_ip": "10.43.0.1", "ip_version": "4", "cidr": "10.43.0.0/28"}]] } MONASCA_VALUE_META = { 'audit_period_beginning': 'None', 'audit_period_ending': 'None', 'availability_zone': 'None', 'cidr': '10.43.0.0/28', 'enable_dhcp': 'true', 'event_type': 'subnet.create.end', 'gateway_ip': '10.43.0.1', 'host': 'network.hlm003-cp1-c1-m2-mgmt', 'ip_version': '4' } class TestCeiloscaMapping(base.BaseTestCase): pipeline_data = yaml.dump({ 'sources': [{ 'name': 'test_pipeline', 'interval': 1, 'meters': ['testbatch', 'testbatch2'], 'resources': ['alpha', 'beta', 'gamma', 'delta'], 'sinks': ['test_sink']}], 'sinks': [{ 'name': 'test_sink', 'transformers': [], 'publishers': ["test"]}] }) cfg = yaml.dump({ 'meter_metric_map': [{ 'user_id': '$.dimensions.user_id', 'name': 'fake_meter', 'resource_id': '$.dimensions.resource_id', 'region': 'NA', 'monasca_metric_name': 'fake_metric', 'source': 'NA', 'project_id': '$.dimensions.tenant_id', 'type': 'gauge', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' }, { 'user_id': '$.dimensions.user_id', 'name': 'fake_meter2', 'resource_id': '$.dimensions.resource_id', 'region': 'NA', 'monasca_metric_name': 'fake_metric2', 'source': 'NA', 'project_id': '$.dimensions.project_id', 'type': 'delta', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' }, { 'user_id': '$.dimensions.user_id', 'name': 'fake_meter3', 'resource_id': '$.dimensions.hostname', 'region': 'NA', 'monasca_metric_name': 'fake_metric3', 'source': 'NA', 'project_id': '$.dimensions.project_id', 'type': 'delta', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' } ] }) def setup_pipeline_file(self, pipeline_data): if six.PY3: pipeline_data = pipeline_data.encode('utf-8') pipeline_cfg_file = fileutils.write_to_tempfile(content=pipeline_data, prefix="pipeline", suffix="yaml") self.addCleanup(os.remove, pipeline_cfg_file) return pipeline_cfg_file def setup_ceilosca_mapping_def_file(self, cfg): if six.PY3: cfg = cfg.encode('utf-8') ceilosca_mapping_file = fileutils.write_to_tempfile( content=cfg, prefix='ceilosca_mapping', suffix='yaml') self.addCleanup(os.remove, ceilosca_mapping_file) return ceilosca_mapping_file class TestGetPipelineReader(TestCeiloscaMapping): def setUp(self): super(TestGetPipelineReader, self).setUp() self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF([], project='ceilometer', validate_default_values=True) def test_pipeline_reader(self): pipeline_cfg_file = self.setup_pipeline_file( self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) test_pipeline_reader = PipelineReader() self.assertEqual(set(['testbatch', 'testbatch2']), test_pipeline_reader.get_pipeline_meters() ) class TestMappingDefinition(base.BaseTestCase): def test_mapping_definition(self): cfg = dict(name="network.outgoing.rate", monasca_metric_name="vm.net.out_bytes_sec", resource_id="$.dimensions.resource_id", project_id="$.dimensions.tenant_id", user_id="$.dimensions.user_id", region="NA", type="gauge", unit="B/s", source="NA", resource_metadata="$.measurements[0][2]") handler = CeiloscaMappingDefinition(cfg) self.assertIsNone(handler.parse_fields("user_id", MONASCA_MEASUREMENT)) self.assertEqual("2fe6e3a9-9bdf-4c98-882c-a826cf0107a1", handler.parse_fields("resource_id", MONASCA_MEASUREMENT)) self.assertEqual("50ce24dd577c43879cede72b77224e2f", handler.parse_fields("project_id", MONASCA_MEASUREMENT)) self.assertEqual(MONASCA_VALUE_META, handler.parse_fields("resource_metadata", MONASCA_MEASUREMENT)) self.assertEqual("$.dimensions.tenant_id", handler.cfg["project_id"]) def test_config_required_missing_fields(self): cfg = dict() try: CeiloscaMappingDefinition(cfg) except CeiloscaMappingDefinitionException as e: self.assertEqual("Required fields [" "'name', 'monasca_metric_name', 'type', 'unit', " "'source', 'resource_metadata', 'resource_id', " "'project_id', 'user_id', 'region'] " "not specified", e.message) def test_bad_type_cfg_definition(self): cfg = dict(name="fake_meter", monasca_metric_name="fake_metric", resource_id="$.dimensions.resource_id", project_id="$.dimensions.tenant_id", user_id="$.dimensions.user_id", region="NA", type="foo", unit="B/s", source="NA", resource_metadata="$.measurements[0][2]") try: CeiloscaMappingDefinition(cfg) except CeiloscaMappingDefinitionException as e: self.assertEqual("Invalid type foo specified", e.message) class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): def setUp(self): super(TestMappedCeiloscaMetricProcessing, self).setUp() self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF([], project='ceilometer', validate_default_values=True) def test_fallback_mapping_file_path(self): self.useFixture(mockpatch.PatchObject(self.CONF, 'find_file', return_value=None)) fall_bak_path = ceilosca_mapping.get_config_file() self.assertIn("ceilosca_mapping/data/ceilosca_mapping.yaml", fall_bak_path) @mock.patch('ceilometer.ceilosca_mapping.ceilosca_mapping.LOG') def test_bad_meter_definition_skip(self, LOG): cfg = yaml.dump({ 'meter_metric_map': [{ 'user_id': '$.dimensions.user_id', 'name': 'fake_meter', 'resource_id': '$.dimensions.resource_id', 'region': 'NA', 'monasca_metric_name': 'fake_metric', 'source': 'NA', 'project_id': '$.dimensions.tenant_id', 'type': 'gauge', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' }, { 'user_id': '$.dimensions.user_id', 'name': 'fake_meter', 'resource_id': '$.dimensions.resource_id', 'region': 'NA', 'monasca_metric_name': 'fake_metric', 'source': 'NA', 'project_id': '$.dimensions.tenant_id', 'type': 'foo', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' }] }) ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') data = ceilosca_mapping.setup_ceilosca_mapping_config() meter_loaded = ceilosca_mapping.load_definitions(data) self.assertEqual(1, len(meter_loaded)) LOG.error.assert_called_with( "Error loading Ceilometer Monasca Mapping Definition : " "Invalid type foo specified") def test_list_of_meters_returned(self): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() ceilosca_mapper.reinitialize() self.assertItemsEqual(['fake_metric', 'fake_metric2', 'fake_metric3'], ceilosca_mapper.get_list_monasca_metrics().keys() ) def test_monasca_metric_name_map_ceilometer_meter(self): cfg = yaml.dump({ 'meter_metric_map': [{ 'user_id': '$.dimensions.user_id', 'name': 'fake_meter', 'resource_id': '$.dimensions.resource_id', 'region': 'NA', 'monasca_metric_name': 'fake_metric', 'source': 'NA', 'project_id': '$.dimensions.tenant_id', 'type': 'gauge', 'resource_metadata': '$.measurements[0][2]', 'unit': 'B/s' }] }) ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() ceilosca_mapper.reinitialize() self.assertEqual('fake_metric', ceilosca_mapper.get_monasca_metric_name('fake_meter') ) self.assertEqual('$.dimensions.tenant_id', ceilosca_mapper. get_ceilosca_mapped_definition_key_val('fake_metric', 'project_id')) # This Class will only test the driver for the mapped meteric # Impl_Monasca Tests will be doing exhaustive tests for non mapped metrics @mock.patch("ceilometer.storage.impl_monasca.MonascaDataFilter") class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): Aggregate = collections.namedtuple("Aggregate", ['func', 'param']) def setUp(self): super(TestMoanscaDriverForMappedMetrics, self).setUp() self.CONF = self.useFixture(fixture_config.Config()).conf self.CONF([], project='ceilometer', validate_default_values=True) pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() ceilosca_mapper.reinitialize() def test_get_samples_for_mapped_meters(self, mdf_mock): with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") ml_mock = mock_client().measurements_list # TODO(this test case needs more work) ml_mock.return_value = ([MONASCA_MEASUREMENT]) sample_filter = storage.SampleFilter( meter='fake_meter', start_timestamp='2015-03-20T00:00:00Z') results = list(conn.get_samples(sample_filter)) self.assertEqual(True, ml_mock.called) self.assertEqual('fake_meter', results[0].counter_name) self.assertEqual(54.0, results[0].counter_volume) self.assertEqual('gauge', results[0].counter_type) self.assertEqual('2fe6e3a9-9bdf-4c98-882c-a826cf0107a1', results[0].resource_id ) self.assertEqual(MONASCA_VALUE_META, results[0].resource_metadata) self.assertEqual('50ce24dd577c43879cede72b77224e2f', results[0].project_id, ) self.assertEqual('B/s', results[0].counter_unit) self.assertIsNone(results[0].user_id) def test_get_meter_for_mapped_meters_non_uniq(self, mdf_mock): data1 = ( [{u'dimensions': {u'datasource': u'ceilometer'}, u'id': u'2015-04-14T18:42:31Z', u'name': u'meter-1'}, {u'dimensions': {u'datasource': u'ceilometer'}, u'id': u'2015-04-15T18:42:31Z', u'name': u'meter-1'}]) data2 = ( [{u'dimensions': {u'datasource': u'ceilometer'}, u'id': u'2015-04-14T18:42:31Z', u'name': u'meter-1'}, {u'dimensions': {u'datasource': u'ceilometer'}, u'id': u'2015-04-15T18:42:31Z', u'name': u'meter-1'}, {u'id': u'fef26f9d27f8027ea44b940cf3626fc398f7edfb', u'name': u'fake_metric', u'dimensions': { u'resource_id': u'2fe6e3a9-9bdf-4c98-882c-a826cf0107a1', u'cloud_name': u'helion-poc-hlm-003', u'component': u'vm', u'control_plane': u'control-plane-1', u'service': u'compute', u'device': u'tap3356676e-a5', u'tenant_id': u'50ce24dd577c43879cede72b77224e2f', u'hostname': u'hlm003-cp1-comp0003-mgmt', u'cluster': u'compute', u'zone': u'nova'} }, {u'dimensions': {}, u'id': u'2015-04-16T18:42:31Z', u'name': u'testbatch'}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.side_effect = [data1, data2] kwargs = dict(limit=4) results = list(conn.get_meters(**kwargs)) # result contains 2 records from data 1 since datasource # = ceilometer, 2 records from data 2, 1 for pipeline # meter but no datasource set to ceilometer and one for # mapped meter self.assertEqual(4, len(results)) self.assertEqual(True, metrics_list_mock.called) self.assertEqual(2, metrics_list_mock.call_count) def test_get_meter_for_mapped_meters_uniq(self, mdf_mock): dummy_metric_names_mocked_return_value = ( [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", "name": "network.delete"}, {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "subnet.update"}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value) kwargs = dict(limit=4, unique=True) results = list(conn.get_meters(**kwargs)) self.assertEqual(2, len(results)) self.assertEqual(True, metric_names_list_mock.called) self.assertEqual(1, metric_names_list_mock.call_count) def test_stats_list_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { 'statistics': [ ['2014-10-24T12:12:12Z', 0.008], ['2014-10-24T12:52:12Z', 0.018] ], 'dimensions': {'unit': 'gb'}, 'columns': ['timestamp', 'min'] } ] sf = storage.SampleFilter() sf.meter = "fake_meter" aggregate = self.Aggregate(func="min", param=None) sf.start_timestamp = timeutils.parse_isotime( '2014-10-24T12:12:42').replace(tzinfo=None) stats = list(conn.get_meter_statistics(sf, aggregate=[aggregate], period=30)) self.assertEqual(2, len(stats)) self.assertEqual('B/s', stats[0].unit) self.assertEqual('B/s', stats[1].unit) self.assertEqual(0.008, stats[0].min) self.assertEqual(0.018, stats[1].min) self.assertEqual(30, stats[0].period) self.assertEqual('2014-10-24T12:12:42', stats[0].period_end.isoformat()) self.assertEqual('2014-10-24T12:52:42', stats[1].period_end.isoformat()) self.assertIsNotNone(stats[0].as_dict().get('aggregate')) self.assertEqual({u'min': 0.008}, stats[0].as_dict()['aggregate']) def test_get_resources_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") dummy_metric_names_mocked_return_value = ( [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", "name": "fake_metric"}, {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "metric1"}]) mnl_mock = mock_client().metric_names_list mnl_mock.return_value = ( dummy_metric_names_mocked_return_value) dummy_get_resources_mocked_return_value = ( [{u'dimensions': {u'resource_id': u'abcd'}, u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], [u'2015-04-15T17:52:31Z', 2.0, {}], [u'2015-04-16T17:52:31Z', 3.0, {}]], u'id': u'2015-04-14T18:42:31Z', u'columns': [u'timestamp', u'value', u'value_meta'], u'name': u'fake_metric'}]) ml_mock = mock_client().measurements_list ml_mock.return_value = ( dummy_get_resources_mocked_return_value) sample_filter = storage.SampleFilter( meter='fake_meter', end_timestamp='2015-04-20T00:00:00Z') resources = list(conn.get_resources(sample_filter, limit=2)) self.assertEqual(2, len(resources)) self.assertEqual(True, ml_mock.called) self.assertEqual(1, ml_mock.call_count) resources_without_limit = list(conn.get_resources(sample_filter)) self.assertEqual(3, len(resources_without_limit)) def test_stats_list_with_groupby_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: conn = impl_monasca.Connection("127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { 'statistics': [ ['2014-10-24T12:12:12Z', 0.008, 1.3, 3, 0.34], ['2014-10-24T12:20:12Z', 0.078, 1.25, 2, 0.21], ['2014-10-24T12:52:12Z', 0.018, 0.9, 4, 0.14] ], 'dimensions': {'hostname': '1234', 'unit': 'gb'}, 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] }, { 'statistics': [ ['2014-10-24T12:14:12Z', 0.45, 2.5, 2, 2.1], ['2014-10-24T12:20:12Z', 0.58, 3.2, 3, 3.4], ['2014-10-24T13:52:42Z', 1.67, 3.5, 1, 5.3] ], 'dimensions': {'hostname': '5678', 'unit': 'gb'}, 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] }] sf = storage.SampleFilter() sf.meter = "fake_meter3" sf.start_timestamp = timeutils.parse_isotime( '2014-10-24T12:12:42').replace(tzinfo=None) groupby = ['resource_id'] stats = list(conn.get_meter_statistics(sf, period=30, groupby=groupby)) self.assertEqual(2, len(stats)) for stat in stats: self.assertIsNotNone(stat.groupby) resource_id = stat.groupby.get('resource_id') self.assertIn(resource_id, ['1234', '5678']) if resource_id == '1234': self.assertEqual(0.008, stat.min) self.assertEqual(1.3, stat.max) self.assertEqual(0.23, stat.avg) self.assertEqual(9, stat.count) self.assertEqual(30, stat.period) self.assertEqual('2014-10-24T12:12:12', stat.period_start.isoformat()) if resource_id == '5678': self.assertEqual(0.45, stat.min) self.assertEqual(3.5, stat.max) self.assertEqual(3.6, stat.avg) self.assertEqual(6, stat.count) self.assertEqual(30, stat.period) self.assertEqual('2014-10-24T13:52:42', stat.period_end.isoformat()) def test_query_samples_for_mapped_meter(self, mock_mdf): SAMPLES = [[ storage_models.Sample( counter_name="fake_meter", counter_type="gauge", counter_unit="instance", counter_volume=1, project_id="123", user_id="456", resource_id="789", resource_metadata={}, source="openstack", recorded_at=timeutils.utcnow(), timestamp=timeutils.utcnow(), message_id="0", message_signature='', ) ]] * 2 samples = SAMPLES[:] def _get_samples(*args, **kwargs): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): conn = impl_monasca.Connection("127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples query = {'and': [{'=': {'counter_name': 'fake_meter'}}, {'or': [{'=': {"project_id": "123"}}, {'=': {"user_id": "456"}}]}]} samples = conn.query_samples(query, None, 100) self.assertEqual(2, len(samples)) self.assertEqual(2, gsm.call_count) samples = SAMPLES[:] query = {'and': [{'=': {'counter_name': 'fake_meter'}}, {'or': [{'=': {"project_id": "123"}}, {'>': {"counter_volume": 2}}]}]} samples = conn.query_samples(query, None, 100) self.assertEqual(1, len(samples)) self.assertEqual(4, gsm.call_count)