From 71fa2161688d3b6368f719460952e52433f2f875 Mon Sep 17 00:00:00 2001 From: Justin Ferrieu Date: Mon, 26 Aug 2019 14:14:05 +0000 Subject: [PATCH] Add support for POST /v2/dataframes API endpoint to the client Support for the ``/v2/dataframes`` endpoint has been added to the client. A new ``dataframes add`` CLI command is also available. Change-Id: I7fe9072d7280f251edc865a653a0b9ed2ab26c90 Story: 2005890 Task: 35970 (cherry picked from commit c8d7a9e1c5518499f8ddfcf9e217bd205f9e2588) --- cloudkittyclient/common/client.py | 3 + cloudkittyclient/tests/functional/base.py | 22 ++- .../tests/functional/v2/test_dataframes.py | 169 ++++++++++++++++++ cloudkittyclient/tests/unit/v2/base.py | 2 + .../tests/unit/v2/test_dataframes.py | 151 ++++++++++++++++ cloudkittyclient/v2/client.py | 2 + cloudkittyclient/v2/dataframes.py | 51 ++++++ cloudkittyclient/v2/dataframes_cli.py | 42 +++++ doc/source/api_reference/v2/dataframes.rst | 6 + doc/source/cli_reference.rst | 3 + ...upport-v2-dataframes-baa398fe5ea1b891.yaml | 5 + setup.cfg | 4 + 12 files changed, 452 insertions(+), 8 deletions(-) create mode 100644 cloudkittyclient/tests/functional/v2/test_dataframes.py create mode 100644 cloudkittyclient/tests/unit/v2/test_dataframes.py create mode 100644 cloudkittyclient/v2/dataframes.py create mode 100644 cloudkittyclient/v2/dataframes_cli.py create mode 100644 doc/source/api_reference/v2/dataframes.rst create mode 100644 releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml diff --git a/cloudkittyclient/common/client.py b/cloudkittyclient/common/client.py index c75608c..07f762b 100644 --- a/cloudkittyclient/common/client.py +++ b/cloudkittyclient/common/client.py @@ -26,6 +26,9 @@ class BaseClient(object): insecure=False, **kwargs): adapter_options.setdefault('service_type', 'rating') + adapter_options.setdefault('additional_headers', { + 'Content-Type': 'application/json', + }) if insecure: verify_cert = False diff --git a/cloudkittyclient/tests/functional/base.py b/cloudkittyclient/tests/functional/base.py index 37890fb..b1d0489 100644 --- a/cloudkittyclient/tests/functional/base.py +++ b/cloudkittyclient/tests/functional/base.py @@ -24,24 +24,30 @@ from cloudkittyclient.tests import utils class BaseFunctionalTest(utils.BaseTestCase): def _run(self, executable, action, - flags='', params='', fmt='-f json', has_output=True): + flags='', params='', fmt='-f json', stdin=None, has_output=True): if not has_output: fmt = '' cmd = ' '.join([executable, flags, action, params, fmt]) cmd = shlex.split(cmd) - p = subprocess.Popen(cmd, env=os.environ.copy(), shell=False, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - stdout, stderr = p.communicate() + p = subprocess.Popen( + cmd, env=os.environ.copy(), shell=False, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + stdin=subprocess.PIPE if stdin else None, + ) + stdout, stderr = p.communicate(input=stdin) if p.returncode != 0: raise RuntimeError('"{cmd}" returned {val}: {msg}'.format( cmd=' '.join(cmd), val=p.returncode, msg=stderr)) return json.loads(stdout) if has_output else None def openstack(self, action, - flags='', params='', fmt='-f json', has_output=True): + flags='', params='', fmt='-f json', + stdin=None, has_output=True): return self._run('openstack rating', action, - flags, params, fmt, has_output) + flags, params, fmt, stdin, has_output) def cloudkitty(self, action, - flags='', params='', fmt='-f json', has_output=True): - return self._run('cloudkitty', action, flags, params, fmt, has_output) + flags='', params='', fmt='-f json', + stdin=None, has_output=True): + return self._run('cloudkitty', action, flags, params, fmt, + stdin, has_output) diff --git a/cloudkittyclient/tests/functional/v2/test_dataframes.py b/cloudkittyclient/tests/functional/v2/test_dataframes.py new file mode 100644 index 0000000..50db400 --- /dev/null +++ b/cloudkittyclient/tests/functional/v2/test_dataframes.py @@ -0,0 +1,169 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import os +import uuid + +from cloudkittyclient.tests.functional import base + + +class CkDataframesTest(base.BaseFunctionalTest): + dataframes_data = """ + { + "dataframes": [ + { + "period": { + "begin": "20190723T122810Z", + "end": "20190723T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 1.2 + }, + "rating": { + "price": 0.04 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 200.4 + }, + "rating": { + "price": 0.06 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + }, + { + "period": { + "begin": "20190823T122810Z", + "end": "20190823T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 2.4 + }, + "rating": { + "price": 0.08 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 400.8 + }, + "rating": { + "price": 0.12 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + } + ] + } + """ + + def __init__(self, *args, **kwargs): + super(CkDataframesTest, self).__init__(*args, **kwargs) + self.runner = self.cloudkitty + + def setUp(self): + super(CkDataframesTest, self).setUp() + + self.fixture_file_name = '{}.json'.format(uuid.uuid4()) + with open(self.fixture_file_name, 'w') as f: + f.write(self.dataframes_data) + + def tearDown(self): + files = os.listdir('.') + if self.fixture_file_name in files: + os.remove(self.fixture_file_name) + + super(CkDataframesTest, self).tearDown() + + def test_dataframes_add_with_no_args(self): + self.assertRaisesRegexp( + RuntimeError, + 'error: too few arguments', + self.runner, + 'dataframes add', + fmt='', + has_output=False, + ) + + def test_dataframes_add(self): + self.runner( + 'dataframes add {}'.format(self.fixture_file_name), + fmt='', + has_output=False, + ) + + def test_dataframes_add_with_hyphen_stdin(self): + with open(self.fixture_file_name, 'r') as f: + self.runner( + 'dataframes add -', + fmt='', + stdin=f.read().encode(), + has_output=False, + ) + + +class OSCDataframesTest(CkDataframesTest): + def __init__(self, *args, **kwargs): + super(OSCDataframesTest, self).__init__(*args, **kwargs) + self.runner = self.openstack diff --git a/cloudkittyclient/tests/unit/v2/base.py b/cloudkittyclient/tests/unit/v2/base.py index f82d42b..70b9af1 100644 --- a/cloudkittyclient/tests/unit/v2/base.py +++ b/cloudkittyclient/tests/unit/v2/base.py @@ -13,6 +13,7 @@ # under the License. from cloudkittyclient.tests import utils +from cloudkittyclient.v2 import dataframes from cloudkittyclient.v2 import scope from cloudkittyclient.v2 import summary @@ -22,5 +23,6 @@ class BaseAPIEndpointTestCase(utils.BaseTestCase): def setUp(self): super(BaseAPIEndpointTestCase, self).setUp() self.api_client = utils.FakeHTTPClient() + self.dataframes = dataframes.DataframesManager(self.api_client) self.scope = scope.ScopeManager(self.api_client) self.summary = summary.SummaryManager(self.api_client) diff --git a/cloudkittyclient/tests/unit/v2/test_dataframes.py b/cloudkittyclient/tests/unit/v2/test_dataframes.py new file mode 100644 index 0000000..cd0cbce --- /dev/null +++ b/cloudkittyclient/tests/unit/v2/test_dataframes.py @@ -0,0 +1,151 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import json + +from cloudkittyclient import exc +from cloudkittyclient.tests.unit.v2 import base + + +class TestDataframes(base.BaseAPIEndpointTestCase): + dataframes_data = """ + { + "dataframes": [ + { + "period": { + "begin": "20190723T122810Z", + "end": "20190723T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 1.2 + }, + "rating": { + "price": 0.04 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 200.4 + }, + "rating": { + "price": 0.06 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + }, + { + "period": { + "begin": "20190823T122810Z", + "end": "20190823T132810Z" + }, + "usage": { + "metric_one": [ + { + "vol": { + "unit": "GiB", + "qty": 2.4 + }, + "rating": { + "price": 0.08 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ], + "metric_two": [ + { + "vol": { + "unit": "MB", + "qty": 400.8 + }, + "rating": { + "price": 0.12 + }, + "groupby": { + "group_one": "one", + "group_two": "two" + }, + "metadata": { + "attr_one": "one", + "attr_two": "two" + } + } + ] + } + } + ] + } + """ + + def test_add_dataframes_with_string(self): + self.dataframes.add_dataframes( + dataframes=self.dataframes_data, + ) + self.api_client.post.assert_called_once_with( + '/v2/dataframes', + data=self.dataframes_data, + ) + + def test_add_dataframes_with_json_object(self): + json_data = json.loads(self.dataframes_data) + + self.dataframes.add_dataframes( + dataframes=json_data, + ) + self.api_client.post.assert_called_once_with( + '/v2/dataframes', + data=json.dumps(json_data), + ) + + def test_add_dataframes_with_neither_string_nor_object_raises_exc(self): + self.assertRaises( + exc.InvalidArgumentError, + self.dataframes.add_dataframes, + dataframes=[open], + ) + + def test_add_dataframes_with_no_args_raises_exc(self): + self.assertRaises( + exc.ArgumentRequired, + self.dataframes.add_dataframes) diff --git a/cloudkittyclient/v2/client.py b/cloudkittyclient/v2/client.py index fd8abd4..0a61184 100644 --- a/cloudkittyclient/v2/client.py +++ b/cloudkittyclient/v2/client.py @@ -14,6 +14,7 @@ # under the License. # from cloudkittyclient.v1 import client +from cloudkittyclient.v2 import dataframes from cloudkittyclient.v2 import scope from cloudkittyclient.v2 import summary @@ -36,5 +37,6 @@ class Client(client.Client): **kwargs ) + self.dataframes = dataframes.DataframesManager(self.api_client) self.scope = scope.ScopeManager(self.api_client) self.summary = summary.SummaryManager(self.api_client) diff --git a/cloudkittyclient/v2/dataframes.py b/cloudkittyclient/v2/dataframes.py new file mode 100644 index 0000000..00c56b1 --- /dev/null +++ b/cloudkittyclient/v2/dataframes.py @@ -0,0 +1,51 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import json +import six + +from cloudkittyclient.common import base +from cloudkittyclient import exc + + +class DataframesManager(base.BaseManager): + """Class used to handle /v2/dataframes endpoint""" + + url = '/v2/dataframes' + + def add_dataframes(self, **kwargs): + """Add DataFrames to the storage backend. Returns nothing. + + :param dataframes: List of dataframes to add to the storage backend. + :type dataframes: list of dataframes + """ + + dataframes = kwargs.get('dataframes') + + if not dataframes: + raise exc.ArgumentRequired("'dataframes' argument is required") + + if not isinstance(dataframes, six.string_types): + try: + dataframes = json.dumps(dataframes) + except TypeError: + raise exc.InvalidArgumentError( + "'dataframes' must be either a string" + "or a JSON serializable object.") + + url = self.get_url(None, kwargs) + return self.api_client.post( + url, + data=dataframes, + ) diff --git a/cloudkittyclient/v2/dataframes_cli.py b/cloudkittyclient/v2/dataframes_cli.py new file mode 100644 index 0000000..92698f2 --- /dev/null +++ b/cloudkittyclient/v2/dataframes_cli.py @@ -0,0 +1,42 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import argparse + +from cliff import command + +from cloudkittyclient import utils + + +class CliDataframesAdd(command.Command): + """Add one or several DataFrame objects to the storage backend.""" + def get_parser(self, prog_name): + parser = super(CliDataframesAdd, self).get_parser(prog_name) + + parser.add_argument( + 'datafile', + type=argparse.FileType('r'), + help="File formatted as a JSON object having a DataFrame list" + "under a 'dataframes' key." + "'-' (hyphen) can be specified for using stdin.", + ) + + return parser + + def take_action(self, parsed_args): + with parsed_args.datafile as dfile: + dataframes = dfile.read() + utils.get_client_from_osc(self).dataframes.add_dataframes( + dataframes=dataframes, + ) diff --git a/doc/source/api_reference/v2/dataframes.rst b/doc/source/api_reference/v2/dataframes.rst new file mode 100644 index 0000000..ba5eb6c --- /dev/null +++ b/doc/source/api_reference/v2/dataframes.rst @@ -0,0 +1,6 @@ +=========================== +dataframes (/v2/dataframes) +=========================== + +.. automodule:: cloudkittyclient.v2.dataframes + :members: diff --git a/doc/source/cli_reference.rst b/doc/source/cli_reference.rst index 350192f..bd81c64 100644 --- a/doc/source/cli_reference.rst +++ b/doc/source/cli_reference.rst @@ -12,6 +12,9 @@ V1 Client V2 Client ========= +.. autoprogram-cliff:: cloudkittyclient.v2 + :command: dataframes add + .. autoprogram-cliff:: cloudkittyclient.v2 :command: scope state get diff --git a/releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml b/releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml new file mode 100644 index 0000000..732337c --- /dev/null +++ b/releasenotes/notes/add-support-v2-dataframes-baa398fe5ea1b891.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Support for the ``/v2/dataframes`` endpoint has been added to the client. + A new ``dataframes add`` CLI command is also available. diff --git a/setup.cfg b/setup.cfg index ac76d92..44a7ec2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -86,6 +86,8 @@ openstack.rating.v1 = rating_pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript openstack.rating.v2 = + rating_dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd + rating_scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet rating_scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset @@ -200,6 +202,8 @@ cloudkittyclient.v1 = pyscript_delete = cloudkittyclient.v1.rating.pyscripts_cli:CliDeleteScript cloudkittyclient.v2 = + dataframes_add = cloudkittyclient.v2.dataframes_cli:CliDataframesAdd + scope_state_get = cloudkittyclient.v2.scope_cli:CliScopeStateGet scope_state_reset = cloudkittyclient.v2.scope_cli:CliScopeStateReset