Client support for instance module feature

This adds support in the python API and Trove CLI
for instance module commands.  These commands include:

    - module-apply
    - module-remove
    - module-query
    - module-retrieve
    - module-list-instance

The parsing of --instance was modified to allow multiple
modules to be specified.  This was extended to 'nics' as well.

Partially Implements: blueprint module-management
Change-Id: If62f5e51d4628cc6a8b10303d5c3893b3bd5057e
This commit is contained in:
Peter Stachowski 2016-03-08 00:22:06 -05:00
parent 3c71e52b40
commit 457360c69f
13 changed files with 790 additions and 131 deletions

View File

@ -13,3 +13,4 @@ testscenarios>=0.4 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
mock>=1.2 # BSD
httplib2>=0.7.5 # MIT
pycrypto>=2.6 # Public Domain

View File

@ -34,10 +34,11 @@ class ClientException(Exception):
class MissingArgs(ClientException):
"""Supplied arguments are not sufficient for calling a function."""
def __init__(self, missing):
def __init__(self, missing, message=None):
self.missing = missing
msg = "Missing argument(s): %s" % ", ".join(missing)
super(MissingArgs, self).__init__(msg)
self.message = message or "Missing argument(s): %s"
self.message %= ", ".join(missing)
super(MissingArgs, self).__init__(self.message)
class ValidationError(ClientException):

View File

@ -31,6 +31,7 @@ import six
from six.moves.urllib import parse
from troveclient.openstack.common.apiclient import client
from troveclient.tests import utils
def assert_has_keys(dct, required=[], optional=[]):
@ -86,8 +87,9 @@ class FakeHTTPClient(client.HTTPClient):
def assert_called(self, method, url, body=None, pos=-1):
"""Assert than an API method was just called.
"""
expected = (method, url)
called = self.callstack[pos][0:2]
expected = (method, utils.order_url(url))
called = (self.callstack[pos][0],
utils.order_url(self.callstack[pos][1]))
assert self.callstack, \
"Expected %s %s but no calls were made." % expected
@ -102,7 +104,7 @@ class FakeHTTPClient(client.HTTPClient):
def assert_called_anytime(self, method, url, body=None):
"""Assert than an API method was called anytime in the test.
"""
expected = (method, url)
expected = (method, utils.order_url(url))
assert self.callstack, \
"Expected %s %s but no calls were made." % expected
@ -110,7 +112,7 @@ class FakeHTTPClient(client.HTTPClient):
found = False
entry = None
for entry in self.callstack:
if expected == entry[0:2]:
if expected == (entry[0], utils.order_url(entry[1])):
found = True
break

View File

@ -35,16 +35,32 @@ def assert_has_keys(dict, required=[], optional=[]):
class FakeClient(client.Client):
URL_QUERY_SEPARATOR = '&'
URL_SEPARATOR = '?'
def __init__(self, *args, **kwargs):
client.Client.__init__(self, 'username', 'password',
'project_id', 'auth_url',
extensions=kwargs.get('extensions'))
self.client = FakeHTTPClient(**kwargs)
def _order_url_query_str(self, url):
"""Returns the url with the query strings ordered, if they exist and
there's more than one. Otherwise the url is returned unaltered.
"""
if self.URL_QUERY_SEPARATOR in url:
parts = url.split(self.URL_SEPARATOR)
if len(parts) == 2:
queries = sorted(parts[1].split(self.URL_QUERY_SEPARATOR))
url = self.URL_SEPARATOR.join(
[parts[0], self.URL_QUERY_SEPARATOR.join(queries)])
return url
def assert_called(self, method, url, body=None, pos=-1):
"""Assert than an API method was just called."""
expected = (method, url)
called = self.client.callstack[pos][0:2]
expected = (method, utils.order_url(url))
called = (self.client.callstack[pos][0],
utils.order_url(self.client.callstack[pos][1]))
assert self.client.callstack, \
"Expected %s %s but no calls were made." % expected
@ -59,14 +75,14 @@ class FakeClient(client.Client):
def assert_called_anytime(self, method, url, body=None):
"""Assert than an API method was called anytime in the test."""
expected = (method, url)
expected = (method, utils.order_url(url))
assert self.client.callstack, \
"Expected %s %s but no calls were made." % expected
found = False
for entry in self.client.callstack:
if expected == entry[0:2]:
if expected == (entry[0], utils.order_url(entry[1])):
found = True
break
@ -389,6 +405,67 @@ class FakeHTTPClient(base_client.HTTPClient):
def get_instances_1234_metadata_key123(self, **kw):
return (200, {}, {"metadata": {}})
def get_modules(self, **kw):
return (200, {}, {"modules": [
{
"id": "4321",
"name": "mod1",
"type": "ping",
"datastore": 'all',
"datastore_version": 'all',
"tenant": 'all',
"auto_apply": 0,
"visible": 1},
{
"id": "8765",
"name": "mod2",
"type": "ping",
"datastore": 'all',
"datastore_version": 'all',
"tenant": 'all',
"auto_apply": 0,
"visible": 1}]})
def get_modules_4321(self, **kw):
r = {'module': self.get_modules()[2]['modules'][0]}
return (200, {}, r)
def get_modules_8765(self, **kw):
r = {'module': self.get_modules()[2]['modules'][1]}
return (200, {}, r)
def post_modules(self, **kw):
r = {'module': self.get_modules()[2]['modules'][0]}
return (200, {}, r)
def put_modules_4321(self, **kw):
return (200, {}, {"module": {'name': 'mod3'}})
def delete_modules_4321(self, **kw):
return (200, {}, None)
def get_instances_1234_modules(self, **kw):
return (200, {}, {"modules": [{"module": {}}]})
def get_modules_4321_instances(self, **kw):
return self.get_instances()
def get_instances_modules(self, **kw):
return (200, {}, None)
def get_instances_member_1_modules(self, **kw):
return self.get_modules()
def get_instances_member_2_modules(self, **kw):
return self.get_modules()
def post_instances_1234_modules(self, **kw):
r = {'modules': [self.get_modules()[2]['modules'][0]]}
return (200, {}, r)
def delete_instances_1234_modules_4321(self, **kw):
return (200, {}, None)
def get_limits(self, **kw):
return (200, {}, {"limits": [
{

View File

@ -99,7 +99,8 @@ class InstancesTest(testtools.TestCase):
['db1', 'db2'], ['u1', 'u2'],
datastore="datastore",
datastore_version="datastore-version",
nics=nics, slave_of='test')
nics=nics, slave_of='test',
modules=['mod_id'])
self.assertEqual("/instances", p)
self.assertEqual("instance", i)
self.assertEqual(['db1', 'db2'], b["instance"]["databases"])
@ -116,6 +117,7 @@ class InstancesTest(testtools.TestCase):
self.assertEqual('test', b['instance']['replica_of'])
self.assertNotIn('slave_of', b['instance'])
self.assertTrue(mock_warn.called)
self.assertEqual([{'id': 'mod_id'}], b["instance"]["modules"])
def test_list(self):
page_mock = mock.Mock()

View File

@ -14,8 +14,10 @@
# under the License.
#
import Crypto.Random
import mock
import testtools
from troveclient.v1 import modules
@ -52,25 +54,29 @@ class TestModules(testtools.TestCase):
def side_effect_func(path, body, mod):
return path, body, mod
self.modules._create = mock.Mock(side_effect=side_effect_func)
path, body, mod = self.modules.create(
self.module_name, "test", "my_contents",
description="my desc",
all_tenants=False,
datastore="ds",
datastore_version="ds-version",
auto_apply=True,
visible=True,
live_update=False)
self.assertEqual("/modules", path)
self.assertEqual("module", mod)
self.assertEqual(self.module_name, body["module"]["name"])
self.assertEqual("ds", body["module"]["datastore"]["type"])
self.assertEqual("ds-version", body["module"]["datastore"]["version"])
self.assertFalse(body["module"]["all_tenants"])
self.assertTrue(body["module"]["auto_apply"])
self.assertTrue(body["module"]["visible"])
self.assertFalse(body["module"]["live_update"])
text_contents = "my_contents"
binary_contents = Crypto.Random.new().read(20)
for contents in [text_contents, binary_contents]:
self.modules._create = mock.Mock(side_effect=side_effect_func)
path, body, mod = self.modules.create(
self.module_name, "test", contents,
description="my desc",
all_tenants=False,
datastore="ds",
datastore_version="ds-version",
auto_apply=True,
visible=True,
live_update=False)
self.assertEqual("/modules", path)
self.assertEqual("module", mod)
self.assertEqual(self.module_name, body["module"]["name"])
self.assertEqual("ds", body["module"]["datastore"]["type"])
self.assertEqual("ds-version",
body["module"]["datastore"]["version"])
self.assertFalse(body["module"]["all_tenants"])
self.assertTrue(body["module"]["auto_apply"])
self.assertTrue(body["module"]["visible"])
self.assertFalse(body["module"]["live_update"])
def test_update(self):
resp = mock.Mock()

View File

@ -15,9 +15,12 @@
# License for the specific language governing permissions and limitations
# under the License.
import Crypto.Random
import os
import six
import tempfile
import testtools
from troveclient import utils
@ -53,3 +56,52 @@ class UtilsTest(testtools.TestCase):
self.assertEqual('not_unicode', utils.slugify('not_unicode'))
self.assertEqual('unicode', utils.slugify(six.u('unicode')))
self.assertEqual('slugify-test', utils.slugify('SLUGIFY% test!'))
def test_encode_decode_data(self):
text_data_str = 'This is a text string'
try:
text_data_bytes = bytes('This is a byte stream', 'utf-8')
except TypeError:
text_data_bytes = bytes('This is a byte stream')
random_data_str = Crypto.Random.new().read(12)
random_data_bytes = bytearray(Crypto.Random.new().read(12))
special_char_str = '\x00\xFF\x00\xFF\xFF\x00'
special_char_bytes = bytearray(
[ord(item) for item in special_char_str])
data = [text_data_str,
text_data_bytes,
random_data_str,
random_data_bytes,
special_char_str,
special_char_bytes]
for datum in data:
# the deserialized data is always a bytearray
try:
expected_deserialized = bytearray(
[ord(item) for item in datum])
except TypeError:
expected_deserialized = bytearray(
[item for item in datum])
serialized_data = utils.encode_data(datum)
self.assertIsNotNone(serialized_data, "'%s' serialized is None" %
datum)
deserialized_data = utils.decode_data(serialized_data)
self.assertIsNotNone(deserialized_data, "'%s' deserialized is None"
% datum)
self.assertEqual(expected_deserialized, deserialized_data,
"Serialize/Deserialize failed")
# Now we write the data to a file and read it back in
# to make sure the round-trip doesn't change anything.
with tempfile.NamedTemporaryFile() as temp_file:
with open(temp_file.name, 'wb') as fh_w:
fh_w.write(
bytearray([ord(item) for item in serialized_data]))
with open(temp_file.name, 'rb') as fh_r:
new_serialized_data = fh_r.read()
new_deserialized_data = utils.decode_data(
new_serialized_data)
self.assertIsNotNone(new_deserialized_data,
"'%s' deserialized is None" % datum)
self.assertEqual(expected_deserialized, new_deserialized_data,
"Serialize/Deserialize with files failed")

View File

@ -13,15 +13,26 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
try:
# handle py34
import builtins
except ImportError:
# and py27
import __builtin__ as builtins
import base64
import fixtures
import mock
import re
import six
import testtools
import troveclient.client
from troveclient import exceptions
import troveclient.shell
from troveclient.tests import fakes
from troveclient.tests import utils
import troveclient.v1.modules
import troveclient.v1.shell
@ -86,6 +97,76 @@ class ShellTest(utils.TestCase):
def assert_called_anytime(self, method, url, body=None):
return self.shell.cs.assert_called_anytime(method, url, body)
def test__strip_option(self):
# Format is: opt_name, opt_string, _strip_options_kwargs,
# expected_value, expected_opt_string, exception_msg
data = [
["volume", "volume=10",
{}, "10", "", None],
["volume", ",volume=10,,type=mine,",
{}, "10", "type=mine", None],
["volume", "type=mine",
{}, "", "type=mine", "Missing option 'volume'.*"],
["volume", "type=mine",
{'is_required': False}, None, "type=mine", None],
["volume", "volume=1, volume=2",
{}, "", "", "Option 'volume' found more than once.*"],
["volume", "volume=1, volume=2",
{'allow_multiple': True}, ['1', '2'], "", None],
["volume", "volume=1, volume=2,, volume=4, volume=6",
{'allow_multiple': True}, ['1', '2', '4', '6'], "", None],
["module", ",flavor=10,,nic='net-id=net',module=test, module=test",
{'allow_multiple': True}, ['test'],
"flavor=10,,nic='net-id=net'", None],
["nic", ",flavor=10,,nic=net-id=net, module=test",
{'quotes_required': True}, "", "",
"Invalid 'nic' option. The value must be quoted.*"],
["nic", ",flavor=10,,nic='net-id=net', module=test",
{'quotes_required': True}, "net-id=net",
"flavor=10,, module=test", None],
["nic",
",nic='port-id=port',flavor=10,,nic='net-id=net', module=test",
{'quotes_required': True, 'allow_multiple': True},
["net-id=net", "port-id=port"],
"flavor=10,, module=test", None],
]
count = 0
for datum in data:
count += 1
opt_name = datum[0]
opts_str = datum[1]
kwargs = datum[2]
expected_value = datum[3]
expected_opt_string = datum[4]
exception_msg = datum[5]
msg = "Error (test data line %s): " % count
try:
value, opt_string = troveclient.v1.shell._strip_option(
opts_str, opt_name, **kwargs)
if exception_msg:
self.assertEqual(True, False,
"%sException not thrown, expecting %s" %
(msg, exception_msg))
if isinstance(expected_value, list):
self.assertEqual(
set(value), set(expected_value),
"%sValue not correct" % msg)
else:
self.assertEqual(value, expected_value,
"%sValue not correct" % msg)
self.assertEqual(opt_string, expected_opt_string,
"%sOption string not correct" % msg)
except Exception as ex:
if exception_msg:
msg = ex.message if hasattr(ex, 'message') else str(ex)
self.assertThat(msg,
testtools.matchers.MatchesRegex(
exception_msg, re.DOTALL),
exception_msg, "%sWrong ex" % msg)
else:
raise
def test_instance_list(self):
self.run_command('list')
self.assert_called('GET', '/instances')
@ -184,6 +265,19 @@ class ShellTest(utils.TestCase):
'replica_count': 1
}})
def test_boot_with_modules(self):
self.run_command('create test-member-1 1 --size 1 --volume_type lvm '
'--module 4321 --module 8765')
self.assert_called_anytime(
'POST', '/instances',
{'instance': {
'volume': {'size': 1, 'type': 'lvm'},
'flavorRef': 1,
'name': 'test-member-1',
'replica_count': 1,
'modules': [{'id': '4321'}, {'id': '8765'}]
}})
def test_boot_by_flavor_name(self):
self.run_command(
'create test-member-1 m1.tiny --size 1 --volume_type lvm')
@ -253,7 +347,7 @@ class ShellTest(utils.TestCase):
cmd = ('cluster-create test-clstr vertica 7.1 --instance volume=2 '
'--instance flavor=2,volume=1')
self.assertRaisesRegexp(
exceptions.MissingArgs, 'Missing argument\(s\): flavor',
exceptions.MissingArgs, "Missing option 'flavor'",
self.run_command, cmd)
def test_cluster_grow(self):
@ -301,7 +395,7 @@ class ShellTest(utils.TestCase):
'--instance flavor=2,volume=1,nic=net-id=some-id,'
'port-id=some-port-id,availability_zone=2')
self.assertRaisesRegexp(
exceptions.ValidationError, "Invalid 'nic' parameter. "
exceptions.ValidationError, "Invalid 'nic' option. "
"The value must be quoted.",
self.run_command, cmd)
@ -427,6 +521,91 @@ class ShellTest(utils.TestCase):
self.run_command('metadata-show 1234 key123')
self.assert_called('GET', '/instances/1234/metadata/key123')
def test_module_list(self):
self.run_command('module-list')
self.assert_called('GET', '/modules')
def test_module_list_datastore(self):
self.run_command('module-list --datastore all')
self.assert_called('GET', '/modules?datastore=all')
def test_module_show(self):
self.run_command('module-show 4321')
self.assert_called('GET', '/modules/4321')
def test_module_create(self):
with mock.patch.object(builtins, 'open'):
return_value = b'mycontents'
expected_contents = str(return_value.decode('utf-8'))
mock_encode = mock.Mock(return_value=return_value)
with mock.patch.object(base64, 'b64encode', mock_encode):
self.run_command('module-create mod1 type filename')
self.assert_called_anytime(
'POST', '/modules',
{'module': {'contents': expected_contents,
'all_tenants': 0,
'module_type': 'type', 'visible': 1,
'auto_apply': 0, 'live_update': 0,
'name': 'mod1'}})
def test_module_update(self):
with mock.patch.object(troveclient.v1.modules.Module, '__repr__',
mock.Mock(return_value='4321')):
self.run_command('module-update 4321 --name mod3')
self.assert_called_anytime(
'PUT', '/modules/4321',
{'module': {'name': 'mod3'}})
def test_module_delete(self):
with mock.patch.object(troveclient.v1.modules.Module, '__repr__',
mock.Mock(return_value='4321')):
self.run_command('module-delete 4321')
self.assert_called_anytime('DELETE', '/modules/4321')
def test_module_list_instance(self):
self.run_command('module-list-instance 1234')
self.assert_called_anytime('GET', '/instances/1234/modules')
def test_module_instances(self):
with mock.patch.object(troveclient.v1.modules.Module, '__repr__',
mock.Mock(return_value='4321')):
self.run_command('module-instances 4321')
self.assert_called_anytime('GET', '/modules/4321/instances')
def test_module_instances_clustered(self):
with mock.patch.object(troveclient.v1.modules.Module, '__repr__',
mock.Mock(return_value='4321')):
self.run_command('module-instances 4321 --include_clustered')
self.assert_called_anytime(
'GET', '/modules/4321/instances?include_clustered=True')
def test_cluster_modules(self):
self.run_command('cluster-modules cls-1234')
self.assert_called_anytime('GET', '/clusters/cls-1234')
def test_module_apply(self):
self.run_command('module-apply 1234 4321 8765')
self.assert_called_anytime('POST', '/instances/1234/modules',
{'modules':
[{'id': '4321'}, {'id': '8765'}]})
def test_module_remove(self):
self.run_command('module-remove 1234 4321')
self.assert_called_anytime('DELETE', '/instances/1234/modules/4321')
def test_module_query(self):
self.run_command('module-query 1234')
self.assert_called('GET', '/instances/1234/modules?from_guest=True')
def test_module_retrieve(self):
with mock.patch.object(troveclient.v1.modules.Module, '__getattr__',
mock.Mock(return_value='4321')):
self.run_command('module-retrieve 1234')
self.assert_called(
'GET',
'/instances/1234/modules?'
'include_contents=True&from_guest=True')
def test_limit_list(self):
self.run_command('limit-list')
self.assert_called('GET', '/limits')

View File

@ -23,6 +23,22 @@ AUTH_URL = "http://localhost:5002/auth_url"
AUTH_URL_V1 = "http://localhost:5002/auth_url/v1.0"
AUTH_URL_V2 = "http://localhost:5002/auth_url/v2.0"
URL_QUERY_SEPARATOR = '&'
URL_SEPARATOR = '?'
def order_url(url):
"""Returns the url with the query strings ordered, if they exist and
there's more than one. Otherwise the url is returned unaltered.
"""
if URL_QUERY_SEPARATOR in url:
parts = url.split(URL_SEPARATOR)
if len(parts) == 2:
queries = sorted(parts[1].split(URL_QUERY_SEPARATOR))
url = URL_SEPARATOR.join(
[parts[0], URL_QUERY_SEPARATOR.join(queries)])
return url
def _patch_mock_to_raise_for_invalid_assert_calls():
def raise_for_invalid_assert_calls(wrapped):

View File

@ -16,6 +16,7 @@
from __future__ import print_function
import base64
import os
import simplejson as json
import sys
@ -301,3 +302,25 @@ def is_uuid_like(val):
return str(uuid.UUID(val)) == val
except (TypeError, ValueError, AttributeError):
return False
def encode_data(data):
"""Encode the data using the base64 codec."""
try:
# py27str - if we've got text data, this should encode it
# py27aa/py34aa - if we've got a bytearray, this should work too
encoded = str(base64.b64encode(data).decode('utf-8'))
except TypeError:
# py34str - convert to bytes first, then we can encode
data_bytes = bytes([ord(item) for item in data])
encoded = base64.b64encode(data_bytes).decode('utf-8')
return encoded
def decode_data(data):
"""Encode the data using the base64 codec."""
# py27 & py34 seem to understand bytearray the same
return bytearray([item for item in base64.b64decode(data)])

View File

@ -15,12 +15,15 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import warnings
from troveclient import base
from troveclient import common
from troveclient import exceptions
from troveclient.i18n import _LW
from troveclient import utils
from troveclient.v1 import modules as core_modules
from swiftclient import client as swift_client
@ -85,7 +88,8 @@ class Instances(base.ManagerWithFind):
def create(self, name, flavor_id, volume=None, databases=None, users=None,
restorePoint=None, availability_zone=None, datastore=None,
datastore_version=None, nics=None, configuration=None,
replica_of=None, slave_of=None, replica_count=None):
replica_of=None, slave_of=None, replica_count=None,
modules=None):
"""Create (boot) a new instance."""
body = {"instance": {
@ -123,6 +127,8 @@ class Instances(base.ManagerWithFind):
body["instance"]["replica_of"] = base.getid(replica_of) or slave_of
if replica_count:
body["instance"]["replica_count"] = replica_count
if modules:
body["instance"]["modules"] = self._get_module_list(modules)
return self._create("/instances", body, "instance")
@ -248,6 +254,85 @@ class Instances(base.ManagerWithFind):
body = {'eject_replica_source': {}}
self._action(instance, body)
def modules(self, instance):
"""Get the list of modules for a specific instance."""
return self._modules_get(instance)
def module_query(self, instance):
"""Query an instance about installed modules."""
return self._modules_get(instance, from_guest=True)
def module_retrieve(self, instance, directory=None, prefix=None):
"""Retrieve the module data file from an instance. This includes
the contents of the module data file.
"""
if directory:
try:
os.makedirs(directory, exist_ok=True)
except TypeError:
# py27
try:
os.makedirs(directory)
except OSError:
if not os.path.isdir(directory):
raise
else:
directory = '.'
prefix = prefix or ''
if prefix and not prefix.endswith('_'):
prefix += '_'
module_list = self._modules_get(
instance, from_guest=True, include_contents=True)
saved_modules = {}
for module in module_list:
filename = '%s%s_%s_%s.dat' % (prefix, module.name,
module.datastore,
module.datastore_version)
full_filename = os.path.expanduser(
os.path.join(directory, filename))
with open(full_filename, 'wb') as fh:
fh.write(utils.decode_data(module.contents))
saved_modules[module.name] = full_filename
return saved_modules
def _modules_get(self, instance, from_guest=None, include_contents=None):
url = "/instances/%s/modules" % base.getid(instance)
query_strings = {}
if from_guest is not None:
query_strings["from_guest"] = from_guest
if include_contents is not None:
query_strings["include_contents"] = include_contents
url = common.append_query_strings(url, **query_strings)
resp, body = self.api.client.get(url)
common.check_for_exceptions(resp, body, url)
return [core_modules.Module(self, module, loaded=True)
for module in body['modules']]
def module_apply(self, instance, modules):
"""Apply modules to an instance."""
url = "/instances/%s/modules" % base.getid(instance)
body = {"modules": self._get_module_list(modules)}
resp, body = self.api.client.post(url, body=body)
common.check_for_exceptions(resp, body, url)
return [core_modules.Module(self, module, loaded=True)
for module in body['modules']]
def _get_module_list(self, modules):
"""Build a list of module ids."""
module_list = []
for module in modules:
module_info = {'id': base.getid(module)}
module_list.append(module_info)
return module_list
def module_remove(self, instance, module):
"""Remove a module from an instance.
"""
url = "/instances/%s/modules/%s" % (base.getid(instance),
base.getid(module))
resp, body = self.api.client.delete(url)
common.check_for_exceptions(resp, body, url)
def log_list(self, instance):
"""Get a list of all guest logs.

View File

@ -14,35 +14,40 @@
# under the License.
#
import base64
from troveclient import base
from troveclient import common
from troveclient import utils
class Module(base.Resource):
NO_CHANGE_TO_ARG = 'no_change_to_argument'
ALL_KEYWORD = 'all'
def __repr__(self):
return "<Module: %s>" % self.name
def __hash__(self):
return hash(repr(self))
def __eq__(self, other):
if isinstance(other, self.__class__):
return self.__dict__ == other.__dict__
else:
return False
class Modules(base.ManagerWithFind):
"""Manage :class:`Module` resources."""
resource_class = Module
def _encode_string(self, data_str):
byte_array = bytearray(data_str, 'utf-8')
return base64.b64encode(byte_array)
def create(self, name, module_type, contents, description=None,
all_tenants=None, datastore=None,
datastore_version=None, auto_apply=None,
visible=None, live_update=None):
"""Create a new module."""
contents = self._encode_string(contents)
contents = utils.encode_data(contents)
body = {"module": {
"name": name,
"module_type": module_type,
@ -86,7 +91,7 @@ class Modules(base.ManagerWithFind):
if module_type is not None:
body["module"]["type"] = module_type
if contents is not None:
contents = self._encode_string(contents)
contents = utils.encode_data(contents)
body["module"]["contents"] = contents
if description is not None:
body["module"]["description"] = description
@ -116,7 +121,7 @@ class Modules(base.ManagerWithFind):
"""Get a list of all modules."""
query_strings = None
if datastore:
query_strings = {"datastore": datastore}
query_strings = {"datastore": base.getid(datastore)}
return self._paginated(
"/modules", "modules", limit, marker, query_strings=query_strings)
@ -130,3 +135,13 @@ class Modules(base.ManagerWithFind):
url = "/modules/%s" % base.getid(module)
resp, body = self.api.client.delete(url)
common.check_for_exceptions(resp, body, url)
def instances(self, module, limit=None, marker=None,
include_clustered=False):
"""Get a list of all instances this module has been applied to."""
url = "/modules/%s/instances" % base.getid(module)
query_strings = {}
if include_clustered:
query_strings['include_clustered'] = include_clustered
return self._paginated(url, "instances", limit, marker,
query_strings=query_strings)

View File

@ -20,8 +20,9 @@ import argparse
import sys
import time
INSTANCE_METAVAR = '"opt=<value>[,opt=<value> ...] "'
INSTANCE_ERROR = ("Instance argument(s) must be of the form --instance "
"<opt=value[,opt=value]> - see help for details.")
+ INSTANCE_METAVAR + " - see help for details.")
NIC_ERROR = ("Invalid NIC argument: %s. Must specify either net-id or port-id "
"but not both. Please refer to help.")
NO_LOG_FOUND_ERROR = "ERROR: No published '%s' log was found for %s."
@ -33,6 +34,7 @@ except ImportError:
from troveclient import exceptions
from troveclient import utils
from troveclient.v1.modules import Module
def _poll_for_status(poll_fn, obj_id, action, final_ok_states,
@ -199,16 +201,22 @@ def do_flavor_show(cs, args):
help='Begin displaying the results for IDs greater than the '
'specified marker. When used with --limit, set this to '
'the last ID displayed in the previous run.')
@utils.arg('--include-clustered', dest='include_clustered',
@utils.arg('--include_clustered', '--include-clustered',
dest='include_clustered',
action="store_true", default=False,
help="Include instances that are part of a cluster "
"(default false).")
"(default %(default)s). --include-clustered may be "
"deprecated in the future, retaining just "
"--include_clustered.")
@utils.service_type('database')
def do_list(cs, args):
"""Lists all the instances."""
instances = cs.instances.list(limit=args.limit, marker=args.marker,
include_clustered=args.include_clustered)
_print_instances(instances)
def _print_instances(instances):
for instance in instances:
setattr(instance, 'flavor_id', instance.flavor['id'])
if hasattr(instance, 'volume'):
@ -287,22 +295,21 @@ def do_cluster_instances(cs, args):
obj_is_dict=True)
@utils.arg('--instance',
metavar="<name=name,flavor=flavor_name_or_id,volume=volume>",
action='append',
dest='instances',
default=[],
help="Add an instance to the cluster. Specify "
"multiple times to create multiple instances.")
@utils.arg('--instance', metavar=INSTANCE_METAVAR,
action='append', dest='instances', default=[],
help="Add an instance to the cluster. Specify multiple "
"times to create multiple instances. Valid options are: "
"name=<name>, flavor=<flavor_name_or_id>, volume=<volume>, "
"module=<module_name_or_id>.")
@utils.arg('cluster', metavar='<cluster>', help='ID or name of the cluster.')
@utils.service_type('database')
def do_cluster_grow(cs, args):
"""Adds more instances to a cluster."""
cluster = _find_cluster(cs, args.cluster)
instances = []
for instance_str in args.instances:
for instance_opts in args.instances:
instance_info = {}
for z in instance_str.split(","):
for z in instance_opts.split(","):
for (k, v) in [z.split("=", 1)[:2]]:
if k == "name":
instance_info[k] = v
@ -324,10 +331,7 @@ def do_cluster_grow(cs, args):
@utils.arg('cluster', metavar='<cluster>', help='ID or name of the cluster.')
@utils.arg('instances',
nargs='+',
metavar='<instance>',
default=[],
@utils.arg('instances', metavar='<instance>', nargs='+', default=[],
help="Drop instance(s) from the cluster. Specify "
"multiple ids to drop multiple instances.")
@utils.service_type('database')
@ -370,11 +374,13 @@ def do_cluster_delete(cs, args):
type=str,
default=None,
help='ID of the configuration reference to attach.')
@utils.arg('--detach-replica-source',
@utils.arg('--detach_replica_source', '--detach-replica-source',
dest='detach_replica_source',
action="store_true",
default=False,
help='Detach the replica instance from its replication source.')
help='Detach the replica instance from its replication source. '
'--detach-replica-source may be deprecated in the future '
'in favor of just --detach_replica_source')
@utils.arg('--remove_configuration',
dest='remove_configuration',
action="store_true",
@ -406,11 +412,11 @@ def do_update(cs, args):
@utils.arg('flavor',
metavar='<flavor>',
help='Flavor ID or name of the instance.')
@utils.arg('--databases', metavar='<databases>',
@utils.arg('--databases', metavar='<database>',
help='Optional list of databases.',
nargs="+", default=[])
@utils.arg('--users', metavar='<users>',
help='Optional list of users in the form user:password.',
@utils.arg('--users', metavar='<user:password>',
help='Optional list of users.',
nargs="+", default=[])
@utils.arg('--backup',
metavar='<backup>',
@ -419,7 +425,7 @@ def do_update(cs, args):
@utils.arg('--availability_zone',
metavar='<availability_zone>',
default=None,
help='The Zone hint to give to nova.')
help='The Zone hint to give to Nova.')
@utils.arg('--datastore',
metavar='<datastore>',
default=None,
@ -429,7 +435,8 @@ def do_update(cs, args):
default=None,
help='A datastore version name or ID.')
@utils.arg('--nic',
metavar="<net-id=net-uuid,v4-fixed-ip=ip-addr,port-id=port-uuid>",
metavar="<net-id=<net-uuid>,v4-fixed-ip=<ip-addr>,"
"port-id=<port-uuid>>",
action='append',
dest='nics',
default=[],
@ -452,7 +459,11 @@ def do_update(cs, args):
metavar='<count>',
type=int,
default=1,
help='Number of replicas to create (defaults to 1).')
help='Number of replicas to create (defaults to %(default)s).')
@utils.arg('--module', metavar='<module>',
type=str, dest='modules', action='append', default=[],
help='ID or name of the module to apply. Specify multiple '
'times to apply multiple modules.')
@utils.service_type('database')
def do_create(cs, args):
"""Creates a new instance."""
@ -476,6 +487,9 @@ def do_create(cs, args):
nic_str.split(",")]])
_validate_nic_info(nic_info, nic_str)
nics.append(nic_info)
modules = []
for module in args.modules:
modules.append(_find_module(cs, module).id)
instance = cs.instances.create(args.name,
flavor_id,
@ -489,7 +503,8 @@ def do_create(cs, args):
nics=nics,
configuration=args.configuration,
replica_of=replica_of_instance,
replica_count=args.replica_count)
replica_count=args.replica_count,
modules=modules)
_print_instance(instance)
@ -499,22 +514,26 @@ def _validate_nic_info(nic_info, nic_str):
raise exceptions.ValidationError(NIC_ERROR % ("nic='%s'" % nic_str))
def _get_flavors(cs, instance_str):
flavor_name = _get_instance_property(instance_str, 'flavor', True)
def _get_flavor(cs, opts_str):
flavor_name, opts_str = _strip_option(opts_str, 'flavor', True)
flavor_id = _find_flavor(cs, flavor_name).id
return str(flavor_id)
return str(flavor_id), opts_str
def _get_networks(instance_str):
nic_args = _dequote(_get_instance_property(instance_str, 'nic',
is_required=False, quoted=True))
nic_info = {}
if nic_args:
net_id = _get_instance_property(nic_args, 'net-id', False)
port_id = _get_instance_property(nic_args, 'port-id', False)
fixed_ipv4 = _get_instance_property(nic_args, 'v4-fixed-ip', False)
def _get_networks(opts_str):
nic_args_list, opts_str = _strip_option(opts_str, 'nic', is_required=False,
quotes_required=True,
allow_multiple=True)
nic_info_list = []
for nic_args in nic_args_list:
orig_nic_args = nic_args = _unquote(nic_args)
nic_info = {}
net_id, nic_args = _strip_option(nic_args, 'net-id', False)
port_id, nic_args = _strip_option(nic_args, 'port-id', False)
fixed_ipv4, nic_args = _strip_option(nic_args, 'v4-fixed-ip', False)
if nic_args:
raise exceptions.ValidationError(
"Unknown args '%s' in 'nic' option" % nic_args)
if net_id:
nic_info.update({'net-id': net_id})
if port_id:
@ -522,13 +541,13 @@ def _get_networks(instance_str):
if fixed_ipv4:
nic_info.update({'v4-fixed-ip': fixed_ipv4})
_validate_nic_info(nic_info, nic_args)
return [nic_info]
_validate_nic_info(nic_info, orig_nic_args)
nic_info_list.append(nic_info)
return None
return nic_info_list, opts_str
def _dequote(value):
def _unquote(value):
def _strip_quotes(value, quote_char):
if value:
return value.strip(quote_char)
@ -537,49 +556,86 @@ def _dequote(value):
return _strip_quotes(_strip_quotes(value, "'"), '"')
def _get_volumes(instance_str):
volume_size = _get_instance_property(instance_str, 'volume', True)
volume_type = _get_instance_property(instance_str, 'volume_type', False)
def _get_volume(opts_str):
volume_size, opts_str = _strip_option(opts_str, 'volume', is_required=True)
volume_type, opts_str = _strip_option(opts_str, 'volume_type',
is_required=False)
volume_info = {"size": volume_size}
if volume_type:
volume_info.update({"type": volume_type})
return volume_info
return volume_info, opts_str
def _get_availability_zones(instance_str):
return _get_instance_property(instance_str, 'availability_zone', False)
def _get_availability_zone(opts_str):
return _strip_option(opts_str, 'availability_zone', is_required=False)
def _get_instance_property(instance_str, property_name, is_required=True,
quoted=False):
if property_name in instance_str:
def _get_modules(cs, opts_str):
modules, opts_str = _strip_option(
opts_str, 'module', is_required=False, allow_multiple=True)
module_list = []
for module in modules:
module_info = {'id': _find_module(cs, module).id}
module_list.append(module_info)
return module_list, opts_str
def _strip_option(opts_str, opt_name, is_required=True,
quotes_required=False, allow_multiple=False):
opt_value = [] if allow_multiple else None
opts_str = opts_str.strip().strip(",")
if opt_name in opts_str:
try:
left = instance_str.split('%s=' % property_name)[1]
split_str = '%s=' % opt_name
parts = opts_str.split(split_str)
before = parts[0]
after = parts[1]
if len(parts) > 2:
if allow_multiple:
after = split_str.join(parts[1:])
value, after = _strip_option(
after, opt_name, is_required=is_required,
quotes_required=quotes_required,
allow_multiple=allow_multiple)
opt_value.extend(value)
else:
raise exceptions.ValidationError((
"Option '%s' found more than once in argument "
"--instance " % opt_name) + INSTANCE_METAVAR)
# Handle complex (quoted) properties. Strip the quotes.
quote = left[0]
quote = after[0]
if quote in ["'", '"']:
left = left[1:]
after = after[1:]
else:
if quoted:
# Fail if quotes are required.
if quotes_required:
raise exceptions.ValidationError(
"Invalid '%s' parameter. The value must be quoted."
% property_name)
"Invalid '%s' option. The value must be quoted. "
"(Or perhaps you're missing quotes around the entire "
"argument string)"
% opt_name)
quote = ''
property_value = left.split('%s,' % quote)[0]
return str(property_value).strip()
split_str = '%s,' % quote
parts = after.split(split_str)
value = str(parts[0]).strip()
if allow_multiple:
opt_value.append(value)
opt_value = list(set(opt_value))
else:
opt_value = value
opts_str = before + split_str.join(parts[1:])
except IndexError:
raise exceptions.ValidationError("Invalid '%s' parameter. %s."
% (property_name, INSTANCE_ERROR))
% (opt_name, INSTANCE_ERROR))
if is_required:
raise exceptions.MissingArgs([property_name])
if is_required and not opt_value:
msg = "Missing option '%s' for argument --instance " + INSTANCE_METAVAR
raise exceptions.MissingArgs([opt_name], message=msg)
return None
return opt_value, opts_str.strip().strip(",")
@utils.arg('name',
@ -592,35 +648,46 @@ def _get_instance_property(instance_str, property_name, is_required=True,
@utils.arg('datastore_version',
metavar='<datastore_version>',
help='A datastore version name or ID.')
@utils.arg('--instance',
metavar='"<opt=value,opt=value,...>"',
@utils.arg('--instance', metavar=INSTANCE_METAVAR,
action='append', dest='instances', default=[],
help="Create an instance for the cluster. Specify multiple "
"times to create multiple instances. "
"Valid options are: flavor=flavor_name_or_id, "
"volume=disk_size_in_GB, volume_type=type, "
"nic='net-id=net-uuid,v4-fixed-ip=ip-addr,port-id=port-uuid' "
"Valid options are: flavor=<flavor_name_or_id>, "
"volume=<disk_size_in_GB>, volume_type=<type>, "
"nic='<net-id=<net-uuid>, v4-fixed-ip=<ip-addr>, "
"port-id=<port-uuid>>' "
"(where net-id=network_id, v4-fixed-ip=IPv4r_fixed_address, "
"port-id=port_id), availability_zone=AZ_hint_for_Nova.",
action='append',
dest='instances',
default=[])
"port-id=port_id), availability_zone=<AZ_hint_for_Nova>, "
"module=<module_name_or_id>.")
@utils.service_type('database')
def do_cluster_create(cs, args):
"""Creates a new cluster."""
instances = []
for instance_str in args.instances:
for instance_opts in args.instances:
instance_info = {}
instance_info["flavorRef"] = _get_flavors(cs, instance_str)
instance_info["volume"] = _get_volumes(instance_str)
flavor, instance_opts = _get_flavor(cs, instance_opts)
instance_info["flavorRef"] = flavor
volume, instance_opts = _get_volume(instance_opts)
instance_info["volume"] = volume
nics = _get_networks(instance_str)
nics, instance_opts = _get_networks(instance_opts)
if nics:
instance_info["nics"] = nics
availability_zones = _get_availability_zones(instance_str)
if availability_zones:
instance_info["availability_zone"] = availability_zones
availability_zone, instance_opts = _get_availability_zone(
instance_opts)
if availability_zone:
instance_info["availability_zone"] = availability_zone
modules, instance_opts = _get_modules(cs, instance_opts)
if modules:
instance_info["modules"] = modules
if instance_opts:
raise exceptions.ValidationError(
"Unknown option(s) '%s' specified for instance" %
instance_opts)
instances.append(instance_info)
@ -1412,18 +1479,30 @@ def do_metadata_delete(cs, args):
@utils.arg('--datastore', metavar='<datastore>',
help='Name or ID of datastore to list modules for.')
help="Name or ID of datastore to list modules for. Use '%s' "
"to list modules that apply to all datastores." %
Module.ALL_KEYWORD)
@utils.service_type('database')
def do_module_list(cs, args):
"""Lists the modules available."""
datastore = None
if args.datastore:
datastore = _find_datastore(cs, args.datastore)
if args.datastore.lower() == Module.ALL_KEYWORD:
datastore = args.datastore.lower()
else:
datastore = _find_datastore(cs, args.datastore)
module_list = cs.modules.list(datastore=datastore)
field_list = ['id', 'name', 'type', 'datastore',
'datastore_version', 'auto_apply', 'tenant', 'visible']
is_admin = False
if hasattr(cs.client, 'auth'):
roles = cs.client.auth.auth_ref['user']['roles']
role_names = [role['name'] for role in roles]
is_admin = 'admin' in role_names
if not is_admin:
field_list = field_list[:-2]
utils.print_list(
module_list,
['id', 'tenant', 'name', 'type', 'datastore',
'datastore_version', 'auto_apply', 'visible'],
module_list, field_list,
labels={'datastore_version': 'Version'})
@ -1441,7 +1520,8 @@ def do_module_show(cs, args):
help='Type of the module. The type must be supported by a '
'corresponding module plugin on the datastore it is '
'applied to.')
@utils.arg('file', metavar='<filename>', type=argparse.FileType('rb', 0),
@utils.arg('file', metavar='<filename>',
type=argparse.FileType(mode='rb', bufsize=0),
help='File containing data contents for the module.')
@utils.arg('--description', metavar='<description>', type=str,
help='Description of the module.')
@ -1534,7 +1614,7 @@ def do_module_create(cs, args):
'already applied to a current instance or cluster.')
@utils.service_type('database')
def do_module_update(cs, args):
"""Create a module."""
"""Update a module."""
module = _find_module(cs, args.module)
contents = args.file.read() if args.file else None
visible = not args.hidden if args.hidden is not None else None
@ -1560,6 +1640,126 @@ def do_module_delete(cs, args):
cs.modules.delete(module)
@utils.arg('instance', metavar='<instance>', type=str,
help='ID or name of the instance.')
@utils.service_type('database')
def do_module_list_instance(cs, args):
"""Lists the modules that have been applied to an instance."""
instance = _find_instance(cs, args.instance)
module_list = cs.instances.modules(instance)
utils.print_list(
module_list, ['id', 'name', 'type', 'md5', 'created', 'updated'])
@utils.arg('module', metavar='<module>', type=str,
help='ID or name of the module.')
@utils.arg('--include_clustered', action="store_true", default=False,
help="Include instances that are part of a cluster "
"(default %(default)s).")
@utils.arg('--limit', metavar='<limit>', default=None,
help='Return up to N number of the most recent results.')
@utils.arg('--marker', metavar='<ID>', type=str, default=None,
help='Begin displaying the results for IDs greater than the '
'specified marker. When used with --limit, set this to '
'the last ID displayed in the previous run.')
@utils.service_type('database')
def do_module_instances(cs, args):
"""Lists the instances that have a particular module applied."""
module = _find_module(cs, args.module)
wrapper = cs.modules.instances(
module, limit=args.limit, marker=args.marker,
include_clustered=args.include_clustered)
instance_list = wrapper.items
while not args.limit and wrapper.next:
wrapper = cs.modules.instances(module, marker=wrapper.next)
instance_list += wrapper.items
_print_instances(instance_list)
@utils.arg('cluster', metavar='<cluster>', help='ID or name of the cluster.')
@utils.service_type('database')
def do_cluster_modules(cs, args):
"""Lists all modules for each instance of a cluster."""
cluster = _find_cluster(cs, args.cluster)
instances = cluster._info['instances']
module_list = []
for instance in instances:
new_list = cs.instances.modules(instance['id'])
for item in new_list:
item.instance_id = instance['id']
item.instance_name = instance['name']
module_list += new_list
utils.print_list(
module_list,
['instance_name', 'name', 'type', 'md5', 'created', 'updated'],
labels={'name': 'Module Name', 'type': 'Module Type'})
@utils.arg('instance', metavar='<instance>', type=str,
help='ID or name of the instance.')
@utils.arg('modules', metavar='<module>', type=str, nargs='+', default=[],
help='ID or name of the module.')
@utils.service_type('database')
def do_module_apply(cs, args):
"""Apply modules to an instance."""
instance = _find_instance(cs, args.instance)
modules = []
for module in args.modules:
modules.append(_find_module(cs, module))
result_list = cs.instances.module_apply(instance, modules)
utils.print_list(
result_list,
['name', 'type', 'datastore',
'datastore_version', 'status', 'message'],
labels={'datastore_version': 'Version'})
@utils.arg('instance', metavar='<instance>', type=str,
help='ID or name of the instance.')
@utils.arg('module', metavar='<module>', type=str,
help='ID or name of the module.')
@utils.service_type('database')
def do_module_remove(cs, args):
"""Remove a module from an instance."""
instance = _find_instance(cs, args.instance)
module = _find_module(cs, args.module)
cs.instances.module_remove(instance, module)
@utils.arg('instance', metavar='<instance>', type=str,
help='ID or name of the instance.')
@utils.service_type('database')
def do_module_query(cs, args):
"""Query the status of the modules on an instance."""
instance = _find_instance(cs, args.instance)
result_list = cs.instances.module_query(instance)
utils.print_list(
result_list,
['name', 'type', 'datastore',
'datastore_version', 'status', 'message', 'created', 'updated'],
labels={'datastore_version': 'Version'})
@utils.arg('instance', metavar='<instance>', type=str,
help='ID or name of the instance.')
@utils.arg('--directory', metavar='<directory>', type=str,
help='Directory to write module content files in. It will '
'be created if it does not exist. Defaults to the '
'current directory.')
@utils.arg('--prefix', metavar='<filename_prefix>', type=str,
help='Prefix to prepend to generated filename for each module.')
@utils.service_type('database')
def do_module_retrieve(cs, args):
"""Retrieve module contents from an instance."""
instance = _find_instance(cs, args.instance)
saved_modules = cs.instances.module_retrieve(
instance, args.directory, args.prefix)
for module_name, filename in saved_modules.items():
print("Module contents for '%s' written to '%s'" %
(module_name, filename))
@utils.arg('instance', metavar='<instance>',
help='Id or Name of the instance.')
@utils.service_type('database')