Add new CLI commands for managing cluster facts

This patch introduces the following commands to the new fuel2
CLI:

  - fuel2 env deployment-facts delete
  - fuel2 env deployment-facts download
  - fuel2 env deployment-facts get-default
  - fuel2 env deployment-facts upload
  - fuel2 env provisioning-facts delete
  - fuel2 env provisioning-facts download
  - fuel2 env provisioning-facts get-default
  - fuel2 env provisioning-facts upload

DocImpact

Change-Id: Ie8ed2b7df794cabbee5c1b6830769deba10e8b2e
This commit is contained in:
Vitalii Myhal 2016-08-09 11:44:19 +03:00
parent 5d3fa73d4e
commit a321ecaf71
6 changed files with 613 additions and 6 deletions

View File

@ -13,12 +13,15 @@
# under the License.
import abc
import argparse
import functools
import os
import shutil
import six
from cliff import show
from oslo_utils import fileutils
import six
from fuelclient.cli import error
from fuelclient.commands import base
@ -31,6 +34,42 @@ class EnvMixIn(object):
supported_file_formats = ('json', 'yaml')
allowed_attr_types = ('network', 'settings')
@staticmethod
def source_dir(directory):
"""Check that the source directory exists and is readable.
:param directory: Path to source directory
:type directory: str
:return: Absolute path to source directory
:rtype: str
"""
path = os.path.abspath(directory)
if not os.path.isdir(path):
raise argparse.ArgumentTypeError(
'"{0}" is not a directory.'.format(path))
if not os.access(path, os.R_OK):
raise argparse.ArgumentTypeError(
'directory "{0}" is not readable'.format(path))
return path
@staticmethod
def destination_dir(directory):
"""Check that the destination directory exists and is writable.
:param directory: Path to destination directory
:type directory: str
:return: Absolute path to destination directory
:rtype: str
"""
path = os.path.abspath(directory)
if not os.path.isdir(path):
raise argparse.ArgumentTypeError(
'"{0}" is not a directory.'.format(path))
if not os.access(path, os.W_OK):
raise argparse.ArgumentTypeError(
'directory "{0}" is not writable'.format(path))
return path
@six.add_metaclass(abc.ABCMeta)
class BaseUploadCommand(EnvMixIn, base.BaseCommand):
@ -552,3 +591,268 @@ class EnvSettingsDownload(BaseDownloadCommand):
@property
def downloader(self):
return self.client.get_settings
class FactsMixIn(object):
@staticmethod
def _get_fact_dir(env_id, fact_type, directory):
return os.path.join(directory, "{0}_{1}".format(fact_type, env_id))
@staticmethod
def _read_deployment_facts_from_file(directory, file_format):
return list(six.moves.map(
lambda f: data_utils.read_from_file(f),
[os.path.join(directory, file_name)
for file_name in os.listdir(directory)
if file_format == os.path.splitext(file_name)[1].lstrip('.')]
))
@staticmethod
def _read_provisioning_facts_from_file(directory, file_format):
node_facts = list(six.moves.map(
lambda f: data_utils.read_from_file(f),
[os.path.join(directory, file_name)
for file_name in os.listdir(directory)
if file_format == os.path.splitext(file_name)[1].lstrip('.')
and 'engine' != os.path.splitext(file_name)[0]]
))
engine_facts = None
engine_file = os.path.join(directory,
"{}.{}".format('engine', file_format))
if os.path.lexists(engine_file):
engine_facts = data_utils.read_from_file(engine_file)
return {'engine': engine_facts, 'nodes': node_facts}
@staticmethod
def _write_deployment_facts_to_file(facts, directory, file_format):
# from 9.0 the deployment info is serialized only per node
for _fact in facts:
file_name = "{role}_{uid}." if 'role' in _fact else "{uid}."
file_name += file_format
data_utils.write_to_file(
os.path.join(directory, file_name.format(**_fact)),
_fact)
@staticmethod
def _write_provisioning_facts_to_file(facts, directory, file_format):
file_name = "{uid}."
file_name += file_format
data_utils.write_to_file(
os.path.join(directory, file_name.format(uid='engine')),
facts['engine'])
for _fact in facts['nodes']:
data_utils.write_to_file(
os.path.join(directory, file_name.format(**_fact)),
_fact)
def download(self, env_id, fact_type, destination_dir, file_format,
nodes=None, default=False):
facts = self.client.download_facts(
env_id, fact_type, nodes=nodes, default=default)
if not facts:
raise error.ServerDataException(
"There are no {} facts for this environment!".format(
fact_type))
facts_dir = self._get_fact_dir(env_id, fact_type, destination_dir)
if os.path.exists(facts_dir):
shutil.rmtree(facts_dir)
os.makedirs(facts_dir)
getattr(self, "_write_{0}_facts_to_file".format(fact_type))(
facts, facts_dir, file_format)
return facts_dir
def upload(self, env_id, fact_type, source_dir, file_format):
facts_dir = self._get_fact_dir(env_id, fact_type, source_dir)
facts = getattr(self, "_read_{0}_facts_from_file".format(fact_type))(
facts_dir, file_format)
if not facts \
or isinstance(facts, dict) and not six.moves.reduce(
lambda a, b: a or b, facts.values()):
raise error.ServerDataException(
"There are no {} facts for this environment!".format(
fact_type))
return self.client.upload_facts(env_id, fact_type, facts)
class BaseEnvFactsDelete(EnvMixIn, base.BaseCommand):
"""Delete current various facts for orchestrator."""
fact_type = ''
def get_parser(self, prog_name):
parser = super(BaseEnvFactsDelete, self).get_parser(prog_name)
parser.add_argument(
'id',
type=int,
help='ID of the environment')
return parser
def take_action(self, parsed_args):
self.client.delete_facts(parsed_args.id, self.fact_type)
self.app.stdout.write(
"{0} facts for the environment {1} were deleted "
"successfully.\n".format(self.fact_type.capitalize(),
parsed_args.id)
)
class EnvDeploymentFactsDelete(BaseEnvFactsDelete):
"""Delete current deployment facts."""
fact_type = 'deployment'
class EnvProvisioningFactsDelete(BaseEnvFactsDelete):
"""Delete current provisioning facts."""
fact_type = 'provisioning'
class BaseEnvFactsDownload(FactsMixIn, EnvMixIn, base.BaseCommand):
"""Download various facts for orchestrator."""
fact_type = ''
fact_default = False
def get_parser(self, prog_name):
parser = super(BaseEnvFactsDownload, self).get_parser(prog_name)
parser.add_argument(
'-e', '--env',
type=int,
required=True,
help='ID of the environment')
parser.add_argument(
'-d', '--directory',
type=self.destination_dir,
default=os.path.curdir,
help='Path to directory to save {} facts. '
'Defaults to the current directory'.format(self.fact_type))
parser.add_argument(
'-n', '--nodes',
type=int,
nargs='+',
help='Get {} facts for nodes with given IDs'.format(
self.fact_type))
parser.add_argument(
'-f', '--format',
choices=self.supported_file_formats,
required=True,
help='Format of serialized {} facts'.format(self.fact_type))
return parser
def take_action(self, parsed_args):
facts_dir = self.download(
parsed_args.env,
self.fact_type,
parsed_args.directory,
parsed_args.format,
nodes=parsed_args.nodes,
default=self.fact_default
)
self.app.stdout.write(
"{0} {1} facts for the environment {2} "
"were downloaded to {3}\n".format(
'Default' if self.fact_default else 'User-defined',
self.fact_type,
parsed_args.env,
facts_dir)
)
class EnvDeploymentFactsDownload(BaseEnvFactsDownload):
"""Download the user-defined deployment facts."""
fact_type = 'deployment'
fact_default = False
class EnvDeploymentFactsGetDefault(BaseEnvFactsDownload):
"""Download the default deployment facts."""
fact_type = 'deployment'
fact_default = True
class EnvProvisioningFactsDownload(BaseEnvFactsDownload):
"""Download the user-defined provisioning facts."""
fact_type = 'provisioning'
fact_default = False
class EnvProvisioningFactsGetDefault(BaseEnvFactsDownload):
"""Download the default provisioning facts."""
fact_type = 'provisioning'
fact_default = True
class BaseEnvFactsUpload(FactsMixIn, EnvMixIn, base.BaseCommand):
"""Upload various facts for orchestrator."""
fact_type = ''
def get_parser(self, prog_name):
parser = super(BaseEnvFactsUpload, self).get_parser(prog_name)
parser.add_argument(
'-e', '--env',
type=int,
required=True,
help='ID of the environment')
parser.add_argument(
'-d', '--directory',
type=self.source_dir,
default=os.path.curdir,
help='Path to directory to read {} facts. '
'Defaults to the current directory'.format(self.fact_type))
parser.add_argument(
'-f', '--format',
choices=self.supported_file_formats,
required=True,
help='Format of serialized {} facts'.format(self.fact_type))
return parser
def take_action(self, parsed_args):
self.upload(
parsed_args.env,
self.fact_type,
parsed_args.directory,
parsed_args.format
)
self.app.stdout.write(
"{0} facts for the environment {1} were uploaded "
"successfully.\n".format(self.fact_type.capitalize(),
parsed_args.env)
)
class EnvDeploymentFactsUpload(BaseEnvFactsUpload):
"""Upload deployment facts."""
fact_type = 'deployment'
class EnvProvisioningFactsUpload(BaseEnvFactsUpload):
"""Upload provisioning facts."""
fact_type = 'provisioning'

View File

@ -13,6 +13,7 @@
# under the License.
import json
import os
import yaml
from fuelclient.cli import error
@ -56,7 +57,7 @@ def safe_load(data_format, stream):
def safe_dump(data_format, stream, data):
# The reason these dumpers are assigned to indivisual variables is
# The reason these dumpers are assigned to individual variables is
# making PEP8 check happy.
yaml_dumper = lambda data, stream: yaml.safe_dump(data,
stream,
@ -70,3 +71,15 @@ def safe_dump(data_format, stream, data):
dumper = dumpers[data_format]
dumper(data, stream)
def read_from_file(file_path):
data_format = os.path.splitext(file_path)[1].lstrip('.')
with open(file_path, 'r') as stream:
return safe_load(data_format, stream)
def write_to_file(file_path, data):
data_format = os.path.splitext(file_path)[1].lstrip('.')
with open(file_path, 'w') as stream:
safe_dump(data_format, stream, data)

View File

@ -17,9 +17,10 @@
import json
import mock
from six import moves
import yaml
from six import moves
from fuelclient.tests.unit.v2.cli import test_engine
from fuelclient.tests.utils import fake_env
from fuelclient.tests.utils import fake_task
@ -382,3 +383,218 @@ class TestEnvCommand(test_engine.BaseCLITest):
self.m_client.set_settings.assert_called_once_with(42,
config,
force=True)
def test_env_deployment_facts_delete(self):
args = "env deployment-facts delete 42"
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.delete_facts.assert_called_once_with(42, 'deployment')
def test_env_provisioning_facts_delete(self):
args = "env provisioning-facts delete 42"
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.delete_facts.assert_called_once_with(42, 'provisioning')
@mock.patch('json.dump')
def _deployment_facts_download_json(self, m_dump, default=False):
command = 'get-default' if default else 'download'
args = "env deployment-facts {}" \
" --env 42 --dir /tmp --nodes 2 --format json".format(command)
data = [{'uid': 2, 'name': 'node'}]
expected_path = '/tmp/deployment_42/2.json'
self.m_client.download_facts.return_value = data
m_open = mock.mock_open()
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.download_facts.assert_called_once_with(
42, 'deployment', nodes=[2], default=default)
m_open.assert_called_once_with(expected_path, 'w')
m_dump.assert_called_once_with(data[0], mock.ANY, indent=4)
def test_env_deployment_facts_download_json(self):
self._deployment_facts_download_json(default=False)
def test_env_deployment_facts_get_default_json(self):
self._deployment_facts_download_json(default=True)
@mock.patch('yaml.safe_dump')
def _deployment_facts_download_yaml(self, m_safe_dump, default=False):
command = 'get-default' if default else 'download'
args = "env deployment-facts {}" \
" --env 42 --dir /tmp --nodes 2 --format yaml".format(command)
data = [{'uid': 2, 'name': 'node'}]
expected_path = '/tmp/deployment_42/2.yaml'
self.m_client.download_facts.return_value = data
m_open = mock.mock_open()
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.download_facts.assert_called_once_with(
42, 'deployment', nodes=[2], default=default)
m_open.assert_called_once_with(expected_path, 'w')
m_safe_dump.assert_called_once_with(data[0], mock.ANY,
default_flow_style=False)
def test_env_deployment_facts_download_yaml(self):
self._deployment_facts_download_yaml(default=False)
def test_env_deployment_facts_get_default_yaml(self):
self._deployment_facts_download_yaml(default=True)
def test_env_deployment_facts_upload_json(self):
args = 'env deployment-facts upload --env 42 --dir /tmp --format json'
data = [{'uid': 2, 'name': 'node'}]
expected_path = '/tmp/deployment_42/2.json'
m_open = mock.mock_open(read_data=json.dumps(data[0]))
with mock.patch('os.listdir', return_value=['2.json']):
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
m_open.assert_called_once_with(expected_path, 'r')
self.m_client.upload_facts.assert_called_once_with(
42, 'deployment', data)
def test_env_deployment_facts_upload_yaml(self):
args = 'env deployment-facts upload --env 42 --dir /tmp --format yaml'
data = [{'uid': 2, 'name': 'node'}]
expected_path = '/tmp/deployment_42/2.yaml'
m_open = mock.mock_open(read_data=yaml.dump(data[0]))
with mock.patch('os.listdir', return_value=['2.yaml']):
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
m_open.assert_called_once_with(expected_path, 'r')
self.m_client.upload_facts.assert_called_once_with(
42, 'deployment', data)
@mock.patch('json.dump')
def _provisioning_facts_download_json(self, m_dump, default=False):
command = 'get-default' if default else 'download'
args = "env provisioning-facts {}" \
" --env 42 --dir /tmp --nodes 2 --format json".format(command)
data = {
'engine': {'foo': 'bar'},
'nodes': [{'uid': 2, 'name': 'node-2'}]
}
expected_path_engine = '/tmp/provisioning_42/engine.json'
expected_path_node = '/tmp/provisioning_42/2.json'
self.m_client.download_facts.return_value = data
m_open = mock.mock_open()
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.download_facts.assert_called_once_with(
42, 'provisioning', nodes=[2], default=default)
m_open.assert_any_call(expected_path_engine, 'w')
m_dump.assert_any_call(data['engine'], mock.ANY, indent=4)
m_open.assert_any_call(expected_path_node, 'w')
m_dump.assert_any_call(data['nodes'][0], mock.ANY, indent=4)
def test_env_provisioning_facts_download_json(self):
self._provisioning_facts_download_json(default=False)
def test_env_provisioning_facts_get_default_json(self):
self._provisioning_facts_download_json(default=True)
@mock.patch('yaml.safe_dump')
def _provisioning_facts_download_yaml(self, m_dump, default=False):
command = 'get-default' if default else 'download'
args = "env provisioning-facts {}" \
" --env 42 --dir /tmp --nodes 2 --format yaml".format(command)
data = {
'engine': {'foo': 'bar'},
'nodes': [{'uid': 2, 'name': 'node-2'}]
}
expected_path_engine = '/tmp/provisioning_42/engine.yaml'
expected_path_node = '/tmp/provisioning_42/2.yaml'
self.m_client.download_facts.return_value = data
m_open = mock.mock_open()
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
self.m_client.download_facts.assert_called_once_with(
42, 'provisioning', nodes=[2], default=default)
m_open.assert_any_call(expected_path_engine, 'w')
m_dump.assert_any_call(data['engine'], mock.ANY,
default_flow_style=False)
m_open.assert_any_call(expected_path_node, 'w')
m_dump.assert_any_call(data['nodes'][0], mock.ANY,
default_flow_style=False)
def test_env_provisioning_facts_download_yaml(self):
self._provisioning_facts_download_yaml(default=False)
def test_env_provisioning_facts_get_default_yaml(self):
self._provisioning_facts_download_yaml(default=True)
def test_env_provisioning_facts_upload_json(self):
args = 'env provisioning-facts upload' \
' --env 42 --dir /tmp --format json'
expected_data = {
'engine': {'foo': 'bar'},
'nodes': [{'foo': 'bar'}]
}
expected_path_engine = '/tmp/provisioning_42/engine.json'
expected_path_node = '/tmp/provisioning_42/2.json'
m_open = mock.mock_open(read_data=json.dumps({'foo': 'bar'}))
with mock.patch('os.listdir', return_value=['engine.json', '2.json']):
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
with mock.patch('os.path.lexists', return_value=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
m_open.assert_any_call(expected_path_engine, 'r')
m_open.assert_any_call(expected_path_node, 'r')
self.m_client.upload_facts.assert_called_once_with(
42, 'provisioning', expected_data)
def test_env_provisioning_facts_upload_yaml(self):
args = 'env provisioning-facts upload' \
' --env 42 --dir /tmp --format yaml'
expected_data = {
'engine': {'foo': 'bar'},
'nodes': [{'foo': 'bar'}]
}
expected_path_engine = '/tmp/provisioning_42/engine.yaml'
expected_path_node = '/tmp/provisioning_42/2.yaml'
m_open = mock.mock_open(read_data=json.dumps({'foo': 'bar'}))
with mock.patch('os.listdir', return_value=['engine.yaml', '2.yaml']):
with mock.patch('fuelclient.common.data_utils.open',
m_open, create=True):
with mock.patch('os.path.lexists', return_value=True):
self.exec_command(args)
self.m_get_client.assert_called_once_with('environment', mock.ANY)
m_open.assert_any_call(expected_path_engine, 'r')
m_open.assert_any_call(expected_path_node, 'r')
self.m_client.upload_facts.assert_called_once_with(
42, 'provisioning', expected_data)

View File

@ -392,3 +392,48 @@ class TestEnvFacade(test_api.BaseLibTest):
self.assertTrue(m_upload.called)
self.assertEqual(test_settings, m_upload.last_request.json())
self.assertEqual(['1'], m_upload.last_request.qs.get('force'))
def test_delete_facts(self):
env_id = 42
fact_type = 'deployment'
expected_uri = self.get_object_uri(
self.res_uri,
env_id,
'/orchestrator/{fact_type}'.format(fact_type=fact_type))
matcher = self.m_request.delete(expected_uri, json={})
self.client.delete_facts(env_id, fact_type)
self.assertTrue(matcher.called)
self.assertIsNone(matcher.last_request.body)
def test_download_facts(self):
env_id = 42
fact_type = 'deployment'
nodes = [2, 5]
expected_uri = self.get_object_uri(
self.res_uri,
env_id,
"/orchestrator/{fact_type}/?nodes={nodes}".format(
fact_type=fact_type, nodes=",".join(map(str, nodes))))
fake_resp = {'foo': 'bar'}
matcher = self.m_request.get(expected_uri, json=fake_resp)
facts = self.client.download_facts(
env_id, fact_type, nodes=nodes, default=False)
self.assertTrue(matcher.called)
self.assertIsNone(matcher.last_request.body)
self.assertEqual(facts, fake_resp)
def test_upload_facts(self):
env_id = 42
fact_type = 'deployment'
facts = {'foo': 'bar'}
expected_uri = self.get_object_uri(
self.res_uri,
env_id,
"/orchestrator/{fact_type}".format(fact_type=fact_type))
matcher = self.m_request.put(expected_uri, json={})
self.client.upload_facts(env_id, fact_type, facts)
self.assertTrue(matcher.called)
self.assertEqual(facts, matcher.last_request.json())

View File

@ -20,7 +20,6 @@ from fuelclient.v1 import base_v1
class EnvironmentClient(base_v1.BaseV1Client):
_entity_wrapper = objects.Environment
_updatable_attributes = ('name',)
provision_nodes_url = 'clusters/{env_id}/provision/?nodes={nodes}'
@ -160,6 +159,28 @@ class EnvironmentClient(base_v1.BaseV1Client):
env = self._entity_wrapper(environment_id)
env.set_settings_data(new_configuration, force=force)
@staticmethod
def _get_fact_url(env_id, fact_type, nodes=None, default=False):
return "clusters/{id}/orchestrator/{fact_type}{default}{nodes}".format(
id=env_id,
fact_type=fact_type,
default="/defaults" if default else '',
nodes="/?nodes={}".format(
",".join(map(str, nodes))) if nodes else ''
)
def delete_facts(self, env_id, fact_type):
return self.connection.delete_request(
self._get_fact_url(env_id, fact_type))
def download_facts(self, env_id, fact_type, nodes=None, default=False):
return self.connection.get_request(self._get_fact_url(
env_id, fact_type, nodes=nodes, default=default))
def upload_facts(self, env_id, fact_type, facts):
return self.connection.put_request(
self._get_fact_url(env_id, fact_type), facts)
def get_client(connection):
return EnvironmentClient(connection)

View File

@ -33,12 +33,20 @@ fuelclient =
env_create=fuelclient.commands.environment:EnvCreate
env_delete=fuelclient.commands.environment:EnvDelete
env_deploy=fuelclient.commands.environment:EnvDeploy
env_nodes_deploy=fuelclient.commands.environment:EnvDeployNodes
env_deployment-facts_delete=fuelclient.commands.environment:EnvDeploymentFactsDelete
env_deployment-facts_download=fuelclient.commands.environment:EnvDeploymentFactsDownload
env_deployment-facts_get-default=fuelclient.commands.environment:EnvDeploymentFactsGetDefault
env_deployment-facts_upload=fuelclient.commands.environment:EnvDeploymentFactsUpload
env_list=fuelclient.commands.environment:EnvList
env_nodes_provision=fuelclient.commands.environment:EnvProvisionNodes
env_network_download=fuelclient.commands.environment:EnvNetworkDownload
env_network_upload=fuelclient.commands.environment:EnvNetworkUpload
env_network_verify=fuelclient.commands.environment:EnvNetworkVerify
env_nodes_deploy=fuelclient.commands.environment:EnvDeployNodes
env_nodes_provision=fuelclient.commands.environment:EnvProvisionNodes
env_provisioning-facts_delete=fuelclient.commands.environment:EnvProvisioningFactsDelete
env_provisioning-facts_download=fuelclient.commands.environment:EnvProvisioningFactsDownload
env_provisioning-facts_get-default=fuelclient.commands.environment:EnvProvisioningFactsGetDefault
env_provisioning-facts_upload=fuelclient.commands.environment:EnvProvisioningFactsUpload
env_redeploy=fuelclient.commands.environment:EnvRedeploy
env_remove_nodes=fuelclient.commands.environment:EnvRemoveNodes
env_settings_download=fuelclient.commands.environment:EnvSettingsDownload