From b2ca94629615208a89d7be5d065d6e17675a6608 Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 22 Nov 2018 01:43:38 +0400 Subject: [PATCH] Do not link with Ansible code Ansible is distributed under GPL-3.0 license and certain restrictions are applied when its code is imported as Python library. The only safe way to call GPL code is via general interface, e.g. CLI. This patch removes all direct linking of Ansible code and executes all actions via command line. It is now user responsibility to install Ansible executable on the system. Change-Id: If879e4ce59bcdac84bc51ea0ac9277783777c80b --- README.rst | 26 +++ os_faults/ansible/executor.py | 164 ++++++++---------- os_faults/tests/unit/ansible/test_executor.py | 143 --------------- requirements.txt | 2 +- test-requirements.txt | 3 + 5 files changed, 101 insertions(+), 237 deletions(-) diff --git a/README.rst b/README.rst index c4ef2b5..af00413 100644 --- a/README.rst +++ b/README.rst @@ -18,6 +18,13 @@ IPMI driver, Universal driver). Installation ------------ +Requirements +~~~~~~~~~~~~ + +Ansible is required and should be installed manually system-wide or in virtual +environment. Please refer to [https://docs.ansible.com/ansible/latest/installation_guide/intro_installation.html] +for installation instructions. + Regular installation:: pip install os-faults @@ -291,3 +298,22 @@ Terminate a container on a random node: container = cloud_management.get_container(name='neutron_ovs_agent') nodes = container.get_nodes().pick() container.restart(nodes) + + +License notes +------------- + +Ansible is distributed under GPL-3.0 license and thus all programs +that link with its code are subject to GPL restrictions [1]. +However these restrictions are not applied to os-faults library +since it invokes Ansible as process [2][3]. + +Ansible modules are provided with Apache license (compatible to GPL) [4]. +Those modules import part of Ansible runtime (modules API) and executed +on remote hosts. os-faults library does not import these module +neither static nor dynamic. + + [1] https://www.gnu.org/licenses/gpl-faq.html#GPLModuleLicense + [2] https://www.gnu.org/licenses/gpl-faq.html#GPLPlugins + [3] https://www.gnu.org/licenses/gpl-faq.html#MereAggregation + [4] https://www.apache.org/licenses/GPL-compatibility.html diff --git a/os_faults/ansible/executor.py b/os_faults/ansible/executor.py index e7e57d7..389273f 100644 --- a/os_faults/ansible/executor.py +++ b/os_faults/ansible/executor.py @@ -11,25 +11,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import print_function + import collections import copy +import json import logging import os +import shlex +import tempfile -from ansible.executor import task_queue_manager -from ansible.parsing import dataloader -from ansible.playbook import play -from ansible.plugins import callback as callback_pkg - -try: - from ansible.inventory.manager import InventoryManager as Inventory - from ansible.vars.manager import VariableManager - PRE_24_ANSIBLE = False -except ImportError: - # pre-2.4 Ansible - from ansible.inventory import Inventory - from ansible.vars import VariableManager - PRE_24_ANSIBLE = True +from oslo_concurrency import processutils +import yaml from os_faults.api import error @@ -61,37 +54,13 @@ AnsibleExecutionRecord = collections.namedtuple( 'AnsibleExecutionRecord', ['host', 'status', 'task', 'payload']) -class MyCallback(callback_pkg.CallbackBase): - - CALLBACK_VERSION = 2.0 - CALLBACK_TYPE = 'stdout' - CALLBACK_NAME = 'myown' - - def __init__(self, storage, display=None): - super(MyCallback, self).__init__(display) - self.storage = storage - - def _store(self, result, status): - record = AnsibleExecutionRecord( - host=result._host.get_name(), status=status, - task=result._task.get_name(), payload=result._result) - self.storage.append(record) - - def v2_runner_on_failed(self, result, ignore_errors=False): - super(MyCallback, self).v2_runner_on_failed(result) - self._store(result, STATUS_FAILED) - - def v2_runner_on_ok(self, result): - super(MyCallback, self).v2_runner_on_ok(result) - self._store(result, STATUS_OK) - - def v2_runner_on_skipped(self, result): - super(MyCallback, self).v2_runner_on_skipped(result) - self._store(result, STATUS_SKIPPED) - - def v2_runner_on_unreachable(self, result): - super(MyCallback, self).v2_runner_on_unreachable(result) - self._store(result, STATUS_UNREACHABLE) +def find_ansible(): + stdout, stderr = processutils.execute( + *shlex.split('which ansible-playbook'), check_exit_code=[0, 1]) + if not stdout: + raise AnsibleExecutionException( + 'Ansible executable is not found in $PATH') + return stdout[:-1] def resolve_relative_path(file_name): @@ -122,14 +91,10 @@ def add_module_paths(paths): def make_module_path_option(): - if PRE_24_ANSIBLE: - # it was a string of colon-separated directories - module_path = os.pathsep.join(get_module_paths()) - else: - # now it is a list of strings (MUST have > 1 element) - module_path = list(get_module_paths()) - if len(module_path) == 1: - module_path += [module_path[0]] + # now it is a list of strings (MUST have > 1 element) + module_path = list(get_module_paths()) + if len(module_path) == 1: + module_path += [module_path[0]] return module_path @@ -166,6 +131,7 @@ class AnsibleRunner(object): become=become, become_method='sudo', become_user='root', verbosity=100, check=False, diff=None) self.serial = serial or 10 + self.ansible = find_ansible() @staticmethod def _build_proxy_arg(jump_user, jump_host, private_key_file=None): @@ -176,55 +142,67 @@ class AnsibleRunner(object): host=jump_host, ssh_args=SSH_COMMON_ARGS)) def _run_play(self, play_source, host_vars): - host_list = play_source['hosts'] - - loader = dataloader.DataLoader() - - # FIXME(jpena): we need to behave differently if we are using - # Ansible >= 2.4.0.0. Remove when only versions > 2.4 are supported - if PRE_24_ANSIBLE: - variable_manager = VariableManager() - inventory_inst = Inventory(loader=loader, - variable_manager=variable_manager, - host_list=host_list) - variable_manager.set_inventory(inventory_inst) - else: - inventory_inst = Inventory(loader=loader, - sources=','.join(host_list) + ',') - variable_manager = VariableManager(loader=loader, - inventory=inventory_inst) + inventory = {} for host, variables in host_vars.items(): - host_inst = inventory_inst.get_host(host) + host_vars = {} + for var_name, value in variables.items(): if value is not None: - variable_manager.set_host_variable( - host_inst, var_name, value) + host_vars[var_name] = value + inventory[host] = host_vars - storage = [] - callback = MyCallback(storage) + inventory[host]['ansible_ssh_common_args'] = ( + self.options.ssh_common_args) + inventory[host]['ansible_connection'] = self.options.connection - tqm = task_queue_manager.TaskQueueManager( - inventory=inventory_inst, - variable_manager=variable_manager, - loader=loader, - options=self.options, - passwords=self.passwords, - stdout_callback=callback, - ) + full_inventory = {'all': {'hosts': inventory}} - # create play - play_inst = play.Play().load(play_source, - variable_manager=variable_manager, - loader=loader) + temp_dir = tempfile.mkdtemp(prefix='os-faults') + inventory_file_name = os.path.join(temp_dir, 'inventory') + playbook_file_name = os.path.join(temp_dir, 'playbook') - # actually run it - try: - tqm.run(play_inst) - finally: - tqm.cleanup() + with open(inventory_file_name, 'w') as fd: + print(yaml.safe_dump(full_inventory, default_flow_style=False), + file=fd) - return storage + play = { + 'hosts': 'all', + 'gather_facts': 'no', + 'tasks': play_source['tasks'], + } + + with open(playbook_file_name, 'w') as fd: + print(yaml.safe_dump([play], default_flow_style=False), file=fd) + + cmd = ('%(ansible)s --module-path %(module_path)s ' + '-i %(inventory)s %(playbook)s' % + {'ansible': self.ansible, + 'module_path': ':'.join(self.options.module_path), + 'inventory': inventory_file_name, + 'playbook': playbook_file_name}) + + logging.info('Executing %s' % cmd) + command_stdout, command_stderr = processutils.execute( + *shlex.split(cmd), + env_variables={'ANSIBLE_STDOUT_CALLBACK': 'json'}, + check_exit_code=False) + + d = json.loads(command_stdout[command_stdout.find('{'):]) + h = d['plays'][0]['tasks'][0]['hosts'] + recs = [] + for h, hv in h.items(): + if hv.get('unreachable'): + status = STATUS_UNREACHABLE + elif hv.get('failed'): + status = STATUS_FAILED + else: + status = STATUS_OK + r = AnsibleExecutionRecord(host=h, status=status, task='', + payload=hv) + recs.append(r) + + return recs def run_playbook(self, playbook, host_vars): result = [] diff --git a/os_faults/tests/unit/ansible/test_executor.py b/os_faults/tests/unit/ansible/test_executor.py index 3b19384..236f10c 100644 --- a/os_faults/tests/unit/ansible/test_executor.py +++ b/os_faults/tests/unit/ansible/test_executor.py @@ -19,87 +19,6 @@ from os_faults.api import node_collection from os_faults.tests.unit import test -class MyCallbackTestCase(test.TestCase): - - def test__store(self,): - ex = executor.MyCallback(mock.Mock()) - - my_host = 'my_host' - my_task = 'my_task' - my_result = 'my_result' - r = mock.Mock() - r._host.get_name.return_value = my_host - r._task.get_name.return_value = my_task - r._result = my_result - stat = 'OK' - - ex._store(r, stat) - ex.storage.append.assert_called_once_with( - executor.AnsibleExecutionRecord(host=my_host, status=stat, - task=my_task, payload=my_result)) - - @mock.patch('ansible.plugins.callback.CallbackBase.v2_runner_on_failed') - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_failed_super(self, mock_store, mock_callback): - ex = executor.MyCallback(mock.Mock()) - result = mock.Mock() - ex.v2_runner_on_failed(result) - mock_callback.assert_called_once_with(result) - - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_failed(self, mock_store): - result = mock.Mock() - ex = executor.MyCallback(mock.Mock()) - ex.v2_runner_on_failed(result) - mock_store.assert_called_once_with(result, executor.STATUS_FAILED) - - @mock.patch('ansible.plugins.callback.CallbackBase.v2_runner_on_ok') - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_ok_super(self, mock_store, mock_callback): - ex = executor.MyCallback(mock.Mock()) - result = mock.Mock() - ex.v2_runner_on_ok(result) - mock_callback.assert_called_once_with(result) - - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_ok(self, mock_store): - result = mock.Mock() - ex = executor.MyCallback(mock.Mock()) - ex.v2_runner_on_ok(result) - mock_store.assert_called_once_with(result, executor.STATUS_OK) - - @mock.patch('ansible.plugins.callback.CallbackBase.v2_runner_on_skipped') - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_skipped_super(self, mock_store, mock_callback): - ex = executor.MyCallback(mock.Mock()) - result = mock.Mock() - ex.v2_runner_on_skipped(result) - mock_callback.assert_called_once_with(result) - - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_skipped(self, mock_store): - result = mock.Mock() - ex = executor.MyCallback(mock.Mock()) - ex.v2_runner_on_skipped(result) - mock_store.assert_called_once_with(result, executor.STATUS_SKIPPED) - - @mock.patch( - 'ansible.plugins.callback.CallbackBase.v2_runner_on_unreachable') - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_unreachable_super(self, mock_store, mock_callback): - ex = executor.MyCallback(mock.Mock()) - result = mock.Mock() - ex.v2_runner_on_unreachable(result) - mock_callback.assert_called_once_with(result) - - @mock.patch('os_faults.ansible.executor.MyCallback._store') - def test_v2_runner_on_unreachable(self, mock_store): - result = mock.Mock() - ex = executor.MyCallback(mock.Mock()) - ex.v2_runner_on_unreachable(result) - mock_store.assert_called_once_with(result, executor.STATUS_UNREACHABLE) - - @ddt.ddt class AnsibleRunnerTestCase(test.TestCase): @@ -191,59 +110,6 @@ class AnsibleRunnerTestCase(test.TestCase): **options_args) self.assertEqual(passwords, runner.passwords) - @mock.patch.object(executor.task_queue_manager, 'TaskQueueManager') - @mock.patch('ansible.playbook.play.Play.load') - @mock.patch('os_faults.ansible.executor.Inventory') - @mock.patch('os_faults.ansible.executor.VariableManager') - @mock.patch('ansible.parsing.dataloader.DataLoader') - def test__run_play(self, mock_dataloader, mock_vmanager, mock_inventory, - mock_play_load, mock_taskqm): - mock_play_load.return_value = 'my_load' - variable_manager = mock_vmanager.return_value - host_inst = mock_inventory.return_value.get_host.return_value - host_vars = { - '0.0.0.0': { - 'ansible_user': 'foo', - 'ansible_ssh_pass': 'bar', - 'ansible_become': True, - 'ansible_ssh_private_key_file': None, - 'ansible_ssh_common_args': '-o Option=yes', - } - } - ex = executor.AnsibleRunner() - ex._run_play({'hosts': ['0.0.0.0']}, host_vars) - - mock_taskqm.assert_called_once() - self.assertEqual(mock_taskqm.mock_calls[1], mock.call().run('my_load')) - self.assertEqual(mock_taskqm.mock_calls[2], mock.call().cleanup()) - - variable_manager.set_host_variable.assert_has_calls(( - mock.call(host_inst, 'ansible_user', 'foo'), - mock.call(host_inst, 'ansible_ssh_pass', 'bar'), - mock.call(host_inst, 'ansible_become', True), - mock.call(host_inst, 'ansible_ssh_common_args', '-o Option=yes'), - ), any_order=True) - - @mock.patch.object(executor.task_queue_manager, 'TaskQueueManager') - @mock.patch('ansible.playbook.play.Play.load') - @mock.patch('os_faults.ansible.executor.Inventory') - @mock.patch('os_faults.ansible.executor.VariableManager') - @mock.patch('ansible.parsing.dataloader.DataLoader') - def test__run_play_no_host_vars( - self, mock_dataloader, mock_vmanager, mock_inventory, - mock_play_load, mock_taskqm): - mock_play_load.return_value = 'my_load' - variable_manager = mock_vmanager.return_value - host_vars = {} - ex = executor.AnsibleRunner() - ex._run_play({'hosts': ['0.0.0.0']}, host_vars) - - mock_taskqm.assert_called_once() - self.assertEqual(mock_taskqm.mock_calls[1], mock.call().run('my_load')) - self.assertEqual(mock_taskqm.mock_calls[2], mock.call().cleanup()) - - self.assertEqual(0, variable_manager.set_host_variable.call_count) - @mock.patch('os_faults.ansible.executor.AnsibleRunner._run_play') def test_run_playbook(self, mock_run_play): ex = executor.AnsibleRunner() @@ -404,22 +270,13 @@ class AnsibleRunnerTestCase(test.TestCase): )) @mock.patch('os_faults.executor.get_module_paths') - @mock.patch('os_faults.executor.PRE_24_ANSIBLE', False) def test_make_module_path_option_ansible_24(self, mock_mp): mock_mp.return_value = ['/path/one', 'path/two'] self.assertEqual(['/path/one', 'path/two'], executor.make_module_path_option()) @mock.patch('os_faults.executor.get_module_paths') - @mock.patch('os_faults.executor.PRE_24_ANSIBLE', False) def test_make_module_path_option_ansible_24_one_item(self, mock_mp): mock_mp.return_value = ['/path/one'] self.assertEqual(['/path/one', '/path/one'], executor.make_module_path_option()) - - @mock.patch('os_faults.executor.get_module_paths') - @mock.patch('os_faults.executor.PRE_24_ANSIBLE', True) - def test_make_module_path_option_ansible_pre24(self, mock_mp): - mock_mp.return_value = ['/path/one', 'path/two'] - self.assertEqual('/path/one:path/two', - executor.make_module_path_option()) diff --git a/requirements.txt b/requirements.txt index 4eabf94..9259726 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,11 +4,11 @@ pbr>=2.0.0 # Apache-2.0 -ansible>=2.2 appdirs>=1.3.0 # MIT License click>=6.7 # BSD iso8601>=0.1.11 # MIT jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT +oslo.concurrency>=3.0.0 # Apache-2.0 oslo.i18n>=2.1.0 # Apache-2.0 oslo.serialization>=1.10.0 # Apache-2.0 oslo.utils>=3.20.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index f72ee21..8602d77 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -21,5 +21,8 @@ testrepository>=0.0.18 # Apache-2.0/BSD testscenarios>=0.4 # Apache-2.0/BSD testtools>=1.4.0 # MIT +# used for testing only +ansible # GPL-3.0 + # releasenotes reno>=1.8.0 # Apache-2.0