diff --git a/Vagrantfile b/Vagrantfile index bf44b4a49..4d4affa3e 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,7 +44,7 @@ NM_CONTROLLED=no EOF sudo ifup eth1 - /vagrant/dev/install.sh + /vagrant/dev/install-dev.sh # Configure the legacy development environment. This has been retained # while transitioning to the new development environment. diff --git a/dev/functions b/dev/functions index 60565bc49..ac3ef71c3 100644 --- a/dev/functions +++ b/dev/functions @@ -87,8 +87,8 @@ function install_dependencies { } function install_venv { - # Install a virtualenv at $1. Install all the packages in proceeding - # arguments using pip. + # Install a virtualenv at $1. The rest of the arguments are passed + # directly to pip. venv_path="$1" shift pip_paths="$@" @@ -117,6 +117,11 @@ function install_kayobe_venv { install_venv "${KAYOBE_VENV_PATH}" "${KAYOBE_SOURCE_PATH}" } +function install_kayobe_dev_venv { + # Install Kayobe in the venv in editable mode. + install_venv "${KAYOBE_VENV_PATH}" -e "${KAYOBE_SOURCE_PATH}" +} + function upgrade_kayobe_venv { echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}" virtualenv "${KAYOBE_VENV_PATH}" @@ -143,8 +148,16 @@ function environment_setup { source "${KAYOBE_VENV_PATH}/bin/activate" set -u source "${KAYOBE_CONFIG_SOURCE_PATH}/kayobe-env" - - cd "${KAYOBE_SOURCE_PATH}" + # FIXME: For upgrades to work, we must change the working directory to the kayobe + # source checkout. This can be removed once the previous version is based on + # the rocky release. + if [ ! -d "${KAYOBE_VENV_PATH}/share/kayobe/ansible" ]; then + cd "${KAYOBE_SOURCE_PATH}" + else + # kayobe should still be able to function when the current working directory + # is not the source checkout + cd /tmp + fi } function run_kayobe { diff --git a/dev/install-dev.sh b/dev/install-dev.sh new file mode 100755 index 000000000..981fbc80f --- /dev/null +++ b/dev/install-dev.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Install kayobe and its dependencies in a virtual environment. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + # Don't require kayobe configuration to exist for installation - it is not + # required for the legacy manual deployment procedure. + KAYOBE_CONFIG_REQUIRED=0 + config_init + install_dependencies + install_kayobe_dev_venv +} + +main diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst index aa9b4e01c..d9cf1a2b2 100644 --- a/doc/source/development/automated.rst +++ b/doc/source/development/automated.rst @@ -81,10 +81,17 @@ If using Vagrant, SSH into the Vagrant VM and change to the shared directory:: vagrant ssh cd /vagrant -If not using Vagrant, run the ``dev/install.sh`` script to install kayobe and +If not using Vagrant, run the ``dev/install-dev.sh`` script to install kayobe and its dependencies in a virtual environment:: - ./dev/install.sh + ./dev/install-dev.sh + +.. note:: + + This will create an :ref:`editable install `. + It is also possible to install kayobe in a non-editable way, such that + changes will not been seen until you reinstall the package. To do this you + can run ``./dev/install.sh``. Run the ``dev/overcloud-deploy.sh`` script to deploy the OpenStack control plane:: @@ -137,10 +144,17 @@ environment. Usage ----- -Run the ``dev/install.sh`` script to install kayobe and its dependencies in a +Run the ``dev/install-dev.sh`` script to install kayobe and its dependencies in a virtual environment:: - ./dev/install.sh + ./dev/install-dev.sh + +.. note:: + + This will create an :ref:`editable install `. + It is also possible to install kayobe in a non-editable way, such that + changes will not been seen until you reinstall the package. To do this you + can run ``./dev/install.sh``. Run the ``dev/seed-hypervisor-deploy.sh`` script to deploy the seed hypervisor:: @@ -180,10 +194,17 @@ environment. Usage ===== -Run the ``dev/install.sh`` script to install kayobe and its dependencies in a +Run the ``dev/install-dev.sh`` script to install kayobe and its dependencies in a virtual environment:: - ./dev/install.sh + ./dev/install-dev.sh + +.. note:: + + This will create an :ref:`editable install `. + It is also possible to install kayobe in a non-editable way, such that + changes will not been seen until you reinstall the package. To do this you + can run ``./dev/install.sh``. Run the ``dev/seed-deploy.sh`` script to deploy the seed VM:: diff --git a/doc/source/installation.rst b/doc/source/installation.rst index e07e64509..33923360e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -1,7 +1,14 @@ +.. _installation: + ============ Installation ============ +Kayobe can be installed via the released Python packages on PyPI, or from +source. Installing from PyPI ensures the use of well used and tested software, +whereas installing from source allows for the use of unreleased or patched +code. Installing from a Python package is supported from Kayobe 5.0.0 onwards. + Prerequisites ============= @@ -15,23 +22,31 @@ To avoid conflicts with python packages installed by the system package manager it is recommended to install Kayobe in a virtualenv. Ensure that the ``virtualenv`` python module is available on the Ansible control host. It is necessary to install the GCC compiler chain in order to build the extensions of -some of kayobe's python dependencies. Finally, for cloning and working with the -kayobe source code repository, Git is required. +some of kayobe's python dependencies. On CentOS:: - $ yum install -y python-devel python-virtualenv gcc git + $ yum install -y python-devel python-virtualenv gcc On Ubuntu:: - $ apt install -y python-dev python-virtualenv gcc git + $ apt install -y python-dev python-virtualenv gcc -Installation -============ +If installing Kayobe from source, then Git is required for cloning and working +with the source code repository. -This guide will describe how to install Kayobe from source in a virtualenv. +On CentOS:: -The directory structure for a kayobe Ansible control host environment is + $ yum install -y git + +On Ubuntu:: + + $ apt install -y git + +Local directory structure +========================= + +The directory structure for a Kayobe Ansible control host environment is configurable, but the following is recommended, where ```` is the path to a top level directory:: @@ -44,6 +59,58 @@ path to a top level directory:: kayobe/ kolla-ansible/ +This pattern ensures that all dependencies for a particular environment are +installed under a single top level path, and nothing is installed to a shared +location. This allows for the option of using multiple Kayobe environments on +the same control host. + +Creation of a ``kayobe-config`` source code repository will be covered in the +:ref:`configuration guide `. The Kolla Ansible source code +checkout and Python virtual environment will be created automatically by +kayobe. + +Not all of these directories will be used in all scenarios - if Kayobe or Kolla +Ansible are installed from a Python package then the source code repository is +not required. + +Installation from PyPI +====================== + +This section describes how to install Kayobe from a Python package in a +virtualenv. This is supported from Kayobe 5.0.0 onwards. + +First, change to the top level directory, and make the directories for source +code repositories and python virtual environments:: + + $ cd + $ mkdir -p src venvs + +Create a virtualenv for Kayobe:: + + $ virtualenv /venvs/kayobe + +Activate the virtualenv and update pip:: + + $ source /venvs/kayobe/bin/activate + (kayobe) $ pip install -U pip + +If using the latest version of Kayobe:: + + (kayobe) $ pip install kayobe + +Alternatively, to install a specific release of Kayobe:: + + (kayobe) $ pip install kayobe==5.0.0 + +Finally, deactivate the virtualenv:: + + (kayobe) $ deactivate + +Installation from source +======================== + +This section describes how to install Kayobe from source in a virtualenv. + First, change to the top level directory, and make the directories for source code repositories and python virtual environments:: @@ -73,7 +140,18 @@ Finally, deactivate the virtualenv:: (kayobe) $ deactivate -Creation of a ``kayobe-config`` source code repository will be covered in the -:ref:`configuration guide `. The kolla-ansible source code -checkout and python virtual environment will be created automatically by -kayobe. +.. _installation-editable: + +Editable source installation +---------------------------- + +From Kayobe 5.0.0 onwards it is possible to create an `editable install +`__ +of Kayobe. In an editable install, any changes to the Kayobe source tree will +immediately be visible when running any Kayobe commands. To create an editable +install, add the ``-e`` flag:: + + (kayobe) $ cd /src/kayobe + (kayobe) $ pip install -e . + +This is particularly useful when installing Kayobe for development. diff --git a/doc/source/upgrading.rst b/doc/source/upgrading.rst index 5324e95a7..831d81d34 100644 --- a/doc/source/upgrading.rst +++ b/doc/source/upgrading.rst @@ -16,18 +16,64 @@ Upgrading Kayobe ================ If a new, suitable version of kayobe is available, it should be installed. -If using kayobe from a git checkout, this may be done by pulling down the new -version from Github. Make sure that any local changes to kayobe are committed. -For example, to pull version 1.0.0 from the ``origin`` remote:: +As described in :ref:`installation`, Kayobe can be installed via the released +Python packages on PyPI, or from source. Installation from a Python package is +supported from Kayobe 5.0.0 onwards. - $ git pull origin 1.0.0 +Upgrading from PyPI +------------------- -If local changes were made to kayobe, these should now be reapplied. +This section describes how to upgrade Kayobe from a Python package in a +virtualenv. This is supported from Kayobe 5.0.0 onwards. -The upgraded kayobe python module and dependencies should be installed:: +Ensure that the virtualenv is activated:: + $ source /venvs/kayobe/bin/activate + +Update the pip package:: + + (kayobe) $ pip install -U pip + +If upgrading to the latest version of Kayobe:: + + (kayobe) $ pip install -U kayobe + +Alternatively, to upgrade to a specific release of Kayobe:: + + (kayobe) $ pip install kayobe==5.0.0 + +Upgrading from source +--------------------- + +This section describes how to install Kayobe from source in a virtualenv. + +First, check out the required version of the Kayobe source code. This may be +done by pulling down the new version from Github. Make sure that any local +changes to kayobe are committed and merged with the new upstream code as +necessary. For example, to pull version 5.0.0 from the ``origin`` remote:: + + $ cd /src/kayobe + $ git pull origin 5.0.0 + +Ensure that the virtualenv is activated:: + + $ source /venvs/kayobe/bin/activate + +Update the pip package:: + + (kayobe) $ pip install -U pip + +If using a non-editable install of Kayobe:: + + (kayobe) $ cd /src/kayobe (kayobe) $ pip install -U . +Alternatively, if using an editable install of Kayobe (version 5.0.0 onwards, +see :ref:`installation-editable` for details):: + + (kayobe) $ cd /src/kayobe + (kayobe) $ pip install -U -e . + Migrating Kayobe Configuration ------------------------------ diff --git a/kayobe/ansible.py b/kayobe/ansible.py index 7a92b2787..296568e87 100644 --- a/kayobe/ansible.py +++ b/kayobe/ansible.py @@ -199,7 +199,8 @@ def config_dump(parsed_args, host=None, hosts=None, var_name=None, extra_vars["dump_facts"] = facts # Don't use check mode for configuration dumps as we won't get any # results back. - run_playbook(parsed_args, "ansible/dump-config.yml", + playbook_path = utils.get_data_files_path("ansible", "dump-config.yml") + run_playbook(parsed_args, playbook_path, extra_vars=extra_vars, tags=tags, quiet=True, verbose_level=verbose_level, check=False) hostvars = {} @@ -230,7 +231,9 @@ def install_galaxy_roles(parsed_args, force=False): :param force: Whether to force reinstallation of roles. """ LOG.info("Installing galaxy role dependencies from kayobe") - utils.galaxy_install("requirements.yml", "ansible/roles", force=force) + requirements = utils.get_data_files_path("requirements.yml") + roles_destination = utils.get_data_files_path('ansible', 'roles') + utils.galaxy_install(requirements, roles_destination, force=force) # Check for requirements in kayobe configuration. kc_reqs_path = os.path.join(parsed_args.config_path, diff --git a/kayobe/cli/commands.py b/kayobe/cli/commands.py index 8ef4b1110..4596af695 100644 --- a/kayobe/cli/commands.py +++ b/kayobe/cli/commands.py @@ -19,12 +19,21 @@ from cliff.command import Command from kayobe import ansible from kayobe import kolla_ansible +from kayobe import utils from kayobe import vault def _build_playbook_list(*playbooks): """Return a list of names of playbook files given their basenames.""" - return ["ansible/%s.yml" % playbook for playbook in playbooks] + return [ + _get_playbook_path(playbook) + for playbook in playbooks + ] + + +def _get_playbook_path(playbook): + """Return the absolute path of a playbook""" + return utils.get_data_files_path("ansible", "%s.yml" % playbook) class VaultMixin(object): @@ -260,7 +269,8 @@ class PhysicalNetworkConfigure(KayobeAnsibleMixin, VaultMixin, Command): if parsed_args.interface_description_limit: extra_vars["physical_network_interface_description_limit"] = ( parsed_args.interface_description_limit) - self.run_kayobe_playbook(parsed_args, "ansible/physical-network.yml", + self.run_kayobe_playbook(parsed_args, + _get_playbook_path('physical-network'), limit=parsed_args.group, extra_vars=extra_vars) @@ -342,11 +352,14 @@ class SeedVMProvision(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, def take_action(self, parsed_args): self.app.LOG.debug("Provisioning seed VM") - self.run_kayobe_playbook(parsed_args, "ansible/ip-allocation.yml", + self.run_kayobe_playbook(parsed_args, + _get_playbook_path("ip-allocation"), limit="seed") - self.run_kayobe_playbook(parsed_args, "ansible/seed-vm-provision.yml") + self.run_kayobe_playbook(parsed_args, + _get_playbook_path("seed-vm-provision")) # Now populate the Kolla Ansible inventory. - self.run_kayobe_playbook(parsed_args, "ansible/kolla-ansible.yml", + self.run_kayobe_playbook(parsed_args, + _get_playbook_path("kolla-ansible"), tags="config") @@ -360,7 +373,7 @@ class SeedVMDeprovision(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, def take_action(self, parsed_args): self.app.LOG.debug("Deprovisioning seed VM") self.run_kayobe_playbook(parsed_args, - "ansible/seed-vm-deprovision.yml") + _get_playbook_path("seed-vm-deprovision")) class SeedHostConfigure(KollaAnsibleMixin, KayobeAnsibleMixin, VaultMixin, @@ -647,13 +660,14 @@ class OvercloudInventoryDiscover(KayobeAnsibleMixin, VaultMixin, Command): # hosts will not be present in the following playbooks in which they # are used to populate other inventories. self.run_kayobe_playbook(parsed_args, - "ansible/overcloud-inventory-discover.yml") + _get_playbook_path( + "overcloud-inventory-discover")) # If necessary, allocate IP addresses for the discovered hosts. self.run_kayobe_playbook(parsed_args, - "ansible/ip-allocation.yml") + _get_playbook_path("ip-allocation")) # Now populate the Kolla Ansible inventory. - self.run_kayobe_playbook(parsed_args, "ansible/kolla-ansible.yml", - tags="config") + self.run_kayobe_playbook(parsed_args, _get_playbook_path( + "kolla-ansible"), tags="config") class OvercloudIntrospectionDataSave(KayobeAnsibleMixin, VaultMixin, Command): diff --git a/kayobe/tests/unit/cli/test_commands.py b/kayobe/tests/unit/cli/test_commands.py index 29372364b..c3d45635c 100644 --- a/kayobe/tests/unit/cli/test_commands.py +++ b/kayobe/tests/unit/cli/test_commands.py @@ -20,6 +20,7 @@ import mock from kayobe import ansible from kayobe.cli import commands +from kayobe import utils class TestApp(cliff.app.App): @@ -44,9 +45,11 @@ class TestCase(unittest.TestCase): self.assertEqual(0, result) mock_install.assert_called_once_with(parsed_args) expected_calls = [ - mock.call(mock.ANY, ["ansible/bootstrap.yml"]), - mock.call(mock.ANY, ["ansible/kolla-ansible.yml"], - tags="install"), + mock.call(mock.ANY, [utils.get_data_files_path( + "ansible", "bootstrap.yml")]), + mock.call(mock.ANY, [ + utils.get_data_files_path("ansible", "kolla-ansible.yml") + ], tags="install"), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -63,9 +66,11 @@ class TestCase(unittest.TestCase): mock_install.assert_called_once_with(parsed_args, force=True) mock_prune.assert_called_once_with(parsed_args) expected_calls = [ - mock.call(mock.ANY, ["ansible/bootstrap.yml"]), - mock.call(mock.ANY, ["ansible/kolla-ansible.yml"], - tags="install"), + mock.call(mock.ANY, [utils.get_data_files_path( + "ansible", "bootstrap.yml")]), + mock.call(mock.ANY, [ + utils.get_data_files_path("ansible", "kolla-ansible.yml") + ], tags="install"), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -80,7 +85,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": False @@ -100,7 +105,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": True @@ -121,7 +126,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": False, @@ -143,7 +148,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": False, @@ -174,7 +179,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": False, @@ -198,7 +203,7 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - "ansible/physical-network.yml", + utils.get_data_files_path("ansible", "physical-network.yml"), limit="switches", extra_vars={ "physical_network_display": False, @@ -218,7 +223,8 @@ class TestCase(unittest.TestCase): result = command.run(parsed_args) self.assertEqual(0, result) expected_calls = [ - mock.call(mock.ANY, ["ansible/network-connectivity.yml"]), + mock.call(mock.ANY, [utils.get_data_files_path( + "ansible", "network-connectivity.yml")]), ] self.assertEqual(expected_calls, mock_run.call_args_list) @@ -245,19 +251,22 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/ip-allocation.yml", - "ansible/ssh-known-host.yml", - "ansible/kayobe-ansible-user.yml", - "ansible/pip.yml", - "ansible/kayobe-target-venv.yml", - "ansible/users.yml", - "ansible/yum.yml", - "ansible/dev-tools.yml", - "ansible/network.yml", - "ansible/sysctl.yml", - "ansible/ntp.yml", - "ansible/lvm.yml", - "ansible/seed-hypervisor-libvirt-host.yml", + utils.get_data_files_path("ansible", "ip-allocation.yml"), + utils.get_data_files_path("ansible", "ssh-known-host.yml"), + utils.get_data_files_path( + "ansible", "kayobe-ansible-user.yml"), + utils.get_data_files_path("ansible", "pip.yml"), + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path("ansible", "users.yml"), + utils.get_data_files_path("ansible", "yum.yml"), + utils.get_data_files_path("ansible", "dev-tools.yml"), + utils.get_data_files_path("ansible", "network.yml"), + utils.get_data_files_path("ansible", "sysctl.yml"), + utils.get_data_files_path("ansible", "ntp.yml"), + utils.get_data_files_path("ansible", "lvm.yml"), + utils.get_data_files_path( + "ansible", "seed-hypervisor-libvirt-host.yml"), ], limit="seed-hypervisor", ), @@ -278,8 +287,10 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/kayobe-target-venv.yml", - "ansible/kolla-target-venv.yml", + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path( + "ansible", "kolla-target-venv.yml"), ], limit="seed-hypervisor", ), @@ -312,37 +323,41 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/ip-allocation.yml", - "ansible/ssh-known-host.yml", - "ansible/kayobe-ansible-user.yml", - "ansible/pip.yml", - "ansible/kayobe-target-venv.yml", - "ansible/users.yml", - "ansible/yum.yml", - "ansible/dev-tools.yml", - "ansible/disable-selinux.yml", - "ansible/network.yml", - "ansible/sysctl.yml", - "ansible/ip-routing.yml", - "ansible/snat.yml", - "ansible/disable-glean.yml", - "ansible/ntp.yml", - "ansible/lvm.yml", + utils.get_data_files_path("ansible", "ip-allocation.yml"), + utils.get_data_files_path("ansible", "ssh-known-host.yml"), + utils.get_data_files_path( + "ansible", "kayobe-ansible-user.yml"), + utils.get_data_files_path("ansible", "pip.yml"), + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path("ansible", "users.yml"), + utils.get_data_files_path("ansible", "yum.yml"), + utils.get_data_files_path("ansible", "dev-tools.yml"), + utils.get_data_files_path( + "ansible", "disable-selinux.yml"), + utils.get_data_files_path("ansible", "network.yml"), + utils.get_data_files_path("ansible", "sysctl.yml"), + utils.get_data_files_path("ansible", "ip-routing.yml"), + utils.get_data_files_path("ansible", "snat.yml"), + utils.get_data_files_path("ansible", "disable-glean.yml"), + utils.get_data_files_path("ansible", "ntp.yml"), + utils.get_data_files_path("ansible", "lvm.yml"), ], limit="seed", ), mock.call( mock.ANY, - ["ansible/kolla-ansible.yml"], + [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ), mock.call( mock.ANY, [ - "ansible/pip.yml", - "ansible/kolla-target-venv.yml", - "ansible/kolla-host.yml", - "ansible/docker.yml", + utils.get_data_files_path("ansible", "pip.yml"), + utils.get_data_files_path( + "ansible", "kolla-target-venv.yml"), + utils.get_data_files_path("ansible", "kolla-host.yml"), + utils.get_data_files_path("ansible", "docker.yml"), ], limit="seed", extra_vars={'pip_applicable_users': [None]}, @@ -350,7 +365,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/docker-registry.yml", + utils.get_data_files_path("ansible", + "docker-registry.yml"), ], limit="seed", extra_vars={'kayobe_action': 'deploy'}, @@ -483,7 +499,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="seed", extra_vars={ @@ -508,7 +525,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="seed", extra_vars={ @@ -533,7 +551,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="seed", extra_vars={ @@ -558,8 +577,10 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/kayobe-target-venv.yml", - "ansible/kolla-target-venv.yml", + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path( + "ansible", "kolla-target-venv.yml"), ], limit="seed", ), @@ -578,9 +599,11 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/container-image-builders-check.yml", - "ansible/kolla-build.yml", - "ansible/container-image-build.yml" + utils.get_data_files_path( + "ansible", "container-image-builders-check.yml"), + utils.get_data_files_path("ansible", "kolla-build.yml"), + utils.get_data_files_path( + "ansible", "container-image-build.yml") ], extra_vars={ "container_image_sets": ( @@ -603,9 +626,11 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/container-image-builders-check.yml", - "ansible/kolla-build.yml", - "ansible/container-image-build.yml" + utils.get_data_files_path( + "ansible", "container-image-builders-check.yml"), + utils.get_data_files_path("ansible", "kolla-build.yml"), + utils.get_data_files_path( + "ansible", "container-image-build.yml") ], extra_vars={ "container_image_regexes": "'^regex1$ ^regex2$'", @@ -629,7 +654,7 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/seed-ipa-build.yml", + utils.get_data_files_path("ansible", "seed-ipa-build.yml"), ], extra_vars={}, ), @@ -650,7 +675,7 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/seed-ipa-build.yml", + utils.get_data_files_path("ansible", "seed-ipa-build.yml"), ], extra_vars={"ipa_image_force_rebuild": True}, ), @@ -672,20 +697,24 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - ["ansible/kolla-ansible.yml"], + [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ), mock.call( mock.ANY, - ["ansible/kolla-bifrost.yml"], + [utils.get_data_files_path("ansible", "kolla-bifrost.yml")], ), mock.call( mock.ANY, [ - "ansible/overcloud-host-image-workaround-resolv.yml", - "ansible/overcloud-host-image-workaround-cloud-init.yml", - "ansible/seed-introspection-rules.yml", - "ansible/dell-switch-bmp.yml", + utils.get_data_files_path( + "ansible", "overcloud-host-image-workaround-resolv.yml"), # noqa + utils.get_data_files_path( + "ansible", "overcloud-host-image-workaround-cloud-init.yml"), # noqa + utils.get_data_files_path( + "ansible", "seed-introspection-rules.yml"), + utils.get_data_files_path( + "ansible", "dell-switch-bmp.yml"), ], ), ] @@ -714,23 +743,32 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - ["ansible/kolla-ansible.yml"], + [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ), mock.call( mock.ANY, [ - "ansible/kolla-bifrost.yml", - "ansible/seed-service-upgrade-prep.yml" + utils.get_data_files_path("ansible", "kolla-bifrost.yml"), + utils.get_data_files_path("ansible", + "seed-service-upgrade-prep.yml") ], ), mock.call( mock.ANY, [ - "ansible/overcloud-host-image-workaround-resolv.yml", - "ansible/overcloud-host-image-workaround-cloud-init.yml", - "ansible/seed-introspection-rules.yml", - "ansible/dell-switch-bmp.yml", + utils.get_data_files_path( + "ansible", + "overcloud-host-image-workaround-resolv.yml"), + utils.get_data_files_path( + "ansible", + "overcloud-host-image-workaround-cloud-init.yml"), + utils.get_data_files_path( + "ansible", + "seed-introspection-rules.yml"), + utils.get_data_files_path( + "ansible", + "dell-switch-bmp.yml"), ], ), ] @@ -757,15 +795,16 @@ class TestCase(unittest.TestCase): expected_calls = [ mock.call( mock.ANY, - 'ansible/overcloud-inventory-discover.yml', + utils.get_data_files_path( + "ansible", "overcloud-inventory-discover.yml"), ), mock.call( mock.ANY, - 'ansible/ip-allocation.yml', + utils.get_data_files_path("ansible", "ip-allocation.yml"), ), mock.call( mock.ANY, - 'ansible/kolla-ansible.yml', + utils.get_data_files_path("ansible", "kolla-ansible.yml"), tags="config", ), ] @@ -785,8 +824,10 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - 'ansible/kolla-bifrost-hostvars.yml', - 'ansible/overcloud-hardware-inspect.yml', + utils.get_data_files_path( + "ansible", "kolla-bifrost-hostvars.yml"), + utils.get_data_files_path( + "ansible", "overcloud-hardware-inspect.yml"), ], ), ] @@ -806,8 +847,10 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - 'ansible/kolla-bifrost-hostvars.yml', - 'ansible/overcloud-provision.yml', + utils.get_data_files_path( + "ansible", "kolla-bifrost-hostvars.yml"), + utils.get_data_files_path( + "ansible", "overcloud-provision.yml"), ], ), ] @@ -827,7 +870,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - 'ansible/overcloud-deprovision.yml', + utils.get_data_files_path( + "ansible", "overcloud-deprovision.yml"), ], ), ] @@ -860,37 +904,43 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/ip-allocation.yml", - "ansible/ssh-known-host.yml", - "ansible/kayobe-ansible-user.yml", - "ansible/pip.yml", - "ansible/kayobe-target-venv.yml", - "ansible/users.yml", - "ansible/yum.yml", - "ansible/dev-tools.yml", - "ansible/disable-selinux.yml", - "ansible/network.yml", - "ansible/sysctl.yml", - "ansible/disable-glean.yml", - "ansible/disable-cloud-init.yml", - "ansible/ntp.yml", - "ansible/lvm.yml", + utils.get_data_files_path("ansible", "ip-allocation.yml"), + utils.get_data_files_path("ansible", "ssh-known-host.yml"), + utils.get_data_files_path( + "ansible", "kayobe-ansible-user.yml"), + utils.get_data_files_path("ansible", "pip.yml"), + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path("ansible", "users.yml"), + utils.get_data_files_path("ansible", "yum.yml"), + utils.get_data_files_path("ansible", "dev-tools.yml"), + utils.get_data_files_path( + "ansible", "disable-selinux.yml"), + utils.get_data_files_path("ansible", "network.yml"), + utils.get_data_files_path("ansible", "sysctl.yml"), + utils.get_data_files_path("ansible", "disable-glean.yml"), + utils.get_data_files_path( + "ansible", "disable-cloud-init.yml"), + utils.get_data_files_path("ansible", "ntp.yml"), + utils.get_data_files_path("ansible", "lvm.yml"), ], limit="overcloud", ), mock.call( mock.ANY, - ["ansible/kolla-ansible.yml"], + [utils.get_data_files_path("ansible", "kolla-ansible.yml")], tags="config", ), mock.call( mock.ANY, [ - "ansible/pip.yml", - "ansible/kolla-target-venv.yml", - "ansible/kolla-host.yml", - "ansible/docker.yml", - "ansible/ceph-block-devices.yml", + utils.get_data_files_path("ansible", "pip.yml"), + utils.get_data_files_path( + "ansible", "kolla-target-venv.yml"), + utils.get_data_files_path("ansible", "kolla-host.yml"), + utils.get_data_files_path("ansible", "docker.yml"), + utils.get_data_files_path( + "ansible", "ceph-block-devices.yml"), ], limit="overcloud", extra_vars={"pip_applicable_users": [None]}, @@ -1023,7 +1073,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="overcloud", extra_vars={ @@ -1048,7 +1099,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="overcloud", extra_vars={ @@ -1073,7 +1125,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/host-package-update.yml", + utils.get_data_files_path( + "ansible", "host-package-update.yml"), ], limit="overcloud", extra_vars={ @@ -1098,10 +1151,14 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/kayobe-target-venv.yml", - "ansible/kolla-target-venv.yml", - "ansible/overcloud-docker-sdk-upgrade.yml", - "ansible/overcloud-etc-hosts-fixup.yml", + utils.get_data_files_path( + "ansible", "kayobe-target-venv.yml"), + utils.get_data_files_path( + "ansible", "kolla-target-venv.yml"), + utils.get_data_files_path( + "ansible", "overcloud-docker-sdk-upgrade.yml"), + utils.get_data_files_path( + "ansible", "overcloud-etc-hosts-fixup.yml"), ], limit="overcloud", ), @@ -1120,9 +1177,11 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/container-image-builders-check.yml", - "ansible/kolla-build.yml", - "ansible/container-image-build.yml" + utils.get_data_files_path( + "ansible", "container-image-builders-check.yml"), + utils.get_data_files_path("ansible", "kolla-build.yml"), + utils.get_data_files_path( + "ansible", "container-image-build.yml") ], extra_vars={ "container_image_sets": ( @@ -1145,9 +1204,11 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/container-image-builders-check.yml", - "ansible/kolla-build.yml", - "ansible/container-image-build.yml" + utils.get_data_files_path( + "ansible", "container-image-builders-check.yml"), + utils.get_data_files_path("ansible", "kolla-build.yml"), + utils.get_data_files_path( + "ansible", "container-image-build.yml") ], extra_vars={ "container_image_regexes": "'^regex1$ ^regex2$'", @@ -1171,7 +1232,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/overcloud-ipa-build.yml", + utils.get_data_files_path( + "ansible", "overcloud-ipa-build.yml"), ], extra_vars={}, ), @@ -1192,7 +1254,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/overcloud-ipa-build.yml", + utils.get_data_files_path( + "ansible", "overcloud-ipa-build.yml"), ], extra_vars={"ipa_image_force_rebuild": True}, ), @@ -1213,11 +1276,14 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - 'ansible/overcloud-ipa-images.yml', - 'ansible/overcloud-introspection-rules.yml', - 'ansible/overcloud-introspection-rules-dell-lldp-workaround.yml', # noqa - 'ansible/provision-net.yml', - 'ansible/overcloud-grafana-configure.yml' + utils.get_data_files_path( + "ansible", "overcloud-ipa-images.yml"), + utils.get_data_files_path( + "ansible", "overcloud-introspection-rules.yml"), + utils.get_data_files_path("ansible", "overcloud-introspection-rules-dell-lldp-workaround.yml"), # noqa + utils.get_data_files_path("ansible", "provision-net.yml"), + utils.get_data_files_path( + "ansible", "overcloud-grafana-configure.yml") ], ), ] @@ -1235,7 +1301,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-inspect.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-inspect.yml"), ], ), ] @@ -1253,7 +1320,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-manage.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-manage.yml"), ], ), ] @@ -1271,7 +1339,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-provide.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-provide.yml"), ], ), ] @@ -1289,7 +1358,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-rename.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-rename.yml"), ], ), ] @@ -1307,7 +1377,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-serial-console.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-serial-console.yml"), ], extra_vars={ @@ -1331,7 +1402,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-serial-console.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-serial-console.yml"), ], extra_vars={ @@ -1354,7 +1426,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-serial-console.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-serial-console.yml"), ], extra_vars={ @@ -1378,7 +1451,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/baremetal-compute-serial-console.yml", + utils.get_data_files_path( + "ansible", "baremetal-compute-serial-console.yml"), ], extra_vars={ @@ -1401,7 +1475,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/overcloud-ipa-images.yml", + utils.get_data_files_path( + "ansible", "overcloud-ipa-images.yml"), ], extra_vars={ "ipa_images_update_ironic_nodes": True, @@ -1424,7 +1499,8 @@ class TestCase(unittest.TestCase): mock.call( mock.ANY, [ - "ansible/overcloud-ipa-images.yml", + utils.get_data_files_path( + "ansible", "overcloud-ipa-images.yml"), ], extra_vars={ "ipa_images_compute_node_limit": "sand-6-1", diff --git a/kayobe/tests/unit/test_ansible.py b/kayobe/tests/unit/test_ansible.py index 35cd0ab3d..e32d94e4c 100644 --- a/kayobe/tests/unit/test_ansible.py +++ b/kayobe/tests/unit/test_ansible.py @@ -296,15 +296,17 @@ class TestCase(unittest.TestCase): "host2": {"var2": "value2"}, } self.assertEqual(result, expected_result) + dump_config_path = utils.get_data_files_path( + "ansible", "dump-config.yml") mock_run.assert_called_once_with(parsed_args, - "ansible/dump-config.yml", + dump_config_path, extra_vars={ "dump_path": dump_dir, }, quiet=True, tags=None, verbose_level=None, check=False) mock_rmtree.assert_called_once_with(dump_dir) - mock_listdir.assert_called_once_with(dump_dir) + mock_listdir.assert_any_call(dump_dir) mock_read.assert_has_calls([ mock.call(os.path.join(dump_dir, "host1.yml")), mock.call(os.path.join(dump_dir, "host2.yml")), @@ -322,8 +324,9 @@ class TestCase(unittest.TestCase): ansible.install_galaxy_roles(parsed_args) - mock_install.assert_called_once_with("requirements.yml", - "ansible/roles", force=False) + mock_install.assert_called_once_with(utils.get_data_files_path( + "requirements.yml"), utils.get_data_files_path( + "ansible", "roles"), force=False) mock_is_readable.assert_called_once_with( "/etc/kayobe/ansible/requirements.yml") self.assertFalse(mock_mkdirs.called) @@ -341,7 +344,9 @@ class TestCase(unittest.TestCase): ansible.install_galaxy_roles(parsed_args) expected_calls = [ - mock.call("requirements.yml", "ansible/roles", force=False), + mock.call(utils.get_data_files_path("requirements.yml"), + utils.get_data_files_path("ansible", "roles"), + force=False), mock.call("/etc/kayobe/ansible/requirements.yml", "/etc/kayobe/ansible/roles", force=False)] self.assertEqual(expected_calls, mock_install.call_args_list) @@ -362,7 +367,9 @@ class TestCase(unittest.TestCase): ansible.install_galaxy_roles(parsed_args, force=True) expected_calls = [ - mock.call("requirements.yml", "ansible/roles", force=True), + mock.call(utils.get_data_files_path("requirements.yml"), + utils.get_data_files_path("ansible", "roles"), + force=True), mock.call("/etc/kayobe/ansible/requirements.yml", "/etc/kayobe/ansible/roles", force=True)] self.assertEqual(expected_calls, mock_install.call_args_list) @@ -384,8 +391,9 @@ class TestCase(unittest.TestCase): self.assertRaises(exception.Error, ansible.install_galaxy_roles, parsed_args) - mock_install.assert_called_once_with("requirements.yml", - "ansible/roles", force=False) + mock_install.assert_called_once_with(utils.get_data_files_path( + "requirements.yml"), utils.get_data_files_path("ansible", "roles"), + force=False) mock_is_readable.assert_called_once_with( "/etc/kayobe/ansible/requirements.yml") mock_mkdirs.assert_called_once_with("/etc/kayobe/ansible/roles") diff --git a/kayobe/utils.py b/kayobe/utils.py index b96179502..20302b824 100644 --- a/kayobe/utils.py +++ b/kayobe/utils.py @@ -12,6 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. +import glob import logging import os import six @@ -23,6 +24,23 @@ import yaml LOG = logging.getLogger(__name__) +_BASE_PATH = os.path.join(sys.prefix, "share", "kayobe") + + +def get_data_files_path(*relative_path): + """Given a relative path to a data file, return the absolute path""" + # Detect editable pip install / python setup.py develop and use a path + # relative to the source directory + egg_glob = os.path.join( + sys.prefix, 'lib*', 'python*', '*-packages', 'kayobe.egg-link' + ) + egg_link = glob.glob(egg_glob) + if egg_link: + with open(egg_link[0], "r") as f: + realpath = f.readline().strip() + return os.path.join(realpath, *relative_path) + return os.path.join(_BASE_PATH, *relative_path) + def yum_install(packages): """Install a list of packages via Yum.""" diff --git a/releasenotes/notes/package-runtime-files-in-python-package-c3dda2bd32844fdf.yaml b/releasenotes/notes/package-runtime-files-in-python-package-c3dda2bd32844fdf.yaml new file mode 100644 index 000000000..9d2c0db48 --- /dev/null +++ b/releasenotes/notes/package-runtime-files-in-python-package-c3dda2bd32844fdf.yaml @@ -0,0 +1,15 @@ +--- +features: + - | + Kayobe no longer requires a checkout of the source code repository to + function. The files needed to run kayobe are now shipped as part of the + python package. Please see: `Story 2004252 `_ + for more details. +upgrade: + - | + Modifications to the kayobe source tree will no longer have an immediate + effect. This is because the ansible playbooks are now shipped as part of the + kayobe package. You must reinstall the package, or use an editable package + install, see: `pip editable-installs + `_, + to replicate the old behaviour. diff --git a/setup.cfg b/setup.cfg index 00dd002cb..cc1c4bf75 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,16 @@ classifier = [files] packages = kayobe +data_files = + share/kayobe/ansible = ansible/* + # We have to include the roles directory explicitly to Workaround PBR bug: + # source prefix replaced globally, see: + # https://bugs.launchpad.net/pbr/+bug/1810804 + share/kayobe/ansible/roles = ansible/roles/* + share/kayobe/doc = doc/* + share/kayobe/etc_examples = etc/* + share/kayobe = setup.cfg + share/kayobe = requirements.yml [entry_points] console_scripts=