Kayobe environment dependencies

Allows you to combine multiple environments by declaring any dependencies.

Story: 2002009
Task: 42911
Change-Id: I4d9f96ec4cf3c6cd0d28dfe5ddb239d863498a72
This commit is contained in:
Will Szumski 2021-07-23 11:18:41 +00:00 committed by Mark Goddard
parent a46b3ebc8e
commit 0ae9e8d489
15 changed files with 494 additions and 89 deletions

View File

@ -16,6 +16,10 @@ kayobe_environment: "{{ lookup('env', 'KAYOBE_ENVIRONMENT') }}"
# environment path appended if kayobe_environment is set. # environment path appended if kayobe_environment is set.
kayobe_env_config_path: "{{ kayobe_config_path ~ ('/environments/' ~ kayobe_environment if kayobe_environment else '') }}" kayobe_env_config_path: "{{ kayobe_config_path ~ ('/environments/' ~ kayobe_environment if kayobe_environment else '') }}"
# Ordered list of paths containing kayobe_env_config_path and all its dependent
# environments.
kayobe_env_search_paths: "{{ query('cached', 'kayobe_environments') }}"
############################################################################### ###############################################################################
# Remote path configuration (seed, seed-hypervisor and overcloud hosts). # Remote path configuration (seed, seed-hypervisor and overcloud hosts).

View File

@ -91,9 +91,9 @@
kolla_external_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy.pem" kolla_external_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy.pem"
kolla_internal_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy-internal.pem" kolla_internal_fqdn_cert: "{{ kolla_config_path }}/certificates/haproxy-internal.pem"
kolla_ansible_passwords_path: "{{ kayobe_env_config_path }}/kolla/passwords.yml" kolla_ansible_passwords_path: "{{ kayobe_env_config_path }}/kolla/passwords.yml"
kolla_overcloud_inventory_search_paths: kolla_overcloud_inventory_search_paths_static:
- "{{ kayobe_config_path }}" - "{{ kayobe_config_path }}"
- "{{ kayobe_env_config_path }}" kolla_overcloud_inventory_search_paths: "{{ kolla_overcloud_inventory_search_paths_static + kayobe_env_search_paths }}"
kolla_ansible_certificates_path: "{{ kayobe_env_config_path }}/kolla/certificates" kolla_ansible_certificates_path: "{{ kayobe_env_config_path }}/kolla/certificates"
# NOTE: This differs from the default SELinux mode in kolla ansible, # NOTE: This differs from the default SELinux mode in kolla ansible,
# which is permissive. The justification for using this mode is twofold: # which is permissive. The justification for using this mode is twofold:
@ -109,9 +109,9 @@
kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}" kolla_inspector_extra_kernel_options: "{{ inspector_extra_kernel_options }}"
kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}" kolla_libvirt_tls: "{{ compute_libvirt_enable_tls | bool }}"
kolla_enable_host_ntp: false kolla_enable_host_ntp: false
kolla_globals_paths_extra: kolla_globals_paths_static:
- "{{ kayobe_config_path }}" - "{{ kayobe_config_path }}"
- "{{ kayobe_env_config_path }}" kolla_globals_paths_extra: "{{ kolla_globals_paths_static + kayobe_env_search_paths }}"
- name: Generate Kolla Ansible host vars for the seed host - name: Generate Kolla Ansible host vars for the seed host
hosts: seed hosts: seed

View File

@ -14,6 +14,6 @@
kolla_bifrost_dnsmasq_dns_servers: "{{ resolv_nameservers | default([]) }}" kolla_bifrost_dnsmasq_dns_servers: "{{ resolv_nameservers | default([]) }}"
kolla_bifrost_domain: "{{ resolv_domain | default }}" kolla_bifrost_domain: "{{ resolv_domain | default }}"
kolla_bifrost_download_ipa: "{{ not ipa_build_images | bool }}" kolla_bifrost_download_ipa: "{{ not ipa_build_images | bool }}"
kolla_bifrost_config_paths_extra: kolla_bifrost_config_paths_static:
- "{{ kayobe_config_path }}" - "{{ kayobe_config_path }}"
- "{{ kayobe_env_config_path }}" kolla_bifrost_config_paths_extra: "{{ kolla_bifrost_config_paths_static + kayobe_env_search_paths }}"

View File

@ -7,7 +7,7 @@
- role: kolla - role: kolla
kolla_install_epel: "{{ dnf_install_epel }}" kolla_install_epel: "{{ dnf_install_epel }}"
- role: kolla-build - role: kolla-build
kolla_build_config_paths_extra: kolla_build_config_paths_static:
- "{{ kayobe_config_path }}" - "{{ kayobe_config_path }}"
- "{{ kayobe_env_config_path }}" kolla_build_config_paths_extra: "{{ kolla_build_config_paths_static + kayobe_env_search_paths }}"
kolla_base_tag: "{{ kolla_base_distro_version }}" kolla_base_tag: "{{ kolla_base_distro_version }}"

View File

@ -170,9 +170,9 @@
kolla_inspector_swift_auth: kolla_inspector_swift_auth:
auth_type: none auth_type: none
endpoint_override: "http://{% raw %}{{ api_interface_address }}{% endraw %}:{{ inspector_store_port }}" endpoint_override: "http://{% raw %}{{ api_interface_address }}{% endraw %}:{{ inspector_store_port }}"
kolla_openstack_custom_config_paths_extra_multi_env: kolla_openstack_custom_config_paths_extra_multi_env_static:
- "{{ kayobe_config_path }}" - "{{ kayobe_config_path }}"
- "{{ kayobe_env_config_path }}" kolla_openstack_custom_config_paths_extra_multi_env: "{{ kolla_openstack_custom_config_paths_extra_multi_env_static + kayobe_env_search_paths }}"
kolla_openstack_custom_config_paths_extra_legacy: kolla_openstack_custom_config_paths_extra_legacy:
- "{{ kayobe_env_config_path }}" - "{{ kayobe_env_config_path }}"
kolla_openstack_custom_config_paths_extra: "{{ kolla_openstack_custom_config_paths_extra_multi_env if kolla_openstack_custom_config_environment_merging_enabled | bool else kolla_openstack_custom_config_paths_extra_legacy }}" kolla_openstack_custom_config_paths_extra: "{{ kolla_openstack_custom_config_paths_extra_multi_env if kolla_openstack_custom_config_environment_merging_enabled | bool else kolla_openstack_custom_config_paths_extra_legacy }}"

View File

@ -0,0 +1,19 @@
# Copyright (c) 2023 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
__metaclass__ = type
import kayobe.plugins.lookup.environments
LookupModule = kayobe.plugins.lookup.environments.LookupModule

View File

@ -379,6 +379,63 @@ This would configure the external FQDN for the staging environment at
``staging-api.example.com``, while the production external FQDN would be at ``staging-api.example.com``, while the production external FQDN would be at
``production-api.example.com``. ``production-api.example.com``.
Environment Dependencies
------------------------
.. warning::
This is an experimental feature and is still subject to change whilst
the design is finalised.
Since the Antelope 14.0.0 release, multiple environments can be layered on top
of each of each other by declaring dependencies in a ``.kayobe-environment``
file located in the environment subdirectory. For example:
.. code-block:: yaml
:caption: ``$KAYOBE_CONFIG_PATH/environments/environment-C/.kayobe-environment``
dependencies:
- environment-B
.. code-block:: yaml
:caption: ``$KAYOBE_CONFIG_PATH/environments/environment-B/.kayobe-environment``
dependencies:
- environment-A
Kayobe uses a dependency resolver to order these environments into a linear
chain. Any dependency cycles in will result in an error. Using the example
above the chain would be resolved to:
.. code-block:: text
C -> B -> A
Where C is the environment with highest precedence. Kayobe will make sure to
include the inventory and extra-vars in an order matching this chain when
running any playbooks.
Mixin environments
^^^^^^^^^^^^^^^^^^
Environment dependencies can be used to design fragments of re-useable
configuration that can be shared across multiple environments. For example:
.. code-block:: yaml
:caption: ``$KAYOBE_CONFIG_PATH/environments/environment-A/.kayobe-environment``
dependencies:
- environment-mixin-1
- environment-mixin-2
- environment-mixin-3
In this case, each environment dependency could provide the configuration
necessary for one or more features. The mixin environments do not necessarily
need to define any dependencies between them, however Kayobe will perform a
topological sort to determine a suitable precedence. Care should be taken to
make sure that environments without an explicit ordering do not modify the same
variables.
Final Considerations Final Considerations
-------------------- --------------------

View File

@ -84,34 +84,26 @@ def add_args(parser):
"specific playbooks. \"all\" skips all playbooks") "specific playbooks. \"all\" skips all playbooks")
def _get_kayobe_environment_path(parsed_args): def _get_inventories_paths(parsed_args, env_paths):
"""Return the path to the Kayobe environment or None if not specified."""
env_path = None
if parsed_args.environment:
# Specified via --environment or KAYOBE_ENVIRONMENT.
kc_environments = os.path.join(parsed_args.config_path, "environments")
env_path = os.path.join(kc_environments, parsed_args.environment)
return env_path
def _get_inventories_paths(parsed_args, env_path):
"""Return the paths to the Kayobe inventories.""" """Return the paths to the Kayobe inventories."""
default_inventory = utils.get_data_files_path("ansible", "inventory") default_inventory = utils.get_data_files_path("ansible", "inventory")
inventories = [default_inventory] inventories = [default_inventory]
if parsed_args.inventory: if parsed_args.inventory:
inventories.extend(parsed_args.inventory) inventories.extend(parsed_args.inventory)
else: return inventories
shared_inventory = os.path.join(parsed_args.config_path, "inventory")
if env_path: shared_inventory = os.path.join(parsed_args.config_path, "inventory")
if os.path.exists(shared_inventory): if env_paths:
inventories.append(shared_inventory) if os.path.exists(shared_inventory):
env_inventory = os.path.join(env_path, "inventory")
if os.path.exists(env_inventory):
inventories.append(env_inventory)
else:
# Preserve existing behaviour: don't check if an inventory
# directory exists when no environment is specified
inventories.append(shared_inventory) inventories.append(shared_inventory)
else:
# Preserve existing behaviour: don't check if an inventory
# directory exists when no environment is specified
inventories.append(shared_inventory)
for env_path in env_paths or []:
env_inventory = os.path.join(env_path, "inventory")
if os.path.exists(env_inventory):
inventories.append(env_inventory)
return inventories return inventories
@ -129,15 +121,18 @@ def _validate_args(parsed_args, playbooks):
"use.") "use.")
sys.exit(1) sys.exit(1)
env_path = _get_kayobe_environment_path(parsed_args) environment_finder = utils.EnvironmentFinder(
if env_path: parsed_args.config_path, parsed_args.environment)
result = utils.is_readable_dir(env_path) env_paths = environment_finder.ordered_paths()
if not result["result"]: for env_path in env_paths:
LOG.error("Kayobe environment %s is invalid: %s", if env_path:
env_path, result["message"]) result = utils.is_readable_dir(env_path)
sys.exit(1) if not result["result"]:
LOG.error("Kayobe environment %s is invalid: %s",
env_path, result["message"])
sys.exit(1)
inventories = _get_inventories_paths(parsed_args, env_path) inventories = _get_inventories_paths(parsed_args, env_paths)
for inventory in inventories: for inventory in inventories:
result = utils.is_readable_dir(inventory) result = utils.is_readable_dir(inventory)
if not result["result"]: if not result["result"]:
@ -184,12 +179,14 @@ def build_args(parsed_args, playbooks,
if list_tasks or (parsed_args.list_tasks and list_tasks is None): if list_tasks or (parsed_args.list_tasks and list_tasks is None):
cmd += ["--list-tasks"] cmd += ["--list-tasks"]
cmd += vault.build_args(parsed_args, "--vault-password-file") cmd += vault.build_args(parsed_args, "--vault-password-file")
env_path = _get_kayobe_environment_path(parsed_args) environment_finder = utils.EnvironmentFinder(
inventories = _get_inventories_paths(parsed_args, env_path) parsed_args.config_path, parsed_args.environment)
env_paths = environment_finder.ordered_paths()
inventories = _get_inventories_paths(parsed_args, env_paths)
for inventory in inventories: for inventory in inventories:
cmd += ["--inventory", inventory] cmd += ["--inventory", inventory]
vars_paths = [parsed_args.config_path] vars_paths = [parsed_args.config_path]
if env_path: for env_path in env_paths:
vars_paths.append(env_path) vars_paths.append(env_path)
vars_files = _get_vars_files(vars_paths) vars_files = _get_vars_files(vars_paths)
for vars_file in vars_files: for vars_file in vars_files:
@ -438,7 +435,8 @@ def prune_galaxy_roles(parsed_args):
def passwords_yml_exists(parsed_args): def passwords_yml_exists(parsed_args):
"""Return whether passwords.yml exists in the kayobe configuration.""" """Return whether passwords.yml exists in the kayobe configuration."""
env_path = _get_kayobe_environment_path(parsed_args) env_path = utils.get_kayobe_environment_path(
parsed_args.config_path, parsed_args.environment)
path = env_path if env_path else parsed_args.config_path path = env_path if env_path else parsed_args.config_path
passwords_path = os.path.join(path, 'kolla', 'passwords.yml') passwords_path = os.path.join(path, 'kolla', 'passwords.yml')
return utils.is_readable_file(passwords_path)["result"] return utils.is_readable_file(passwords_path)["result"]

View File

@ -78,22 +78,23 @@ def _get_inventory_paths(parsed_args, inventory_filename):
else: else:
paths = [os.path.join(parsed_args.kolla_config_path, "inventory", paths = [os.path.join(parsed_args.kolla_config_path, "inventory",
inventory_filename)] inventory_filename)]
def append_path(directory):
candidate_path = os.path.join(
parsed_args.kolla_config_path, "extra-inventories",
directory)
if utils.is_readable_dir(candidate_path)["result"]:
paths.append(candidate_path)
# Inventory in the base layer is placed in the "kayobe" # Inventory in the base layer is placed in the "kayobe"
# directory. This means that you can't have an environment # directory. This means that you can't have an environment
# called kayobe as it would conflict. # called kayobe as it would conflict.
append_path("kayobe") environments = ["kayobe"]
if parsed_args.environment: if parsed_args.environment:
append_path(parsed_args.environment) environments.append(parsed_args.environment)
else:
environment_finder = utils.EnvironmentFinder(
parsed_args.config_path, parsed_args.environment)
for environment in environment_finder.ordered():
environments.append(environment)
for environment in environments:
candidate_path = os.path.join(
parsed_args.kolla_config_path, "extra-inventories",
environment)
if utils.is_readable_dir(candidate_path)["result"]:
paths.append(candidate_path)
return paths return paths

View File

@ -0,0 +1,49 @@
# Copyright (c) 2023 StackHPC Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from ansible.errors import AnsibleError
from ansible.plugins.loader import lookup_loader
from ansible.plugins.lookup import LookupBase
from kayobe.utils import EnvironmentFinder
__version__ = "1.0.0"
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
class LookupModule(LookupBase):
def run(self, terms, variables=None, **kwargs):
lookup = lookup_loader.get(
'vars', loader=self._loader, templar=self._templar
)
# Values in variables are untemplated, e.g:
# {{ lookup('env', 'KAYOBE_CONFIG_PATH') | default('/etc/kayobe', true) }} # noqa
environment = lookup.run(
["kayobe_environment"],
variables=variables, default='')[0]
kayobe_config_path = lookup.run(
["kayobe_config_path"],
variables=variables, default='')[0]
if not environment:
return []
if not kayobe_config_path:
raise AnsibleError("kayobe_config_path is unset")
environment_finder = EnvironmentFinder(kayobe_config_path, environment)
return environment_finder.ordered_paths()

View File

@ -877,11 +877,14 @@ class TestCase(unittest.TestCase):
def test_multiple_inventories(self, mock_validate, mock_vars, mock_run, def test_multiple_inventories(self, mock_validate, mock_vars, mock_run,
mock_exists): mock_exists):
mock_vars.return_value = [] mock_vars.return_value = []
# os.path.exists gets called three times:
# 1) shared inventory def exists_replacement(path):
# 2) environment inventory if path == "/etc/kayobe/inventory":
# 3) ansible.cfg return True
mock_exists.side_effect = [True, True, False] if path == "/etc/kayobe/environments/test-env/inventory":
return True
return False
mock_exists.side_effect = exists_replacement
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
ansible.add_args(parser) ansible.add_args(parser)
vault.add_args(parser) vault.add_args(parser)
@ -907,12 +910,6 @@ class TestCase(unittest.TestCase):
"ANSIBLE_FILTER_PLUGINS": mock.ANY, "ANSIBLE_FILTER_PLUGINS": mock.ANY,
"ANSIBLE_TEST_PLUGINS": mock.ANY, "ANSIBLE_TEST_PLUGINS": mock.ANY,
} }
expected_calls = [
mock.call("/etc/kayobe/inventory"),
mock.call("/etc/kayobe/environments/test-env/inventory"),
mock.call("/etc/kayobe/ansible.cfg"),
]
self.assertListEqual(expected_calls, mock_exists.mock_calls)
mock_run.assert_called_once_with(expected_cmd, check_output=False, mock_run.assert_called_once_with(expected_cmd, check_output=False,
quiet=False, env=expected_env) quiet=False, env=expected_env)
mock_vars.assert_called_once_with( mock_vars.assert_called_once_with(
@ -925,11 +922,12 @@ class TestCase(unittest.TestCase):
def test_shared_inventory_only(self, mock_validate, mock_vars, mock_run, def test_shared_inventory_only(self, mock_validate, mock_vars, mock_run,
mock_exists): mock_exists):
mock_vars.return_value = [] mock_vars.return_value = []
# os.path.exists gets called three times:
# 1) shared inventory def exists_replacement(path):
# 2) environment inventory if path == "/etc/kayobe/inventory":
# 3) ansible.cfg return True
mock_exists.side_effect = [True, False, False] return False
mock_exists.side_effect = exists_replacement
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
ansible.add_args(parser) ansible.add_args(parser)
vault.add_args(parser) vault.add_args(parser)
@ -954,12 +952,6 @@ class TestCase(unittest.TestCase):
"ANSIBLE_FILTER_PLUGINS": mock.ANY, "ANSIBLE_FILTER_PLUGINS": mock.ANY,
"ANSIBLE_TEST_PLUGINS": mock.ANY, "ANSIBLE_TEST_PLUGINS": mock.ANY,
} }
expected_calls = [
mock.call("/etc/kayobe/inventory"),
mock.call("/etc/kayobe/environments/test-env/inventory"),
mock.call("/etc/kayobe/ansible.cfg"),
]
self.assertListEqual(expected_calls, mock_exists.mock_calls)
mock_run.assert_called_once_with(expected_cmd, check_output=False, mock_run.assert_called_once_with(expected_cmd, check_output=False,
quiet=False, env=expected_env) quiet=False, env=expected_env)
mock_vars.assert_called_once_with( mock_vars.assert_called_once_with(
@ -972,11 +964,13 @@ class TestCase(unittest.TestCase):
def test_env_inventory_only(self, mock_validate, mock_vars, mock_run, def test_env_inventory_only(self, mock_validate, mock_vars, mock_run,
mock_exists): mock_exists):
mock_vars.return_value = [] mock_vars.return_value = []
# os.path.exists gets called three times: # We only want it to find the inventory in the environment
# 1) shared inventory
# 2) environment inventory def exists_replacement(path):
# 3) ansible.cfg if path == "/etc/kayobe/environments/test-env/inventory":
mock_exists.side_effect = [False, True, False] return True
return False
mock_exists.side_effect = exists_replacement
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
ansible.add_args(parser) ansible.add_args(parser)
vault.add_args(parser) vault.add_args(parser)
@ -1001,13 +995,138 @@ class TestCase(unittest.TestCase):
"ANSIBLE_FILTER_PLUGINS": mock.ANY, "ANSIBLE_FILTER_PLUGINS": mock.ANY,
"ANSIBLE_TEST_PLUGINS": mock.ANY, "ANSIBLE_TEST_PLUGINS": mock.ANY,
} }
expected_calls = [
mock.call("/etc/kayobe/inventory"),
mock.call("/etc/kayobe/environments/test-env/inventory"),
mock.call("/etc/kayobe/ansible.cfg"),
]
self.assertListEqual(expected_calls, mock_exists.mock_calls)
mock_run.assert_called_once_with(expected_cmd, check_output=False, mock_run.assert_called_once_with(expected_cmd, check_output=False,
quiet=False, env=expected_env) quiet=False, env=expected_env)
mock_vars.assert_called_once_with( mock_vars.assert_called_once_with(
["/etc/kayobe", "/etc/kayobe/environments/test-env"]) ["/etc/kayobe", "/etc/kayobe/environments/test-env"])
@mock.patch.object(utils.EnvironmentFinder, "ordered")
@mock.patch.object(os.path, "exists")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_multi_env_inventory_only(self, mock_validate, mock_vars,
mock_run, mock_exists, mock_finder):
mock_vars.return_value = []
mock_finder.return_value = ["dependency-env", "test-env"]
def exists_replacement(path):
if path == "/etc/kayobe/environments/test-env/inventory":
return True
if path == "/etc/kayobe/environments/dependency-env/inventory":
return True
return False
mock_exists.side_effect = exists_replacement
parser = argparse.ArgumentParser()
ansible.add_args(parser)
vault.add_args(parser)
args = [
"--environment", "test-env",
]
parsed_args = parser.parse_args(args)
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
expected_cmd = [
"ansible-playbook",
"--inventory", utils.get_data_files_path("ansible", "inventory"),
"--inventory", "/etc/kayobe/environments/dependency-env/inventory",
"--inventory", "/etc/kayobe/environments/test-env/inventory",
"playbook1.yml",
"playbook2.yml",
]
expected_env = {
"KAYOBE_CONFIG_PATH": "/etc/kayobe",
"KAYOBE_ENVIRONMENT": "test-env",
"ANSIBLE_ROLES_PATH": mock.ANY,
"ANSIBLE_COLLECTIONS_PATH": mock.ANY,
"ANSIBLE_ACTION_PLUGINS": mock.ANY,
"ANSIBLE_FILTER_PLUGINS": mock.ANY,
"ANSIBLE_TEST_PLUGINS": mock.ANY,
}
mock_run.assert_called_once_with(expected_cmd, check_output=False,
quiet=False, env=expected_env)
mock_vars.assert_called_once_with(
["/etc/kayobe",
"/etc/kayobe/environments/dependency-env",
"/etc/kayobe/environments/test-env"]
)
@mock.patch.object(utils.EnvironmentFinder, "ordered")
@mock.patch.object(os.path, "exists")
@mock.patch.object(utils, "run_command")
@mock.patch.object(ansible, "_get_vars_files")
@mock.patch.object(ansible, "_validate_args")
def test_multi_env_vars(self, mock_validate, mock_vars,
mock_run, mock_exists, mock_finder):
def get_vars_replacement(paths):
result = []
for path in paths:
if path == "/etc/kayobe/environments/test-env":
result.extend(
["vars-test-env-1.yml", "vars-test-env-2.yml"]
)
continue
if path == "/etc/kayobe/environments/dependency-env":
result.extend(
["vars-dependency-env-1.yml",
"vars-dependency-env-2.yml"]
)
continue
if path == "/etc/kayobe":
result.extend(
["vars-1.yml", "vars-2.yml"]
)
continue
return result
mock_vars.side_effect = get_vars_replacement
mock_finder.return_value = ["dependency-env", "test-env"]
def exists_replacement(path):
if path == "/etc/kayobe/environments/test-env/inventory":
return True
if path == "/etc/kayobe/environments/dependency-env/inventory":
return True
return False
mock_exists.side_effect = exists_replacement
parser = argparse.ArgumentParser()
ansible.add_args(parser)
vault.add_args(parser)
args = [
"--environment", "test-env",
]
parsed_args = parser.parse_args(args)
ansible.run_playbooks(parsed_args, ["playbook1.yml", "playbook2.yml"])
expected_cmd = [
"ansible-playbook",
"--inventory", utils.get_data_files_path("ansible", "inventory"),
"--inventory", "/etc/kayobe/environments/dependency-env/inventory",
"--inventory", "/etc/kayobe/environments/test-env/inventory",
'-e', '@vars-1.yml',
'-e', '@vars-2.yml',
'-e', '@vars-dependency-env-1.yml',
'-e', '@vars-dependency-env-2.yml',
'-e', '@vars-test-env-1.yml',
'-e', '@vars-test-env-2.yml',
"playbook1.yml",
"playbook2.yml",
]
expected_env = {
"KAYOBE_CONFIG_PATH": "/etc/kayobe",
"KAYOBE_ENVIRONMENT": "test-env",
"ANSIBLE_ROLES_PATH": mock.ANY,
"ANSIBLE_COLLECTIONS_PATH": mock.ANY,
"ANSIBLE_ACTION_PLUGINS": mock.ANY,
"ANSIBLE_FILTER_PLUGINS": mock.ANY,
"ANSIBLE_TEST_PLUGINS": mock.ANY,
}
mock_run.assert_called_once_with(expected_cmd, check_output=False,
quiet=False, env=expected_env)
mock_vars.assert_called_once_with(
["/etc/kayobe",
"/etc/kayobe/environments/dependency-env",
"/etc/kayobe/environments/test-env"]
)

View File

@ -223,3 +223,56 @@ key2: value2
def test_intersect_limits_arg_and_cli_colon(self): def test_intersect_limits_arg_and_cli_colon(self):
result = utils.intersect_limits("foo:bar", "baz") result = utils.intersect_limits("foo:bar", "baz")
self.assertEqual("foo:bar:&baz", result) self.assertEqual("foo:bar:&baz", result)
def test_environment_finder_with_single_environment(self):
finder = utils.EnvironmentFinder('/etc/kayobe', 'environment-A')
environments = finder.ordered()
expected = ["environment-A"]
self.assertEqual(expected, environments)
expected = ["/etc/kayobe/environments/environment-A"]
paths = finder.ordered_paths()
self.assertEqual(expected, paths)
@mock.patch.object(utils.EnvironmentFinder, "_read_metadata")
def test_environment_finder_with_dependency_chain(self, mock_yaml):
def yaml_replacement(path):
if path == ("/etc/kayobe/environments/environment-C/"
".kayobe-environment"):
return {"dependencies": ["environment-A", "environment-B"]}
if path == ("/etc/kayobe/environments/environment-B/"
".kayobe-environment"):
return {"dependencies": ["environment-A"]}
return {}
mock_yaml.side_effect = yaml_replacement
finder = utils.EnvironmentFinder('/etc/kayobe', 'environment-C')
result = finder.ordered()
expected = ["environment-A", "environment-B", "environment-C"]
self.assertEqual(expected, result)
expected = ["/etc/kayobe/environments/environment-A",
"/etc/kayobe/environments/environment-B",
"/etc/kayobe/environments/environment-C"]
paths = finder.ordered_paths()
self.assertEqual(expected, paths)
@mock.patch.object(utils.EnvironmentFinder, "_read_metadata")
def test_environment_finder_with_cycle(self, mock_yaml):
# The cycle is: C - B - C
def yaml_replacement(path):
if path == ("/etc/kayobe/environments/environment-C/"
".kayobe-environment"):
return {"dependencies": ["environment-A", "environment-B"]}
if path == ("/etc/kayobe/environments/environment-B/"
".kayobe-environment"):
return {"dependencies": ["environment-A", "environment-C"]}
return {}
mock_yaml.side_effect = yaml_replacement
finder = utils.EnvironmentFinder('/etc/kayobe', 'environment-C')
self.assertRaises(exception.Error, finder.ordered)
self.assertRaises(exception.Error, finder.ordered_paths)
def test_environment_finder_no_environment(self):
finder = utils.EnvironmentFinder('/etc/kayobe', None)
self.assertEqual([], finder.ordered())
self.assertEqual([], finder.ordered_paths())

View File

@ -13,7 +13,9 @@
# under the License. # under the License.
import base64 import base64
from collections import defaultdict
import glob import glob
import graphlib
import logging import logging
import os import os
import shutil import shutil
@ -265,3 +267,99 @@ def copy_dir(src, dest, exclude=None):
copy_dir(src_path, dest_path) copy_dir(src_path, dest_path)
else: else:
shutil.copy2(src_path, dest_path) shutil.copy2(src_path, dest_path)
def get_kayobe_environment_path(base_path, environment):
"""Return the path to the Kayobe environment or None if not specified."""
env_path = None
if environment:
# Specified via --environment or KAYOBE_ENVIRONMENT.
kc_environments = os.path.join(base_path, "environments")
env_path = os.path.join(kc_environments, environment)
return env_path
class EnvironmentFinder(object):
"""Dependency resolver for kayobe environments
The constraints are specified via a .kayobe-environment file.
"""
def __new__(cls, base_path, environment):
# Singleton instance so we don't have to resolve dependencies multiple
# times or pass round a single instance.
it = cls.__dict__.get("__it__")
if it is None:
it = {}
if (base_path, environment) in it:
return it[(base_path, environment)]
singleton = object.__new__(cls)
singleton._init(base_path, environment)
it[(base_path, environment)] = singleton
return singleton
def _init(self, base_path, environment):
self._base_path = base_path
self._environment = environment
self._ordering = None
@staticmethod
def _read_metadata(path):
if os.path.exists(path) and os.path.isfile(path):
metadata = read_yaml_file(path)
return metadata
return {}
def _collect(self, environment, result, visited):
# Updates result to contain dependency graph
base = self._base_path
env_path = os.path.join(base, 'environments', environment)
dot_environment_path = os.path.join(env_path, '.kayobe-environment')
if dot_environment_path in visited:
return
visited.add(dot_environment_path)
metadata = EnvironmentFinder._read_metadata(dot_environment_path)
dependencies = metadata.get("dependencies", [])
if not isinstance(dependencies, list):
raise exception.Error(".kayobe-environment: dependencies field "
"should be a list")
result[environment] |= set(dependencies)
for dependency in dependencies:
if not isinstance(dependency, str):
raise exception.Error("Kayobe environment dependency items "
"should be strings")
self._collect(dependency, result, visited)
def ordered(self):
"""List of environments ordered by the constraints"""
environment = self._environment
if not environment:
return []
if self._ordering is not None:
return self._ordering.copy()
graph = defaultdict(set)
self._collect(environment, graph, set())
ts = graphlib.TopologicalSorter(graph)
try:
ordering = list(ts.static_order())
except graphlib.CycleError as e:
# https://docs.python.org/3/library/graphlib.html#graphlib.CycleError
cycle = e.args[1]
raise exception.Error("You have created a cycle with your "
"environment dependencies. Please break "
"this cycle and try again. The cycle is: %s"
% cycle)
self._ordering = ordering if ordering else [environment]
return self._ordering.copy()
def ordered_paths(self):
"""Paths to each environment ordered by the constraints"""
result = []
environments = self.ordered()
for environment in environments:
full_path = get_kayobe_environment_path(
self._base_path,
environment
)
result.append(full_path)
return result

View File

@ -0,0 +1,5 @@
---
features:
- |
Adds a experimental support for layering multiple environments using a
.kayobe-environment file.

View File

@ -12,3 +12,5 @@ jsonschema<5 # MIT
wcmatch>=8.2,<=9.0 # MIT wcmatch>=8.2,<=9.0 # MIT
hvac>=0.10.1 hvac>=0.10.1
ansible-cached-lookup<=2.0.0 # MIT ansible-cached-lookup<=2.0.0 # MIT
# NOTE(wszusmki): Remove this when min python>=3.9
graphlib-backport<2.0.0; python_version<"3.9" # PSF