From c6632efa75a6dfd0d7101d303de3f814f53bec29 Mon Sep 17 00:00:00 2001 From: Steve Noyes Date: Mon, 18 Jun 2018 16:45:10 -0400 Subject: [PATCH] Add ability to load inventory from a file It would be helpful to have the ability to populate the kolla-cli inventory file from any inventory file, not just the all-in-one file from kolla. To do this, The config reset command has been updated to take an optional --inventory path argument that will cause the new inventory to be generated from that anible inventory file. While doing this found a few things to fix: - renamed allineone.py & AllInOne to AnsibleInventory to better reflect what they are now doing. - In kolla_actions, preserve the empty globals.yml file (otherwise the soft linkage is broken) and fix the kolla-cli directory name. Change-Id: Ib250c5de8e00818b41c1a4539aba8c165a4ad2e2 --- kolla_cli/api/config.py | 26 +++++++++++- kolla_cli/commands/config.py | 22 +++++++++- .../{allinone.py => ansible_inventory.py} | 40 +++++++++++++------ kolla_cli/common/inventory.py | 23 ++++++----- .../tests/functional/inventory_test_file | 5 +++ .../tests/functional/test_client_upgrade.py | 6 +-- kolla_cli/tests/functional/test_config.py | 16 +++++++- kolla_cli/tests/functional/test_deploy.py | 6 +-- kolla_cli/tests/functional/test_group.py | 8 ++-- kolla_cli/tests/functional/test_service.py | 32 +++++++-------- tools/kolla_actions.py | 11 ++--- 11 files changed, 136 insertions(+), 59 deletions(-) rename kolla_cli/common/{allinone.py => ansible_inventory.py} (75%) create mode 100644 kolla_cli/tests/functional/inventory_test_file diff --git a/kolla_cli/api/config.py b/kolla_cli/api/config.py index 9aa2451..93b52ee 100644 --- a/kolla_cli/api/config.py +++ b/kolla_cli/api/config.py @@ -11,20 +11,38 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import os + import kolla_cli.i18n as u from kolla_cli.api.exceptions import FailedOperation +from kolla_cli.api.exceptions import InvalidArgument +from kolla_cli.common.inventory import Inventory from kolla_cli.common import utils +from kolla_cli.common.utils import check_arg class ConfigApi(object): @staticmethod - def config_reset(): + def config_reset(inventory_path=None): + # type: (str) -> None """Config Reset. - Resets the kolla-ansible configuration to its release defaults. + Resets the kolla-ansible configuration to its release defaults. If + an inventory path is provided, the inventory file will be imported + after the reset, + + :param inventory_path: absolute path to inventory file to import + :type inventory_path: string """ + if inventory_path: + check_arg(inventory_path, u._('Inventory path'), str) + if not os.path.isfile(inventory_path): + raise InvalidArgument( + u._('Inventory file {path} is not valid.').format( + path=inventory_path)) + actions_path = utils.get_kolla_actions_path() cmd = ('%s config_reset' % actions_path) err_msg, output = utils.run_cmd(cmd, print_output=False) @@ -32,3 +50,7 @@ class ConfigApi(object): raise FailedOperation( u._('Configuration reset failed. {error} {message}') .format(error=err_msg, message=output)) + + if inventory_path: + inventory = Inventory(inventory_path) + Inventory.save(inventory) diff --git a/kolla_cli/commands/config.py b/kolla_cli/commands/config.py index c6abb6d..fe33fa2 100644 --- a/kolla_cli/commands/config.py +++ b/kolla_cli/commands/config.py @@ -14,6 +14,8 @@ import logging import traceback +import kolla_cli.i18n as u + from cliff.command import Command from kolla_cli.api.client import ClientApi @@ -23,10 +25,26 @@ LOG = logging.getLogger(__name__) class ConfigReset(Command): - """Resets the kolla-ansible configuration to its release defaults.""" + """Resets the kolla-ansible configuration + + The properties and inventory will be reset to their original + values. If an inventory path is provided, the groups, + hosts, and host vars in the provided inventory file will be + imported into the kolla-cli inventory file. + """ + + def get_parser(self, prog_name): + parser = super(ConfigReset, self).get_parser(prog_name) + parser.add_argument('--inventory', nargs='?', + metavar='', + help=u._('Path to inventory file')) + return parser def take_action(self, parsed_args): try: - CLIENT.config_reset() + inventory_path = None + if parsed_args.inventory: + inventory_path = parsed_args.inventory.strip() + CLIENT.config_reset(inventory_path) except Exception: raise Exception(traceback.format_exc()) diff --git a/kolla_cli/common/allinone.py b/kolla_cli/common/ansible_inventory.py similarity index 75% rename from kolla_cli/common/allinone.py rename to kolla_cli/common/ansible_inventory.py index 17937cd..d719005 100644 --- a/kolla_cli/common/allinone.py +++ b/kolla_cli/common/ansible_inventory.py @@ -15,21 +15,22 @@ import os from kolla_cli.common.service import Service - from kolla_cli.common.utils import get_kolla_ansible_home -class AllInOne(object): - """AllInOne helper class +class AnsibleInventory(object): + """AnsibleInventory helper class - This class parses the kolla all-in-one file and provides an + This class parses an anisble inventory file and provides an easier to use way to represent that file. """ - def __init__(self): + def __init__(self, inventory_path=None): self.groups = [] self.services = {} + self.group_vars = {} + self.host_vars = {} - self._load() + self._load(inventory_path) def add_service(self, servicename): if servicename not in self.services: @@ -41,8 +42,8 @@ class AllInOne(object): if groupname not in self.groups: self.groups.append(groupname) - def _load(self): - """load all-in-one inventory file + def _load(self, inventory_path): + """load an ansible inventory file Note: This assumes that there will be a blank line between each section: @@ -54,10 +55,11 @@ class AllInOne(object): [mistral-executor:children] mistral """ - allinone_path = os.path.join(get_kolla_ansible_home(), 'ansible', - 'inventory', - 'all-in-one') - with open(allinone_path, 'r') as ain1: + if not inventory_path: + inventory_path = os.path.join( + get_kolla_ansible_home(), 'ansible', 'inventory', + 'all-in-one') + with open(inventory_path, 'r') as ain1: ain1_inv = ain1.read() lines = [x for x in ain1_inv.split('\n') if not x.startswith('#')] @@ -66,6 +68,12 @@ class AllInOne(object): if not line.startswith('['): continue line.strip() + if ':vars' in line: + # this is defining a set of group vars in the next set + # of lines. + groupname = line.split(':')[0] + i = self._process_group_vars(groupname, lines, i) + continue if ':children' not in line: groupname = line[1:len(line) - 1] self.add_group(groupname) @@ -103,3 +111,11 @@ class AllInOne(object): parent = self.services[parentname] if parent: parent.add_childname(service.name) + + def _process_group_vars(self, groupname, lines, i): + # currently group vars are ignored + while True: + i += 1 + line = lines[i].strip() + if not line: + break diff --git a/kolla_cli/common/inventory.py b/kolla_cli/common/inventory.py index b7d4666..0ecfe24 100644 --- a/kolla_cli/common/inventory.py +++ b/kolla_cli/common/inventory.py @@ -28,7 +28,7 @@ from kolla_cli.api.exceptions import InvalidArgument from kolla_cli.api.exceptions import InvalidConfiguration from kolla_cli.api.exceptions import MissingArgument from kolla_cli.api.exceptions import NotInInventory -from kolla_cli.common.allinone import AllInOne +from kolla_cli.common.ansible_inventory import AnsibleInventory from kolla_cli.common.host import Host from kolla_cli.common.host_group import HostGroup from kolla_cli.common.service import Service @@ -79,7 +79,7 @@ class Inventory(object): 2: (v2.1.1) added ceilometer 1: (v2.0.1) initial release """ - def __init__(self): + def __init__(self, inventory_path=None): self._groups = {} # kv = name:object self._hosts = {} # kv = name:object self._services = {} # kv = name:object @@ -87,8 +87,9 @@ class Inventory(object): self.version = self.__class__.class_version self.remote_mode = True - # initialize the inventory to its defaults - self._create_default_inventory() + # initialize the inventory to its defaults as they are + # described in the kolla-ansible all-in-one inventory file. + self.import_inventory(inventory_path) def upgrade(self): # check for new services or subservices in the all-in-one file @@ -101,16 +102,16 @@ class Inventory(object): Inventory.save(self) def _upgrade_services(self): - allinone = AllInOne() + ansible_inventory = AnsibleInventory() # add new services - for servicename, service in allinone.services.items(): + for servicename, service in ansible_inventory.services.items(): if servicename not in self._services: self._services[servicename] = service # remove obsolete services for servicename in copy(self._services).keys(): - if servicename not in allinone.services: + if servicename not in ansible_inventory.services: self.delete_service(servicename) @staticmethod @@ -665,9 +666,9 @@ class Inventory(object): filtered_service_dict[service.name] = service return filtered_service_dict - def _create_default_inventory(self): - allin1 = AllInOne() - for groupname in allin1.groups: + def import_inventory(self, inventory_path): + ansible_inventory = AnsibleInventory(inventory_path) + for groupname in ansible_inventory.groups: self.add_group(groupname) - for servicename, service in allin1.services.items(): + for servicename, service in ansible_inventory.services.items(): self._services[servicename] = service diff --git a/kolla_cli/tests/functional/inventory_test_file b/kolla_cli/tests/functional/inventory_test_file new file mode 100644 index 0000000..41f987d --- /dev/null +++ b/kolla_cli/tests/functional/inventory_test_file @@ -0,0 +1,5 @@ +# Test ansible inventory +[aardvark] + +[chipmunk] + diff --git a/kolla_cli/tests/functional/test_client_upgrade.py b/kolla_cli/tests/functional/test_client_upgrade.py index eef3fa6..2c12e86 100644 --- a/kolla_cli/tests/functional/test_client_upgrade.py +++ b/kolla_cli/tests/functional/test_client_upgrade.py @@ -20,7 +20,7 @@ import shutil import unittest from kolla_cli.api.client import ClientApi -from kolla_cli.common.allinone import AllInOne +from kolla_cli.common.ansible_inventory import AnsibleInventory from kolla_cli.common.utils import get_kolla_cli_etc INV_NAME = 'inventory.json' @@ -109,10 +109,10 @@ class TestFunctional(KollaCliTest): # in v1 > v2, ceilometer was added, check that it's there # and verify that all ceilometer groups are in the same groups # as heat. - allinone = AllInOne() + ansible_inventory = AnsibleInventory() heat = CLIENT.service_get(['heat'])[0] expected_groups = sorted(heat.get_groups()) - ceilometer = allinone.services['ceilometer'] + ceilometer = ansible_inventory.services['ceilometer'] expected_services = ceilometer.get_sub_servicenames() expected_services.append('ceilometer') expected_services = sorted(expected_services) diff --git a/kolla_cli/tests/functional/test_config.py b/kolla_cli/tests/functional/test_config.py index 0f92e00..ae05d23 100644 --- a/kolla_cli/tests/functional/test_config.py +++ b/kolla_cli/tests/functional/test_config.py @@ -12,9 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. # -import kolla_cli.api.properties +import os import unittest +import kolla_cli.api.properties + from kolla_cli.api.client import ClientApi from kolla_cli.tests.functional.common import KollaCliTest @@ -96,7 +98,19 @@ class TestFunctional(KollaCliTest): self.assertIs(set(host_list).issubset(fetched_list), False, 'inventory reset config failed') + # test config reset with a different inventory file + expected_group_names = ['chipmunk', 'aardvark'] + test_inventory_path = os.path.join( + os.getcwd(), 'kolla_cli', 'tests', 'functional', + 'inventory_test_file') + CLIENT.config_reset(inventory_path=test_inventory_path) + groups = CLIENT.group_get_all() + self.assertEqual(len(groups), len(expected_group_names)) + for group in groups: + self.assertIn(group.name, expected_group_names) + # need to populate the password file or many other tests will fail + CLIENT.config_reset() CLIENT.password_init() @staticmethod diff --git a/kolla_cli/tests/functional/test_deploy.py b/kolla_cli/tests/functional/test_deploy.py index 3e50eb2..31d4242 100644 --- a/kolla_cli/tests/functional/test_deploy.py +++ b/kolla_cli/tests/functional/test_deploy.py @@ -15,7 +15,7 @@ from kolla_cli.tests.functional.common import KollaCliTest from kolla_cli.api.client import ClientApi -from kolla_cli.common.allinone import AllInOne +from kolla_cli.common.ansible_inventory import AnsibleInventory from kolla_cli.common.inventory import Inventory import json @@ -46,8 +46,8 @@ class TestFunctional(KollaCliTest): self.assertIn(host1, msg, '%s not in json_gen output: %s' % (host1, msg)) - allinone = AllInOne() - services = allinone.services + ansible_inventory = AnsibleInventory() + services = ansible_inventory.services for servicename, service in services.items(): self.assertIn(servicename, msg, '%s not in json_gen output: %s' % (servicename, msg)) diff --git a/kolla_cli/tests/functional/test_group.py b/kolla_cli/tests/functional/test_group.py index 49fb15f..3d6b18d 100644 --- a/kolla_cli/tests/functional/test_group.py +++ b/kolla_cli/tests/functional/test_group.py @@ -19,7 +19,7 @@ import unittest from kolla_cli.api.client import ClientApi from kolla_cli.api.exceptions import NotInInventory -from kolla_cli.common.allinone import AllInOne +from kolla_cli.common.ansible_inventory import AnsibleInventory CLIENT = ClientApi() @@ -231,14 +231,14 @@ class TestFunctional(KollaCliTest): } } """ - allinone = AllInOne() - groupnames = allinone.groups + ansible_inventory = AnsibleInventory() + groupnames = ansible_inventory.groups groups = {} for groupname in groupnames: groups[groupname] = {'Services': [], 'Hosts': []} - for servicename, service in allinone.services.items(): + for servicename, service in ansible_inventory.services.items(): if groupname in service.get_groupnames(): groups[groupname]['Services'].append(servicename) diff --git a/kolla_cli/tests/functional/test_service.py b/kolla_cli/tests/functional/test_service.py index 4fb6e0b..8b1534c 100644 --- a/kolla_cli/tests/functional/test_service.py +++ b/kolla_cli/tests/functional/test_service.py @@ -18,7 +18,7 @@ import json import unittest from kolla_cli.api.client import ClientApi -from kolla_cli.common.allinone import AllInOne +from kolla_cli.common.ansible_inventory import AnsibleInventory from kolla_cli.common.inventory import Inventory CLIENT = ClientApi() @@ -47,18 +47,18 @@ class TestFunctional(KollaCliTest): """ msg = self.run_cli_cmd('service list -f json') cli_services = json.loads(msg) - allinone = AllInOne() - allinone_services = [] - allinone_service_names = [] - for service in allinone.services.values(): + ansible_inventory = AnsibleInventory() + ansible_inventory_services = [] + ansible_inventory_service_names = [] + for service in ansible_inventory.services.values(): if service.is_supported(): - allinone_services.append(service) - allinone_service_names.append(service.name) - num_services = len(allinone_services) + ansible_inventory_services.append(service) + ansible_inventory_service_names.append(service.name) + num_services = len(ansible_inventory_services) self.assertEqual(num_services, len(cli_services), '# of cli services != expected services.' + '\n\nexpected services: %s' - % allinone_service_names + + % ansible_inventory_service_names + '\n\ncli services: %s' % cli_services) def test_listgroups(self): @@ -81,14 +81,14 @@ class TestFunctional(KollaCliTest): msg = self.run_cli_cmd('service listgroups -f json') cli_services = json.loads(msg) - allinone = AllInOne() - allinone_services = [] - allinone_service_names = [] - for service in allinone.services.values(): + ansible_inventory = AnsibleInventory() + ansible_inventory_services = [] + ansible_inventory_service_names = [] + for service in ansible_inventory.services.values(): if service.is_supported(): - allinone_services.append(service) - allinone_service_names.append(service.name) - num_services = len(allinone_services) + ansible_inventory_services.append(service) + ansible_inventory_service_names.append(service.name) + num_services = len(ansible_inventory_services) self.assertEqual(num_services, len(cli_services), '# of cli services (%s) ' % len(cli_services) + '!= # of expected services (%s).' % num_services + diff --git a/tools/kolla_actions.py b/tools/kolla_actions.py index ae4794c..f9319ae 100755 --- a/tools/kolla_actions.py +++ b/tools/kolla_actions.py @@ -186,16 +186,17 @@ def _config_reset_cmd(): clear_all_passwords() # nuke all files under the kolla etc base, skipping everything - # in the kollacli directory and the passwords.yml file + # in the kolla-cli directory and the globals.yml and passwords.yml files for dir_path, dir_names, file_names in os.walk(kolla_etc, topdown=False): - if 'kollacli' not in dir_path: + if 'kolla-cli' not in dir_path: for dir_name in dir_names: - if dir_name != 'kollacli': + if dir_name != 'kolla-cli': os.rmdir(os.path.join(dir_path, dir_name)) for file_name in file_names: - if file_name != 'passwords.yml': - os.remove(os.path.join(dir_path, file_name)) + if file_name == 'passwords.yml' or file_name == 'globals.yml': + continue + os.remove(os.path.join(dir_path, file_name)) # nuke all property files under the kolla-ansible base other than # all.yml and the global property file which we truncate above