Consolidating path resolution and searching for program files
- This commit is refactoring only, and attempts to avoid modifying functionality. Not a blueprint nor bug fix. - The goal is to be more clear and explicit when searching for program files. - The current kolla-cli client searches across many paths to find services, bootstrap, and configuation files. The changes in this git commit refactor this functionality into a single objectified PathFinder class. - It tries to make consistent and clear how it searches various locations to find service and bootstrap files. - The current code when locating service and bootstrap files locks onto the first path that exists. This code does not change that. - The current code when locating config files locks onto the first path where the file exists. This code does not change that. - Made kolla-kubernetes search paths work with editable development installs. - Added a bootstrap-dir in addition to services-dir as a command line override. Hoping to remove both in the future if people are okay with it. - Refactored kolla_kubernetes/common/file_utils.py into pathfinder.py, attempting to de-dup code paths. - Added a new KollaFileNotFoundException. - I'm happy to modify the search order and remove search paths in subsequent pull requests. The less variations we must support the better. Change-Id: I95e2423b0f29e8b3908b1a22ee68c5decd6d9fee
This commit is contained in:
David C Wang
parent
6d2d006457
commit
59ca222550
|
|
@ -44,7 +44,10 @@ log.set_defaults(
|
|||
cli_opts = [
|
||||
cfg.StrOpt('service-dir',
|
||||
default=utils.env('K8S_SERVICE_DIR'),
|
||||
help='Directory with services, (Env: K8S_SERVICE_DIR)'),
|
||||
help='kolla-kubernetes services dir (Env: K8S_SERVICE_DIR)'),
|
||||
cfg.StrOpt('bootstrap-dir',
|
||||
default=utils.env('K8S_BOOTSTRAP_DIR'),
|
||||
help='kolla-kubernetes bootstrap dir (Env: K8S_BOOTSTRAP_DIR)'),
|
||||
]
|
||||
CONF.register_cli_opts(cli_opts)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,75 +0,0 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from oslo_utils import importutils
|
||||
|
||||
from kolla_kubernetes import exception
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def find_config_file(filename):
|
||||
filepath = os.path.join('/etc/kolla', filename)
|
||||
if os.access(filepath, os.R_OK):
|
||||
return filepath
|
||||
|
||||
filepath = os.path.join('/etc/kolla-kubernetes', filename)
|
||||
if os.access(filepath, os.R_OK):
|
||||
return filepath
|
||||
|
||||
raise exception.KollaDirNotFoundException(
|
||||
'Unable to detect kolla-kubernetes directory'
|
||||
)
|
||||
|
||||
|
||||
def get_service_config_files(service_name):
|
||||
directory = os.path.join('/etc/kolla/', service_name)
|
||||
for dirpath, _, filenames in os.walk(directory):
|
||||
for f in filenames:
|
||||
yield os.path.abspath(os.path.join(dirpath, f))
|
||||
|
||||
|
||||
def get_src_dir():
|
||||
kolla_kubernetes = importutils.import_module('kolla_kubernetes')
|
||||
mod_path = os.path.abspath(kolla_kubernetes.__file__)
|
||||
# remove the file and module to get to the base.
|
||||
return os.path.dirname(os.path.dirname(mod_path))
|
||||
|
||||
|
||||
def get_shared_directory():
|
||||
script_path = os.path.abspath(os.path.join(os.path.dirname(
|
||||
os.path.realpath(sys.argv[0])), '../..'))
|
||||
base_script_path = os.path.basename(script_path)
|
||||
|
||||
if base_script_path == 'kolla':
|
||||
return script_path
|
||||
elif os.path.exists('/usr/local/share/kolla'):
|
||||
return '/usr/local/share/kolla'
|
||||
elif os.path.exists('/usr/share/kolla'):
|
||||
return '/usr/share/kolla'
|
||||
raise exception.KollaDirNotFoundException(
|
||||
'Unable to detect kolla-kubernetes directory'
|
||||
)
|
||||
|
||||
|
||||
def find_base_dir():
|
||||
if os.path.exists('/usr/local/share/kolla-kubernetes'):
|
||||
return '/usr/local/share/kolla-kubernetes'
|
||||
elif os.path.exists('/usr/share/kolla-kubernetes'):
|
||||
return '/usr/share/kolla-kubernetes'
|
||||
else:
|
||||
return get_src_dir()
|
||||
|
|
@ -0,0 +1,132 @@
|
|||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
from kolla_kubernetes import exception
|
||||
from oslo_config import cfg
|
||||
|
||||
CONF = cfg.CONF
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class PathFinder(object):
|
||||
|
||||
@staticmethod
|
||||
def find_installed_root():
|
||||
# Full installs use this root path to locate ./share/kolla
|
||||
# For system, resolves to /usr/local
|
||||
# For virtualenv, resolves to /path/to/venv
|
||||
return os.path.abspath(os.path.join(os.path.dirname(
|
||||
os.path.realpath(sys.argv[0])), '../'))
|
||||
|
||||
@staticmethod
|
||||
def find_development_root():
|
||||
# Editable installs (aka. Development: pip install --editable .)
|
||||
# use this root path to locate ../kolla
|
||||
# For editable, resolves to /path/to/git/repo/kolla-kubernetes
|
||||
return os.path.dirname(os.path.dirname(
|
||||
os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
@staticmethod
|
||||
def find_kolla_dir():
|
||||
return PathFinder._find_dir(KOLLA_SEARCH_PATHS, None)
|
||||
|
||||
@staticmethod
|
||||
def find_kolla_service_config_files(service_name):
|
||||
path = PathFinder.find_service_config_dir(service_name)
|
||||
return PathFinder._list_dir_files(path)
|
||||
|
||||
@staticmethod
|
||||
def find_config_file(filename):
|
||||
search_paths = CONFIG_SEARCH_PATHS
|
||||
for d in search_paths:
|
||||
f = os.path.join(d, filename)
|
||||
if os.path.isfile(f):
|
||||
return f
|
||||
raise exception.KollaFileNotFoundException(
|
||||
"Unable to locate file=[{}] in search_paths=[{}]".format(
|
||||
filename, ", ".join(search_paths))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def find_service_config_dir(service_name):
|
||||
return PathFinder._find_dir(CONFIG_SEARCH_PATHS, service_name)
|
||||
|
||||
@staticmethod
|
||||
def find_service_dir():
|
||||
if CONF.service_dir:
|
||||
return CONF.service_dir
|
||||
return PathFinder._find_dir(KOLLA_KUBERNETES_SEARCH_PATHS, 'services')
|
||||
|
||||
@staticmethod
|
||||
def find_bootstrap_dir():
|
||||
if CONF.bootstrap_dir:
|
||||
return CONF.bootstrap_dir
|
||||
return PathFinder._find_dir(KOLLA_KUBERNETES_SEARCH_PATHS, 'bootstrap')
|
||||
|
||||
@staticmethod
|
||||
def _find_dir(search_paths, dir_name):
|
||||
# returns the first directory that exists
|
||||
for path in search_paths:
|
||||
p = path
|
||||
if dir_name is not None:
|
||||
p = os.path.join(path, dir_name)
|
||||
if os.path.isdir(p):
|
||||
return p
|
||||
raise exception.KollaDirNotFoundException(
|
||||
"Unable to locate {} directory in search_paths=[{}]".format(
|
||||
dir_name, ", ".join(search_paths))
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _list_dir_files(path):
|
||||
paths = [os.path.join(path, fn) for fn in next(os.walk(path))[2]]
|
||||
return paths
|
||||
|
||||
|
||||
# prioritize directories to search for /etc files
|
||||
CONFIG_SEARCH_PATHS = [
|
||||
# Search installation paths first
|
||||
'/etc/kolla',
|
||||
'/etc/kolla-kubernetes',
|
||||
# Then development paths
|
||||
os.path.abspath(os.path.join(PathFinder.find_development_root(),
|
||||
'../kolla/etc/kolla')),
|
||||
os.path.abspath(os.path.join(PathFinder.find_development_root(),
|
||||
'./etc/kolla-kubernetes')),
|
||||
]
|
||||
|
||||
# prioritize directories to search for kolla sources
|
||||
KOLLA_SEARCH_PATHS = [
|
||||
# Search installation paths first
|
||||
os.path.abspath(os.path.join(PathFinder.find_installed_root(),
|
||||
'./share/kolla')),
|
||||
'/usr/share/kolla',
|
||||
# Then search development paths
|
||||
os.path.abspath(os.path.join(PathFinder.find_development_root(),
|
||||
'../kolla')),
|
||||
os.path.abspath(os.path.join(PathFinder.find_development_root(),
|
||||
'kolla')),
|
||||
]
|
||||
|
||||
# prioritize directories to search for kolla-kubernetes sources
|
||||
KOLLA_KUBERNETES_SEARCH_PATHS = [
|
||||
# Search installation paths first
|
||||
os.path.abspath(os.path.join(PathFinder.find_installed_root(),
|
||||
'./share/kolla-kubernetes')),
|
||||
'/usr/share/kolla-kubernetes',
|
||||
# Then search development paths
|
||||
os.path.abspath(os.path.join(PathFinder.find_development_root())),
|
||||
]
|
||||
|
|
@ -19,6 +19,10 @@ class KollaDirNotFoundException(KollaException):
|
|||
pass
|
||||
|
||||
|
||||
class KollaFileNotFoundException(KollaException):
|
||||
pass
|
||||
|
||||
|
||||
class KollaNotFoundException(KollaException):
|
||||
def __init__(self, message, entity='file'):
|
||||
super(KollaNotFoundException, self).__init__(
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ from oslo_config import cfg
|
|||
from oslo_log import log as logging
|
||||
import yaml
|
||||
|
||||
from kolla_kubernetes.common import file_utils
|
||||
from kolla_kubernetes.common.pathfinder import PathFinder
|
||||
|
||||
from kolla_kubernetes.common import jinja_utils
|
||||
from kolla_kubernetes.common import utils
|
||||
from kolla_kubernetes import service_definition
|
||||
|
|
@ -46,24 +47,22 @@ def _create_working_directory(target='services'):
|
|||
|
||||
def _load_variables_from_file(project_name):
|
||||
jvars = utils.JvarsDict()
|
||||
f = file_utils.find_config_file('kolla-kubernetes.yml')
|
||||
if os.path.exists(f):
|
||||
with open(f, 'r') as gf:
|
||||
jvars.set_global_vars(yaml.load(gf))
|
||||
else:
|
||||
LOG.warning('Unable to load %s', f)
|
||||
f = file_utils.find_config_file('globals.yml')
|
||||
if os.path.exists(f):
|
||||
with open(f, 'r') as gf:
|
||||
jvars.set_global_vars(yaml.load(gf))
|
||||
else:
|
||||
LOG.warning('Unable to load %s', f)
|
||||
f = file_utils.find_config_file('passwords.yml')
|
||||
|
||||
for file_ in ['kolla-kubernetes.yml', 'globals.yml']:
|
||||
f = PathFinder.find_config_file(file_)
|
||||
if os.path.exists(f):
|
||||
with open(f, 'r') as gf:
|
||||
jvars.set_global_vars(yaml.load(gf))
|
||||
else:
|
||||
LOG.warning('Unable to load %s', f)
|
||||
|
||||
f = PathFinder.find_config_file('passwords.yml')
|
||||
if os.path.exists(f):
|
||||
with open(f, 'r') as gf:
|
||||
jvars.update(yaml.load(gf))
|
||||
else:
|
||||
LOG.warning('Unable to load %s', f)
|
||||
|
||||
# Apply the basic variables that aren't defined in any config file.
|
||||
jvars.update({
|
||||
'deployment_id': CONF.kolla.deployment_id,
|
||||
|
|
@ -71,7 +70,7 @@ def _load_variables_from_file(project_name):
|
|||
'timestamp': str(time.time())
|
||||
})
|
||||
|
||||
dir = file_utils.get_shared_directory()
|
||||
dir = PathFinder.find_kolla_dir()
|
||||
all_yml = os.path.join(dir, 'ansible/group_vars/all.yml')
|
||||
local_dir = os.path.join(PROJECT_ROOT, 'kolla/ansible/')
|
||||
|
||||
|
|
@ -188,7 +187,7 @@ def _bootstrap_instance(directory, service_name):
|
|||
for container in container_list:
|
||||
cmd = [CONF.kolla_kubernetes.kubectl_path, server, "create",
|
||||
"configmap", '%s-configmap' % container]
|
||||
for f in file_utils.get_service_config_files(container):
|
||||
for f in PathFinder.find_kolla_service_config_files(container):
|
||||
cmd = cmd + ['--from-file=%s=%s' % (
|
||||
os.path.basename(f).replace("_", "-"), f)]
|
||||
|
||||
|
|
@ -211,7 +210,7 @@ def _deploy_instance(directory, service_name, pod_list):
|
|||
for container in container_list:
|
||||
cmd = [CONF.kolla_kubernetes.kubectl_path, server, "create",
|
||||
"configmap", '%s-configmap' % container]
|
||||
for f in file_utils.get_service_config_files(container):
|
||||
for f in PathFinder.find_kolla_service_config_files(container):
|
||||
cmd = cmd + ['--from-file=%s=%s' % (
|
||||
os.path.basename(f).replace("_", "-"), f)]
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import os.path
|
|||
from oslo_config import cfg
|
||||
from oslo_log import log
|
||||
|
||||
from kolla_kubernetes.common import file_utils
|
||||
from kolla_kubernetes.common.pathfinder import PathFinder
|
||||
from kolla_kubernetes import exception
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
|
@ -91,13 +91,8 @@ def get_container_definition(container):
|
|||
return CONTAINER_DEFINITIONS[container]
|
||||
|
||||
|
||||
def get_services_directory():
|
||||
return (CONF.service_dir or
|
||||
os.path.join(file_utils.find_base_dir(), 'services'))
|
||||
|
||||
|
||||
def find_bootstrap_files(service_name):
|
||||
bootstrap_dir = os.path.join(get_services_directory(), '../bootstrap/')
|
||||
bootstrap_dir = os.path.join(PathFinder.find_bootstrap_dir(), service_name)
|
||||
LOG.debug('Looking for bootstrap files in %s', bootstrap_dir)
|
||||
if not os.path.exists(bootstrap_dir):
|
||||
LOG.info('No bootstrap job for service %s', service_name)
|
||||
|
|
@ -113,7 +108,7 @@ def find_bootstrap_files(service_name):
|
|||
|
||||
|
||||
def find_service_files(service_name):
|
||||
service_dir = get_services_directory()
|
||||
service_dir = os.path.join(PathFinder.find_service_dir(), service_name)
|
||||
LOG.debug('Looking for services files in %s', service_dir)
|
||||
|
||||
if not os.path.exists(service_dir):
|
||||
|
|
|
|||
|
|
@ -10,12 +10,26 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from kolla_kubernetes.common import file_utils
|
||||
from kolla_kubernetes.common.pathfinder import PathFinder
|
||||
from kolla_kubernetes.tests import base
|
||||
|
||||
|
||||
class FindBaseDirTest(base.BaseTestCase):
|
||||
|
||||
def test_when_is_a_test(self):
|
||||
tdir = file_utils.find_base_dir()
|
||||
self.assertIsNotNone(tdir)
|
||||
def test_find_installed_root(self):
|
||||
d = PathFinder.find_installed_root()
|
||||
|
||||
# check for non-null
|
||||
self.assertIsNotNone(d)
|
||||
|
||||
# check that project_root is not empty
|
||||
self.assertTrue(len(d) > 0)
|
||||
|
||||
def test_find_development_root(self):
|
||||
d = PathFinder.find_development_root()
|
||||
|
||||
# check for non-null
|
||||
self.assertIsNotNone(d)
|
||||
|
||||
# check that project_root is not empty
|
||||
self.assertTrue(len(d) > 0)
|
||||
Loading…
Reference in New Issue