Refactor configuration to use dynaconf
This changes the configuration engine from everett to dynaconf. dynaconf allows loading configuration from files (json, ini, yaml, toml) as well as environment variables prefixed by ARA_. Our usage of dynaconf is similar to the use case from the Pulp [1] project and they have documented an issue when loading database parameters [2]. This issue is worked around by importing dynaconf in the different entry points. This introduces some other changes as well: - We're now creating a default configuration and data directory at ~/.ara. The location of this directory is controlled with the ARA_BASE_DIR environment variable. - We're now creating a default configuration template in ~/.ara/default_config.yaml. - The default database is now located at ~/.ara/ara.sqlite. The location of this database can be customized with the ARA_DATABASE_NAME environment variable. Note that ARA 0.x used "~/.ara/ansible.sqlite" -- the file name change is deliberate in order to avoid user databases clashing between versions. More documentation on this will be available in an upcoming patch. [1]: https://github.com/pulp/pulp [2]: https://github.com/rochacbruno/dynaconf/issues/89 Change-Id: I8178b4ca9f2b4d7f4c45c296c08391e84e8b990d
This commit is contained in:
David Moreau Simard
Florian Apolloner
parent
6c00d7552b
commit
16aa41eaf8
|
|
@ -4,10 +4,9 @@ import sys
|
|||
|
||||
|
||||
def main():
|
||||
from ara import server
|
||||
|
||||
os.environ.setdefault("ARA_CFG", os.path.dirname(server.__file__) + "/configs/dev.cfg")
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings")
|
||||
# https://github.com/rochacbruno/dynaconf/issues/89
|
||||
from dynaconf.contrib import django_dynaconf # noqa
|
||||
|
||||
from django.core.management import execute_from_command_line
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +0,0 @@
|
|||
[ara]
|
||||
debug = true
|
||||
secret_key = dev
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
[ara]
|
||||
debug = true
|
||||
log_level = DEBUG
|
||||
secret_key = integration
|
||||
allowed_hosts = localhost
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
[ara]
|
||||
secret_key = test
|
||||
|
|
@ -1,20 +1,47 @@
|
|||
import os
|
||||
import sys
|
||||
import textwrap
|
||||
|
||||
from .utils import EverettEnviron
|
||||
import yaml
|
||||
|
||||
env = EverettEnviron(DEBUG=(bool, False))
|
||||
# Ensure default base configuration/data directory exists
|
||||
BASE_DIR = os.environ.get("ARA_BASE_DIR", os.path.expanduser("~/.ara"))
|
||||
SERVER_DIR = os.path.join(BASE_DIR, "server")
|
||||
if not os.path.isdir(SERVER_DIR):
|
||||
os.makedirs(SERVER_DIR, mode=0o700)
|
||||
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
SECRET_KEY = env("SECRET_KEY")
|
||||
|
||||
DEBUG = env.bool("DEBUG", default=False)
|
||||
|
||||
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[])
|
||||
# Django built-in server and npm development server
|
||||
ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
|
||||
CORS_ORIGIN_WHITELIST = ["127.0.0.1:8000", "localhost:3000"]
|
||||
CORS_ORIGIN_ALLOW_ALL = True
|
||||
|
||||
ADMINS = ()
|
||||
|
||||
# Dynaconf Configuration
|
||||
SECRET_KEY = True
|
||||
GLOBAL_ENV_FOR_DYNACONF = "ARA"
|
||||
ENVVAR_FOR_DYNACONF = "ARA_SETTINGS"
|
||||
SETTINGS_MODULE_FOR_DYNACONF = "ara.server.settings"
|
||||
|
||||
# We're not expecting ARA to use multiple concurrent databases.
|
||||
# Make it easier for users to specify the configuration for a single database.
|
||||
DATABASE_ENGINE = os.environ.get("ARA_DATABASE_ENGINE", "django.db.backends.sqlite3")
|
||||
DATABASE_NAME = os.environ.get("ARA_DATABASE_NAME", os.path.join(SERVER_DIR, "ansible.sqlite"))
|
||||
DATABASE_USER = os.environ.get("ARA_DATABASE_USER", None)
|
||||
DATABASE_PASSWORD = os.environ.get("ARA_DATABASE_PASSWORD", None)
|
||||
DATABASE_HOST = os.environ.get("ARA_DATABASE_HOST", None)
|
||||
DATABASE_PORT = os.environ.get("ARA_DATABASE_PORT", None)
|
||||
|
||||
DATABASES = {
|
||||
"default": {
|
||||
"ENGINE": DATABASE_ENGINE,
|
||||
"NAME": DATABASE_NAME,
|
||||
"USER": DATABASE_USER,
|
||||
"PASSWORD": DATABASE_PASSWORD,
|
||||
"HOST": DATABASE_HOST,
|
||||
"PORT": DATABASE_PORT,
|
||||
}
|
||||
}
|
||||
|
||||
INSTALLED_APPS = [
|
||||
"django.contrib.auth",
|
||||
"django.contrib.contenttypes",
|
||||
|
|
@ -39,15 +66,6 @@ MIDDLEWARE = [
|
|||
"django.middleware.clickjacking.XFrameOptionsMiddleware",
|
||||
]
|
||||
|
||||
# TODO: Only needed in dev?
|
||||
CORS_ORIGIN_ALLOW_ALL = True
|
||||
|
||||
# Django built-in server and npm development server
|
||||
CORS_ORIGIN_WHITELIST = ("127.0.0.1:8000", "localhost:3000")
|
||||
|
||||
ROOT_URLCONF = "ara.server.urls"
|
||||
APPEND_SLASH = False
|
||||
|
||||
TEMPLATES = [
|
||||
{
|
||||
"BACKEND": "django.template.backends.django.DjangoTemplates",
|
||||
|
|
@ -64,10 +82,6 @@ TEMPLATES = [
|
|||
}
|
||||
]
|
||||
|
||||
WSGI_APPLICATION = "ara.server.wsgi.application"
|
||||
|
||||
DATABASES = {"default": env.db(default="sqlite:///%s" % os.path.join(BASE_DIR, "db.sqlite3"))}
|
||||
|
||||
AUTH_PASSWORD_VALIDATORS = [
|
||||
{"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"},
|
||||
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"},
|
||||
|
|
@ -75,51 +89,22 @@ AUTH_PASSWORD_VALIDATORS = [
|
|||
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"},
|
||||
]
|
||||
|
||||
LANGUAGE_CODE = "en-us"
|
||||
|
||||
USE_TZ = True
|
||||
TIME_ZONE = "UTC"
|
||||
|
||||
USE_I18N = True
|
||||
|
||||
USE_L10N = True
|
||||
|
||||
USE_TZ = True
|
||||
LANGUAGE_CODE = "en-us"
|
||||
|
||||
STATIC_URL = "/static/"
|
||||
|
||||
STATIC_ROOT = os.path.join(BASE_DIR, "www", "static")
|
||||
STATIC_ROOT = os.path.join(SERVER_DIR, "www", "static")
|
||||
|
||||
MEDIA_URL = "/media/"
|
||||
MEDIA_ROOT = os.path.join(SERVER_DIR, "www", "media")
|
||||
|
||||
MEDIA_ROOT = os.path.join(BASE_DIR, "www", "media")
|
||||
|
||||
# fmt: off
|
||||
LOGGING = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}},
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "normal",
|
||||
"level": env("LOG_LEVEL", default="INFO"),
|
||||
"stream": sys.stdout,
|
||||
}
|
||||
},
|
||||
"loggers": {
|
||||
"ara": {
|
||||
"handlers": ["console"],
|
||||
"level": env("LOG_LEVEL", default="INFO"),
|
||||
"propagate": 0
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"handlers": ["console"],
|
||||
"level": env("LOG_LEVEL", default="DEBUG")
|
||||
},
|
||||
}
|
||||
# fmt: on
|
||||
|
||||
WSGI_APPLICATION = "ara.server.wsgi.application"
|
||||
ROOT_URLCONF = "ara.server.urls"
|
||||
APPEND_SLASH = False
|
||||
REST_FRAMEWORK = {
|
||||
"DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination",
|
||||
"PAGE_SIZE": 100,
|
||||
|
|
@ -135,3 +120,62 @@ REST_FRAMEWORK = {
|
|||
),
|
||||
"TEST_REQUEST_DEFAULT_FORMAT": "json",
|
||||
}
|
||||
|
||||
DEBUG = False
|
||||
LOG_LEVEL = "INFO"
|
||||
# fmt: off
|
||||
LOGGING = {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": False,
|
||||
"formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}},
|
||||
"handlers": {
|
||||
"console": {
|
||||
"class": "logging.StreamHandler",
|
||||
"formatter": "normal",
|
||||
"level": LOG_LEVEL,
|
||||
"stream": "ext://sys.stdout",
|
||||
}
|
||||
},
|
||||
"loggers": {
|
||||
"ara": {
|
||||
"handlers": ["console"],
|
||||
"level": LOG_LEVEL,
|
||||
"propagate": 0
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"handlers": ["console"],
|
||||
"level": LOG_LEVEL
|
||||
},
|
||||
}
|
||||
# fmt: on
|
||||
|
||||
# TODO: Split this out to a CLI command (django-admin command ?)
|
||||
DEFAULT_CONFIG = os.path.join(SERVER_DIR, "default_config.yaml")
|
||||
if not os.path.exists(DEFAULT_CONFIG):
|
||||
CONFIG = dict(
|
||||
BASE_DIR=BASE_DIR,
|
||||
ALLOWED_HOSTS=ALLOWED_HOSTS,
|
||||
CORS_ORIGIN_WHITELIST=CORS_ORIGIN_WHITELIST,
|
||||
CORS_ORIGIN_ALLOW_ALL=CORS_ORIGIN_ALLOW_ALL,
|
||||
SECRET_KEY="please-change-this",
|
||||
DATABASES=DATABASES,
|
||||
STATIC_URL=STATIC_URL,
|
||||
STATIC_ROOT=STATIC_ROOT,
|
||||
MEDIA_URL=MEDIA_URL,
|
||||
MEDIA_ROOT=MEDIA_ROOT,
|
||||
DEBUG=DEBUG,
|
||||
LOG_LEVEL=LOG_LEVEL,
|
||||
LOGGING=LOGGING,
|
||||
)
|
||||
with open(DEFAULT_CONFIG, "w+") as config_file:
|
||||
comment = f"""
|
||||
---
|
||||
# This is a default configuration template generated by ARA.
|
||||
# To use a configuration file such as this one, you need to export the
|
||||
# ARA_SETTINGS configuration variable like so:
|
||||
# $ export ARA_SETTINGS={DEFAULT_CONFIG}
|
||||
|
||||
"""
|
||||
config_file.write(textwrap.dedent(comment))
|
||||
yaml.dump({"default": CONFIG}, config_file, default_flow_style=False)
|
||||
|
|
|
|||
|
|
@ -1,89 +0,0 @@
|
|||
import os
|
||||
|
||||
import environ
|
||||
from configobj import ConfigObj
|
||||
from django.core.exceptions import ImproperlyConfigured
|
||||
from everett import ConfigurationError
|
||||
from everett.manager import ConfigEnvFileEnv, ConfigIniEnv, ConfigManager, ConfigOSEnv, listify
|
||||
|
||||
__all__ = ["EverettEnviron"]
|
||||
|
||||
|
||||
class DumbConfigIniEnv(ConfigIniEnv):
|
||||
"""Simple ConfigIniEnv with disabled list parsing that actually aborts after the first file."""
|
||||
|
||||
# TODO: Remove once upstream is fixed (https://github.com/willkg/everett/pull/71)
|
||||
|
||||
def __init__(self, possible_paths):
|
||||
self.cfg = {}
|
||||
possible_paths = listify(possible_paths)
|
||||
|
||||
for path in possible_paths:
|
||||
if not path:
|
||||
continue
|
||||
|
||||
path = os.path.abspath(os.path.expanduser(path.strip()))
|
||||
if path and os.path.isfile(path):
|
||||
self.cfg = self.parse_ini_file(path)
|
||||
break
|
||||
|
||||
def parse_ini_file(cls, path):
|
||||
cfgobj = ConfigObj(path, list_values=False)
|
||||
|
||||
def extract_section(namespace, d):
|
||||
cfg = {}
|
||||
for key, val in d.items():
|
||||
if isinstance(d[key], dict):
|
||||
cfg.update(extract_section(namespace + [key], d[key]))
|
||||
else:
|
||||
cfg["_".join(namespace + [key]).upper()] = val
|
||||
|
||||
return cfg
|
||||
|
||||
return extract_section([], cfgobj.dict())
|
||||
|
||||
|
||||
class EnvironProxy:
|
||||
def __init__(self, cfg):
|
||||
self.cfg = cfg
|
||||
|
||||
def __contains__(self, key):
|
||||
try:
|
||||
self.cfg(key)
|
||||
except ConfigurationError:
|
||||
return False
|
||||
return True
|
||||
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return self.cfg(key, raw_value=True)
|
||||
except ConfigurationError as err:
|
||||
raise KeyError("Missing key %r" % key) from err
|
||||
|
||||
|
||||
class EverettEnviron(environ.Env):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
||||
self.ENVIRON = EnvironProxy(
|
||||
ConfigManager(
|
||||
[
|
||||
ConfigOSEnv(),
|
||||
ConfigEnvFileEnv(".env"),
|
||||
DumbConfigIniEnv([os.environ.get("ARA_CFG"), "~/.config/ara/server.cfg", "/etc/ara/server.cfg"]),
|
||||
]
|
||||
).with_namespace("ara")
|
||||
)
|
||||
|
||||
def get_value(self, var, cast=None, default=environ.Env.NOTSET, parse_default=False):
|
||||
try:
|
||||
return super().get_value(var, cast, default, parse_default)
|
||||
except ImproperlyConfigured as e:
|
||||
# Rewrite the django-environ exception to match our configs better
|
||||
if default is self.NOTSET and str(e) == "Set the {0} environment variable".format(var):
|
||||
error_msg = (
|
||||
"Set the ARA_{0} environment variable or set {1} in the [ara] section "
|
||||
"of your configuration file."
|
||||
).format(var, var.lower())
|
||||
raise ImproperlyConfigured(error_msg)
|
||||
raise
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
import os
|
||||
|
||||
from django.core.wsgi import get_wsgi_application
|
||||
|
||||
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings")
|
||||
# https://github.com/rochacbruno/dynaconf/issues/89
|
||||
from dynaconf.contrib import django_dynaconf # noqa
|
||||
from django.core.wsgi import get_wsgi_application # noqa
|
||||
|
||||
application = get_wsgi_application()
|
||||
|
|
|
|||
|
|
@ -6,4 +6,4 @@ django-cors-headers
|
|||
drf-extensions
|
||||
django-filter
|
||||
django-environ
|
||||
everett
|
||||
dynaconf
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,ara/api/migrations
|
|||
[isort]
|
||||
known_first_party = ara
|
||||
default_section = THIRDPARTY
|
||||
skip = build,.git,.tox,.cache,.venv,ara/api/migrations
|
||||
skip = build,.git,.tox,.cache,.venv,ara/api/migrations,ara/server/wsgi.py
|
||||
not_skip = __init__.py
|
||||
multi_line_output=3
|
||||
include_trailing_comma=True
|
||||
|
|
|
|||
10
tox.ini
10
tox.ini
|
|
@ -30,6 +30,10 @@ commands =
|
|||
ara-manage migrate
|
||||
ara-manage collectstatic --clear --no-input
|
||||
ara-manage runserver
|
||||
setenv =
|
||||
ARA_DEBUG=true
|
||||
ARA_LOG_LEVEL=DEBUG
|
||||
ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara
|
||||
|
||||
# Temporary venv to help bootstrap integration
|
||||
[testenv:ansible-integration]
|
||||
|
|
@ -37,11 +41,13 @@ deps =
|
|||
git+https://git.openstack.org/openstack/ara-plugins@master#egg=ara_plugins
|
||||
git+https://git.openstack.org/openstack/ara-clients@master#egg=ara_clients
|
||||
commands =
|
||||
rm -f {toxinidir}/db.sqlite3
|
||||
rm -rf {toxinidir}/.tox/ansible-integration/tmp/ara
|
||||
bash -c 'ANSIBLE_CALLBACK_PLUGINS=$(python -c "import os,ara.plugins; print(os.path.dirname(ara.plugins.__file__))")/callback ansible-playbook {toxinidir}/hacking/test-playbook.yml'
|
||||
python {toxinidir}/hacking/validate.py
|
||||
setenv =
|
||||
ARA_CFG={toxinidir}/ara/server/configs/integration.cfg
|
||||
ARA_DEBUG=true
|
||||
ARA_LOG_LEVEL=DEBUG
|
||||
ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara
|
||||
whitelist_externals =
|
||||
rm
|
||||
bash
|
||||
|
|
|
|||
Loading…
Reference in New Issue