From 16aa41eaf88d0abf7f0fa3f2cf0905f9a11f591f Mon Sep 17 00:00:00 2001 From: David Moreau Simard Date: Thu, 13 Dec 2018 14:47:31 -0500 Subject: [PATCH] Refactor configuration to use dynaconf This changes the configuration engine from everett to dynaconf. dynaconf allows loading configuration from files (json, ini, yaml, toml) as well as environment variables prefixed by ARA_. Our usage of dynaconf is similar to the use case from the Pulp [1] project and they have documented an issue when loading database parameters [2]. This issue is worked around by importing dynaconf in the different entry points. This introduces some other changes as well: - We're now creating a default configuration and data directory at ~/.ara. The location of this directory is controlled with the ARA_BASE_DIR environment variable. - We're now creating a default configuration template in ~/.ara/default_config.yaml. - The default database is now located at ~/.ara/ara.sqlite. The location of this database can be customized with the ARA_DATABASE_NAME environment variable. Note that ARA 0.x used "~/.ara/ansible.sqlite" -- the file name change is deliberate in order to avoid user databases clashing between versions. More documentation on this will be available in an upcoming patch. [1]: https://github.com/pulp/pulp [2]: https://github.com/rochacbruno/dynaconf/issues/89 Change-Id: I8178b4ca9f2b4d7f4c45c296c08391e84e8b990d --- ara/server/__main__.py | 5 +- ara/server/configs/dev.cfg | 3 - ara/server/configs/integration.cfg | 5 - ara/server/configs/test.cfg | 2 - ara/server/settings.py | 162 ++++++++++++++++++----------- ara/server/utils.py | 89 ---------------- ara/server/wsgi.py | 5 +- requirements.txt | 2 +- setup.cfg | 2 +- tox.ini | 10 +- 10 files changed, 118 insertions(+), 167 deletions(-) delete mode 100644 ara/server/configs/dev.cfg delete mode 100644 ara/server/configs/integration.cfg delete mode 100644 ara/server/configs/test.cfg diff --git a/ara/server/__main__.py b/ara/server/__main__.py index 9c44113..2ed1602 100644 --- a/ara/server/__main__.py +++ b/ara/server/__main__.py @@ -4,10 +4,9 @@ import sys def main(): - from ara import server - - os.environ.setdefault("ARA_CFG", os.path.dirname(server.__file__) + "/configs/dev.cfg") os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings") + # https://github.com/rochacbruno/dynaconf/issues/89 + from dynaconf.contrib import django_dynaconf # noqa from django.core.management import execute_from_command_line diff --git a/ara/server/configs/dev.cfg b/ara/server/configs/dev.cfg deleted file mode 100644 index c51bdf1..0000000 --- a/ara/server/configs/dev.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[ara] -debug = true -secret_key = dev diff --git a/ara/server/configs/integration.cfg b/ara/server/configs/integration.cfg deleted file mode 100644 index 1458f0a..0000000 --- a/ara/server/configs/integration.cfg +++ /dev/null @@ -1,5 +0,0 @@ -[ara] -debug = true -log_level = DEBUG -secret_key = integration -allowed_hosts = localhost diff --git a/ara/server/configs/test.cfg b/ara/server/configs/test.cfg deleted file mode 100644 index 0f6a133..0000000 --- a/ara/server/configs/test.cfg +++ /dev/null @@ -1,2 +0,0 @@ -[ara] -secret_key = test diff --git a/ara/server/settings.py b/ara/server/settings.py index 03e39e3..8945d5a 100644 --- a/ara/server/settings.py +++ b/ara/server/settings.py @@ -1,20 +1,47 @@ import os -import sys +import textwrap -from .utils import EverettEnviron +import yaml -env = EverettEnviron(DEBUG=(bool, False)) +# Ensure default base configuration/data directory exists +BASE_DIR = os.environ.get("ARA_BASE_DIR", os.path.expanduser("~/.ara")) +SERVER_DIR = os.path.join(BASE_DIR, "server") +if not os.path.isdir(SERVER_DIR): + os.makedirs(SERVER_DIR, mode=0o700) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -SECRET_KEY = env("SECRET_KEY") - -DEBUG = env.bool("DEBUG", default=False) - -ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", default=[]) +# Django built-in server and npm development server +ALLOWED_HOSTS = ["127.0.0.1", "localhost"] +CORS_ORIGIN_WHITELIST = ["127.0.0.1:8000", "localhost:3000"] +CORS_ORIGIN_ALLOW_ALL = True ADMINS = () +# Dynaconf Configuration +SECRET_KEY = True +GLOBAL_ENV_FOR_DYNACONF = "ARA" +ENVVAR_FOR_DYNACONF = "ARA_SETTINGS" +SETTINGS_MODULE_FOR_DYNACONF = "ara.server.settings" + +# We're not expecting ARA to use multiple concurrent databases. +# Make it easier for users to specify the configuration for a single database. +DATABASE_ENGINE = os.environ.get("ARA_DATABASE_ENGINE", "django.db.backends.sqlite3") +DATABASE_NAME = os.environ.get("ARA_DATABASE_NAME", os.path.join(SERVER_DIR, "ansible.sqlite")) +DATABASE_USER = os.environ.get("ARA_DATABASE_USER", None) +DATABASE_PASSWORD = os.environ.get("ARA_DATABASE_PASSWORD", None) +DATABASE_HOST = os.environ.get("ARA_DATABASE_HOST", None) +DATABASE_PORT = os.environ.get("ARA_DATABASE_PORT", None) + +DATABASES = { + "default": { + "ENGINE": DATABASE_ENGINE, + "NAME": DATABASE_NAME, + "USER": DATABASE_USER, + "PASSWORD": DATABASE_PASSWORD, + "HOST": DATABASE_HOST, + "PORT": DATABASE_PORT, + } +} + INSTALLED_APPS = [ "django.contrib.auth", "django.contrib.contenttypes", @@ -39,15 +66,6 @@ MIDDLEWARE = [ "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -# TODO: Only needed in dev? -CORS_ORIGIN_ALLOW_ALL = True - -# Django built-in server and npm development server -CORS_ORIGIN_WHITELIST = ("127.0.0.1:8000", "localhost:3000") - -ROOT_URLCONF = "ara.server.urls" -APPEND_SLASH = False - TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", @@ -64,10 +82,6 @@ TEMPLATES = [ } ] -WSGI_APPLICATION = "ara.server.wsgi.application" - -DATABASES = {"default": env.db(default="sqlite:///%s" % os.path.join(BASE_DIR, "db.sqlite3"))} - AUTH_PASSWORD_VALIDATORS = [ {"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator"}, {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator"}, @@ -75,51 +89,22 @@ AUTH_PASSWORD_VALIDATORS = [ {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator"}, ] -LANGUAGE_CODE = "en-us" - +USE_TZ = True TIME_ZONE = "UTC" USE_I18N = True - USE_L10N = True - -USE_TZ = True +LANGUAGE_CODE = "en-us" STATIC_URL = "/static/" - -STATIC_ROOT = os.path.join(BASE_DIR, "www", "static") +STATIC_ROOT = os.path.join(SERVER_DIR, "www", "static") MEDIA_URL = "/media/" +MEDIA_ROOT = os.path.join(SERVER_DIR, "www", "media") -MEDIA_ROOT = os.path.join(BASE_DIR, "www", "media") - -# fmt: off -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}}, - "handlers": { - "console": { - "class": "logging.StreamHandler", - "formatter": "normal", - "level": env("LOG_LEVEL", default="INFO"), - "stream": sys.stdout, - } - }, - "loggers": { - "ara": { - "handlers": ["console"], - "level": env("LOG_LEVEL", default="INFO"), - "propagate": 0 - } - }, - "root": { - "handlers": ["console"], - "level": env("LOG_LEVEL", default="DEBUG") - }, -} -# fmt: on - +WSGI_APPLICATION = "ara.server.wsgi.application" +ROOT_URLCONF = "ara.server.urls" +APPEND_SLASH = False REST_FRAMEWORK = { "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.LimitOffsetPagination", "PAGE_SIZE": 100, @@ -135,3 +120,62 @@ REST_FRAMEWORK = { ), "TEST_REQUEST_DEFAULT_FORMAT": "json", } + +DEBUG = False +LOG_LEVEL = "INFO" +# fmt: off +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": {"normal": {"format": "%(asctime)s %(levelname)s %(name)s: %(message)s"}}, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "formatter": "normal", + "level": LOG_LEVEL, + "stream": "ext://sys.stdout", + } + }, + "loggers": { + "ara": { + "handlers": ["console"], + "level": LOG_LEVEL, + "propagate": 0 + } + }, + "root": { + "handlers": ["console"], + "level": LOG_LEVEL + }, +} +# fmt: on + +# TODO: Split this out to a CLI command (django-admin command ?) +DEFAULT_CONFIG = os.path.join(SERVER_DIR, "default_config.yaml") +if not os.path.exists(DEFAULT_CONFIG): + CONFIG = dict( + BASE_DIR=BASE_DIR, + ALLOWED_HOSTS=ALLOWED_HOSTS, + CORS_ORIGIN_WHITELIST=CORS_ORIGIN_WHITELIST, + CORS_ORIGIN_ALLOW_ALL=CORS_ORIGIN_ALLOW_ALL, + SECRET_KEY="please-change-this", + DATABASES=DATABASES, + STATIC_URL=STATIC_URL, + STATIC_ROOT=STATIC_ROOT, + MEDIA_URL=MEDIA_URL, + MEDIA_ROOT=MEDIA_ROOT, + DEBUG=DEBUG, + LOG_LEVEL=LOG_LEVEL, + LOGGING=LOGGING, + ) + with open(DEFAULT_CONFIG, "w+") as config_file: + comment = f""" + --- + # This is a default configuration template generated by ARA. + # To use a configuration file such as this one, you need to export the + # ARA_SETTINGS configuration variable like so: + # $ export ARA_SETTINGS={DEFAULT_CONFIG} + + """ + config_file.write(textwrap.dedent(comment)) + yaml.dump({"default": CONFIG}, config_file, default_flow_style=False) diff --git a/ara/server/utils.py b/ara/server/utils.py index c1dc695..e69de29 100644 --- a/ara/server/utils.py +++ b/ara/server/utils.py @@ -1,89 +0,0 @@ -import os - -import environ -from configobj import ConfigObj -from django.core.exceptions import ImproperlyConfigured -from everett import ConfigurationError -from everett.manager import ConfigEnvFileEnv, ConfigIniEnv, ConfigManager, ConfigOSEnv, listify - -__all__ = ["EverettEnviron"] - - -class DumbConfigIniEnv(ConfigIniEnv): - """Simple ConfigIniEnv with disabled list parsing that actually aborts after the first file.""" - - # TODO: Remove once upstream is fixed (https://github.com/willkg/everett/pull/71) - - def __init__(self, possible_paths): - self.cfg = {} - possible_paths = listify(possible_paths) - - for path in possible_paths: - if not path: - continue - - path = os.path.abspath(os.path.expanduser(path.strip())) - if path and os.path.isfile(path): - self.cfg = self.parse_ini_file(path) - break - - def parse_ini_file(cls, path): - cfgobj = ConfigObj(path, list_values=False) - - def extract_section(namespace, d): - cfg = {} - for key, val in d.items(): - if isinstance(d[key], dict): - cfg.update(extract_section(namespace + [key], d[key])) - else: - cfg["_".join(namespace + [key]).upper()] = val - - return cfg - - return extract_section([], cfgobj.dict()) - - -class EnvironProxy: - def __init__(self, cfg): - self.cfg = cfg - - def __contains__(self, key): - try: - self.cfg(key) - except ConfigurationError: - return False - return True - - def __getitem__(self, key): - try: - return self.cfg(key, raw_value=True) - except ConfigurationError as err: - raise KeyError("Missing key %r" % key) from err - - -class EverettEnviron(environ.Env): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.ENVIRON = EnvironProxy( - ConfigManager( - [ - ConfigOSEnv(), - ConfigEnvFileEnv(".env"), - DumbConfigIniEnv([os.environ.get("ARA_CFG"), "~/.config/ara/server.cfg", "/etc/ara/server.cfg"]), - ] - ).with_namespace("ara") - ) - - def get_value(self, var, cast=None, default=environ.Env.NOTSET, parse_default=False): - try: - return super().get_value(var, cast, default, parse_default) - except ImproperlyConfigured as e: - # Rewrite the django-environ exception to match our configs better - if default is self.NOTSET and str(e) == "Set the {0} environment variable".format(var): - error_msg = ( - "Set the ARA_{0} environment variable or set {1} in the [ara] section " - "of your configuration file." - ).format(var, var.lower()) - raise ImproperlyConfigured(error_msg) - raise diff --git a/ara/server/wsgi.py b/ara/server/wsgi.py index 45655fd..0980556 100644 --- a/ara/server/wsgi.py +++ b/ara/server/wsgi.py @@ -1,7 +1,8 @@ import os -from django.core.wsgi import get_wsgi_application - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "ara.server.settings") +# https://github.com/rochacbruno/dynaconf/issues/89 +from dynaconf.contrib import django_dynaconf # noqa +from django.core.wsgi import get_wsgi_application # noqa application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index 364ba86..a3f41c3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,4 @@ django-cors-headers drf-extensions django-filter django-environ -everett +dynaconf diff --git a/setup.cfg b/setup.cfg index 6b20cfa..7484e6c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -75,7 +75,7 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,ara/api/migrations [isort] known_first_party = ara default_section = THIRDPARTY -skip = build,.git,.tox,.cache,.venv,ara/api/migrations +skip = build,.git,.tox,.cache,.venv,ara/api/migrations,ara/server/wsgi.py not_skip = __init__.py multi_line_output=3 include_trailing_comma=True diff --git a/tox.ini b/tox.ini index 28d8c61..9b51b88 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,10 @@ commands = ara-manage migrate ara-manage collectstatic --clear --no-input ara-manage runserver +setenv = + ARA_DEBUG=true + ARA_LOG_LEVEL=DEBUG + ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara # Temporary venv to help bootstrap integration [testenv:ansible-integration] @@ -37,11 +41,13 @@ deps = git+https://git.openstack.org/openstack/ara-plugins@master#egg=ara_plugins git+https://git.openstack.org/openstack/ara-clients@master#egg=ara_clients commands = - rm -f {toxinidir}/db.sqlite3 + rm -rf {toxinidir}/.tox/ansible-integration/tmp/ara bash -c 'ANSIBLE_CALLBACK_PLUGINS=$(python -c "import os,ara.plugins; print(os.path.dirname(ara.plugins.__file__))")/callback ansible-playbook {toxinidir}/hacking/test-playbook.yml' python {toxinidir}/hacking/validate.py setenv = - ARA_CFG={toxinidir}/ara/server/configs/integration.cfg + ARA_DEBUG=true + ARA_LOG_LEVEL=DEBUG + ARA_BASE_DIR={toxinidir}/.tox/ansible-integration/tmp/ara whitelist_externals = rm bash