Allow configuring logging via config file

The fileConfig stanza sets up alembic migration logging with the in-tree
logging config file. In some cases (zuul) that winds up stepping on
other logging config.

Add a config setting that can be used to pass a logging config file.
This file can be in logging ini format, or in yaml or json format. If
given it will apply settings to ara and alembic migrations.

If a file is not given, the current logging behavior remains. However,
logging config was removed from alembic.ini and instead set during
configure_logging so that the migrations don't step on other logging
config.

Change-Id: Ie555ab6ba6a177d084e62cd1bf2b89954894fea1

ara/config.py

@@ -69,6 +69,7 @@ DEFAULTS = {
     'ARA_IGNORE_EMPTY_GENERATION': True,
     'ARA_IGNORE_MIMETYPE_WARNINGS': True,
     'ARA_IGNORE_PARAMETERS': ['extra_vars'],
+    'ARA_LOG_CONFIG': None,
     'ARA_LOG_FORMAT': '%(asctime)s - %(name)s - %(levelname)s - %(message)s',
     'ARA_LOG_LEVEL': 'INFO',
     'ARA_PATH_MAX': 40,
@@ -101,6 +102,7 @@ ARA_HOST = _ara_config(config, 'host', 'ARA_HOST')
 ARA_IGNORE_PARAMETERS = _ara_config(config, 'ignore_parameters',
                                     'ARA_IGNORE_PARAMETERS',
                                     value_type='list')
+ARA_LOG_CONFIG = _ara_config(config, 'logconfig', 'ARA_LOG_CONFIG')
 ARA_LOG_FILE = _ara_config(config, 'logfile', 'ARA_LOG_FILE')
 ARA_LOG_FORMAT = _ara_config(config, 'logformat', 'ARA_LOG_FORMAT')
 ARA_LOG_LEVEL = _ara_config(config, 'loglevel', 'ARA_LOG_LEVEL')

ara/db/alembic.ini

@@ -7,39 +7,3 @@
 # set to 'true' to run the environment during
 # the 'revision' command, regardless of autogenerate
 # revision_environment = false
-
-
-# Logging configuration
-[loggers]
-keys = root,sqlalchemy,alembic
-
-[handlers]
-keys = console
-
-[formatters]
-keys = generic
-
-[logger_root]
-level = WARN
-handlers = console
-qualname =
-
-[logger_sqlalchemy]
-level = WARN
-handlers =
-qualname = sqlalchemy.engine
-
-[logger_alembic]
-level = INFO
-handlers =
-qualname = alembic
-
-[handler_console]
-class = StreamHandler
-args = (sys.stderr,)
-level = INFO
-formatter = generic
-
-[formatter_generic]
-format = %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %H:%M:%S

ara/db/env.py

@@ -18,7 +18,6 @@
 from __future__ import with_statement
 from alembic import context
 from sqlalchemy import engine_from_config, pool
-from logging.config import fileConfig
 from flask import current_app
 
 import logging
@@ -27,9 +26,6 @@ import logging
 # access to the values within the .ini file in use.
 config = context.config
 
-# Interpret the config file for Python logging.
-# This line sets up loggers basically.
-fileConfig(config.config_file_name)
 logger = logging.getLogger('alembic.env')
 
 # add your model's MetaData object here

ara/webapp.py

@@ -20,6 +20,7 @@ import ara.views
 import flask_migrate
 import logging
 import os
+import yaml
 
 from alembic.migration import MigrationContext
 from alembic.script import ScriptDirectory
@@ -133,7 +134,15 @@ def configure_db(app):
 
 
 def configure_logging(app):
-    if app.config['ARA_LOG_FILE']:
+    if app.config['ARA_LOG_CONFIG'] and os.path.exists(
+            app.config['ARA_LOG_CONFIG']):
+        config_path = app.config['ARA_LOG_CONFIG']
+        if os.path.splitext(config_path)[1] in ('.yml', '.yaml', '.json'):
+            # yaml.safe_load can load json as well as yaml
+            logging.config.dictConfig(yaml.safe_load(open(config_path, 'r')))
+        else:
+            logging.config.fileConfig(config_path)
+    elif app.config['ARA_LOG_FILE']:
         handler = logging.FileHandler(app.config['ARA_LOG_FILE'])
         # Set the ARA log format or fall back to the flask debugging format
         handler.setFormatter(
@@ -144,11 +153,11 @@ def configure_logging(app):
         del logger.handlers[:]
         logger.addHandler(handler)
 
-        # TODO: Log things from Alembic to ARA_LOG_FILE properly
-        alembic_logger = logging.getLogger('alembic')
-        alembic_logger.setLevel(logging.WARNING)
-        del alembic_logger.handlers[:]
-        alembic_logger.addHandler(handler)
+        for name in ('alembic', 'sqlalchemy.engine'):
+            other_logger = logging.getLogger(name)
+            other_logger.setLevel(logging.WARNING)
+            del other_logger.handlers[:]
+            other_logger.addHandler(handler)
 
 
 def configure_static_route(app):

doc/source/configuration.rst

@@ -105,6 +105,8 @@ Parameters and their defaults
 +-------------------------------+----------------------------+-------------------------------------------+
 | ARA_PORT_                     | port                       | 9191                                      |
 +-------------------------------+----------------------------+-------------------------------------------+
+| ARA_LOG_CONFIG_               | logconfig                  | None                                      |
++-------------------------------+----------------------------+-------------------------------------------+
 | ARA_LOG_FILE_                 | logfile                    | ~/.ara/ara.log                            |
 +-------------------------------+----------------------------+-------------------------------------------+
 | ARA_LOG_LEVEL_                | loglevel                   | INFO                                      |
@@ -214,6 +216,24 @@ It is equivalent to the ``-p`` or ``--port`` argument of the
 ARA_LOG_FILE
 ~~~~~~~~~~~~
 
+Path to a python logging config file.
+
+If the filename ends in ``.yaml`` or ``.yml`` the file will be loaded as yaml.
+If the filename ends in ``.json`` the file will be loaded as json. The
+resulting dict for either will be treated as a `logging config dict`_
+and passed to `logging.config.dictConfig`.
+
+Otherwise it will be assumed to a `logging config file`_ and the path will be
+passed to `logging.config.fileConfig`.
+
+If this option is given it superseeds the other individual log options.
+
+.. _logging config dict: https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
+.. _logging config file: https://docs.python.org/3/library/logging.config.html#logging-config-fileformat
+
+ARA_LOG_FILE
+~~~~~~~~~~~~
+
 Path to the logfile to store ARA logs in.
 
 ARA_LOG_LEVEL