From 53ac2cfeaef6be509e7d4ff1d2876e2de85268aa Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Mon, 26 Feb 2018 14:12:35 +1100 Subject: [PATCH] Remove per-image logging from nodepool-builder logging config Change Ia7415d2fbbb320f8eddc4e46c3a055414df5f997 removed per-image logging from nodepool; it now keeps a series of named log files. The "image" logger is removed, along with all the specific build logs. I972a66f9c153f52df934d25ab2f77596bb2e5429 removed the image_log_document_root argument; which wasn't doing anything in nodepool::builder, but was actually being used to fill-out the builder-logging.conf file. Thus we need this to not have a bad path in the config file. Change-Id: I6fdc3b622454b069f3ad2cf42da584d8df23a110 --- .../nodepool-builder.logging.conf.erb | 267 +------------- tools/nodepool_log_config.py | 332 ------------------ 2 files changed, 2 insertions(+), 597 deletions(-) delete mode 100755 tools/nodepool_log_config.py diff --git a/modules/openstack_project/templates/nodepool/nodepool-builder.logging.conf.erb b/modules/openstack_project/templates/nodepool/nodepool-builder.logging.conf.erb index 2a0d4ce3f5..52a6bb938f 100644 --- a/modules/openstack_project/templates/nodepool/nodepool-builder.logging.conf.erb +++ b/modules/openstack_project/templates/nodepool/nodepool-builder.logging.conf.erb @@ -1,27 +1,8 @@ - - -# -# THIS FILE HAS BEEN AUTOGENERATED -# Regenerate it with tools/nodepool_log_config.py -# - [loggers] -keys=root,nodepool,requests,shade,image,kazoo, - dib_centos-7, dib_debian-jessie, dib_fedora-26, dib_fedora-27, - dib_gentoo, dib_opensuse-423, dib_ubuntu-trusty, dib_ubuntu-xenial, - citycloud-kna1_upload, citycloud-la1_upload, citycloud-lon1_upload, - citycloud-sto2_upload, ovh-bhs1_upload, ovh-gra1_upload, - inap-mtl01_upload, rax-dfw_upload, rax-iad_upload, rax-ord_upload, - vexxhost-ca-ymq-1_upload +keys=root,nodepool,requests,shade,kazoo [handlers] -keys=console,debug,normal,image, - dib_centos-7, dib_debian-jessie, dib_fedora-26, dib_fedora-27, - dib_gentoo, dib_opensuse-423, dib_ubuntu-trusty, dib_ubuntu-xenial, - citycloud-kna1_upload, citycloud-la1_upload, citycloud-lon1_upload, - citycloud-sto2_upload, ovh-bhs1_upload, ovh-gra1_upload, - inap-mtl01_upload, rax-dfw_upload, rax-iad_upload, rax-ord_upload, - vexxhost-ca-ymq-1_upload +keys=console,debug,normal [formatters] keys=simple @@ -45,12 +26,6 @@ level=DEBUG handlers=debug,normal qualname=nodepool -[logger_image] -level=DEBUG -handlers=image -qualname=nodepool.image.build -propagate=0 - [logger_kazoo] level=INFO handlers=debug,normal @@ -75,244 +50,6 @@ class=logging.handlers.TimedRotatingFileHandler formatter=simple args=('/var/log/nodepool/nodepool-builder.log', 'H', 8, 30,) -[handler_image] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/image.log', 'H', 8, 30,) - [formatter_simple] format=%(asctime)s %(levelname)s %(name)s: %(message)s datefmt= - -# ==== loggers and handlers ==== - - -[logger_dib_centos-7] -level=DEBUG -handlers=dib_centos-7 -qualname=nodepool.image.build.centos-7 -propagate=0 - -[handler_dib_centos-7] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.centos-7.log', 'H', 8, 30,) - -[logger_dib_debian-jessie] -level=DEBUG -handlers=dib_debian-jessie -qualname=nodepool.image.build.debian-jessie -propagate=0 - -[handler_dib_debian-jessie] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.debian-jessie.log', 'H', 8, 30,) - -[logger_dib_fedora-26] -level=DEBUG -handlers=dib_fedora-26 -qualname=nodepool.image.build.fedora-26 -propagate=0 - -[handler_dib_fedora-26] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.fedora-26.log', 'H', 8, 30,) - -[logger_dib_fedora-27] -level=DEBUG -handlers=dib_fedora-27 -qualname=nodepool.image.build.fedora-27 -propagate=0 - -[handler_dib_fedora-27] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.fedora-27.log', 'H', 8, 30,) - -[logger_dib_gentoo] -level=DEBUG -handlers=dib_gentoo -qualname=nodepool.image.build.gentoo -propagate=0 - -[handler_dib_gentoo] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.gentoo.log', 'H', 8, 30,) - -[logger_dib_opensuse-423] -level=DEBUG -handlers=dib_opensuse-423 -qualname=nodepool.image.build.opensuse-423 -propagate=0 - -[handler_dib_opensuse-423] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.opensuse-423.log', 'H', 8, 30,) - -[logger_dib_ubuntu-trusty] -level=DEBUG -handlers=dib_ubuntu-trusty -qualname=nodepool.image.build.ubuntu-trusty -propagate=0 - -[handler_dib_ubuntu-trusty] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.ubuntu-trusty.log', 'H', 8, 30,) - -[logger_dib_ubuntu-xenial] -level=DEBUG -handlers=dib_ubuntu-xenial -qualname=nodepool.image.build.ubuntu-xenial -propagate=0 - -[handler_dib_ubuntu-xenial] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @image_log_document_root %>/dib.ubuntu-xenial.log', 'H', 8, 30,) - -[logger_citycloud-kna1_upload] -level=DEBUG -handlers=citycloud-kna1_upload -qualname=nodepool.image.upload.citycloud-kna1 -propagate=0 - -[handler_citycloud-kna1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/citycloud-kna1.log', 'H', 8, 30,) - -[logger_citycloud-la1_upload] -level=DEBUG -handlers=citycloud-la1_upload -qualname=nodepool.image.upload.citycloud-la1 -propagate=0 - -[handler_citycloud-la1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/citycloud-la1.log', 'H', 8, 30,) - -[logger_citycloud-lon1_upload] -level=DEBUG -handlers=citycloud-lon1_upload -qualname=nodepool.image.upload.citycloud-lon1 -propagate=0 - -[handler_citycloud-lon1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/citycloud-lon1.log', 'H', 8, 30,) - -[logger_citycloud-sto2_upload] -level=DEBUG -handlers=citycloud-sto2_upload -qualname=nodepool.image.upload.citycloud-sto2 -propagate=0 - -[handler_citycloud-sto2_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/citycloud-sto2.log', 'H', 8, 30,) - -[logger_ovh-bhs1_upload] -level=DEBUG -handlers=ovh-bhs1_upload -qualname=nodepool.image.upload.ovh-bhs1 -propagate=0 - -[handler_ovh-bhs1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/ovh-bhs1.log', 'H', 8, 30,) - -[logger_ovh-gra1_upload] -level=DEBUG -handlers=ovh-gra1_upload -qualname=nodepool.image.upload.ovh-gra1 -propagate=0 - -[handler_ovh-gra1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/ovh-gra1.log', 'H', 8, 30,) - -[logger_inap-mtl01_upload] -level=DEBUG -handlers=inap-mtl01_upload -qualname=nodepool.image.upload.inap-mtl01 -propagate=0 - -[handler_inap-mtl01_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/inap-mtl01.log', 'H', 8, 30,) - -[logger_rax-dfw_upload] -level=DEBUG -handlers=rax-dfw_upload -qualname=nodepool.image.upload.rax-dfw -propagate=0 - -[handler_rax-dfw_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/rax-dfw.log', 'H', 8, 30,) - -[logger_rax-iad_upload] -level=DEBUG -handlers=rax-iad_upload -qualname=nodepool.image.upload.rax-iad -propagate=0 - -[handler_rax-iad_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/rax-iad.log', 'H', 8, 30,) - -[logger_rax-ord_upload] -level=DEBUG -handlers=rax-ord_upload -qualname=nodepool.image.upload.rax-ord -propagate=0 - -[handler_rax-ord_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/rax-ord.log', 'H', 8, 30,) - -[logger_vexxhost-ca-ymq-1_upload] -level=DEBUG -handlers=vexxhost-ca-ymq-1_upload -qualname=nodepool.image.upload.vexxhost-ca-ymq-1 -propagate=0 - -[handler_vexxhost-ca-ymq-1_upload] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('<%= @upload_log_document_root %>/vexxhost-ca-ymq-1.log', 'H', 8, 30,) - diff --git a/tools/nodepool_log_config.py b/tools/nodepool_log_config.py deleted file mode 100755 index 288cec96f9..0000000000 --- a/tools/nodepool_log_config.py +++ /dev/null @@ -1,332 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -"""Generate a sample logging configuration file - -use log_config_generator.generate_log_config() to generate a sample -logging configuration file. - -The sample splits up the log output for general logging and image -builds and applys some sensible rotation defaults. -""" - -import argparse -import logging -import textwrap -import yaml - -# default paths and outputs -MODULES_PATH = '../modules/openstack_project/templates/nodepool' -# default the project-config repo path to the same folder as this repo -CONFIG_FILE = '../../project-config/nodepool/nodepool.yaml' -LOGGING_CONFIG_FILE = MODULES_PATH + '/nodepool-builder.logging.conf.erb' -LOG_DIR = '/var/log/nodepool' -IMAGE_LOG_DIR = '<%= @image_log_document_root %>' -UPLOAD_LOG_DIR = '<%= @upload_log_document_root %>' - -_BASIC_FILE = """ - -# -# THIS FILE HAS BEEN AUTOGENERATED -# Regenerate it with tools/nodepool_log_config.py -# - -[loggers] -keys=root,nodepool,requests,shade,image,kazoo, - %(logger_titles)s - -[handlers] -keys=console,debug,normal,image, - %(handler_titles)s - -[formatters] -keys=simple - -[logger_root] -level=WARNING -handlers=console - -[logger_requests] -level=WARNING -handlers=debug,normal -qualname=requests - -[logger_shade] -level=WARNING -handlers=debug,normal -qualname=shade - -[logger_nodepool] -level=DEBUG -handlers=debug,normal -qualname=nodepool - -[logger_image] -level=DEBUG -handlers=image -qualname=nodepool.image.build -propagate=0 - -[logger_kazoo] -level=INFO -handlers=debug,normal -qualname=kazoo -propagate=0 - -[handler_console] -level=WARNING -class=StreamHandler -formatter=simple -args=(sys.stdout,) - -[handler_debug] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('%(log_dir)s/builder-debug.log', 'H', 8, 30,) - -[handler_normal] -level=INFO -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('%(log_dir)s/nodepool-builder.log', 'H', 8, 30,) - -[handler_image] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('%(image_log_dir)s/image.log', 'H', 8, 30,) - -[formatter_simple] -format=%%(asctime)s %%(levelname)s %%(name)s: %%(message)s -datefmt= - -# ==== loggers and handlers ==== - -%(loggers_and_handlers)s -""" - -_IMAGE_HANDLER = """ -[handler_%(title)s] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('%(image_log_dir)s/%(filename)s', 'H', 8, 30,) -""" - -_IMAGE_LOGGER = """ -[logger_%(title)s] -level=DEBUG -handlers=%(handler)s -qualname=nodepool.image.build.%(qualname)s -propagate=0 -""" - -_UPLOAD_HANDLER = """ -[handler_%(title)s] -level=DEBUG -class=logging.handlers.TimedRotatingFileHandler -formatter=simple -args=('%(upload_log_dir)s/%(filename)s', 'H', 8, 30,) -""" - -_UPLOAD_LOGGER = """ -[logger_%(title)s] -level=DEBUG -handlers=%(handler)s -qualname=nodepool.image.upload.%(qualname)s -propagate=0 -""" - - -def _get_providers_and_images(config): - ret = [] - # diskimages have a special provider - if 'diskimages' in config: - for diskimage in config['diskimages']: - ret.append(('dib', diskimage['name'])) - logging.debug("Added dib image: %s" % diskimage['name']) - - return ret - - -def _get_upload_providers(config): - ret = [] - for provider in config['providers']: - ret.append(provider['name']) - logging.debug("Added upload provider: %s" % provider['name']) - return ret - - -def _generate_image_logger_and_handler(image_log_dir, provider, image): - handler = _IMAGE_HANDLER % { - 'image_log_dir': image_log_dir, - 'title': '%s_%s' % (provider, image), - 'filename': '%s.%s.log' % (provider, image), - } - - logger = _IMAGE_LOGGER % { - 'title': '%s_%s' % (provider, image), - 'handler': '%s_%s' % (provider, image), - 'qualname': '%s%s' % (provider + "." if provider != 'dib' else '', - image), - } - - return { - 'handler_title': '%s_%s' % (provider, image), - 'logger_title': '%s_%s' % (provider, image), - 'handler': handler, - 'logger': logger, - } - - -def _generate_upload_logger_and_handler(upload_log_dir, provider): - handler = _UPLOAD_HANDLER % { - 'upload_log_dir': upload_log_dir, - 'title': '%s_upload' % (provider), - 'filename': '%s.log' % (provider), - } - - logger = _UPLOAD_LOGGER % { - 'title': '%s_upload' % (provider), - 'handler': '%s_upload' % (provider), - 'qualname': '%s' % (provider), - } - - return { - 'handler_title': '%s_upload' % (provider), - 'logger_title': '%s_upload' % (provider), - 'handler': handler, - 'logger': logger, - } - - -def generate_log_config(config, log_dir, image_log_dir, - upload_log_dir, output): - - """Generate a sample logging file - - The logging output will have the correct formatters and handlers - to split all image-build logs out into separate files grouped by - provider. e.g. - - providers: - - name: foo - ... - - images: - - name: image1 - ... - - name: image2 - ... - - name: moo - ... - - images: - - name: image1 - ... - - name: image2 - ... - - Will result in log files (in `image_log_dir`) of foo.image1.log, - foo.image2.log, moo.image1.log, moo.image2.log - - diskimage-builder built images will have special provider "dib" - - :param config: input config file - :param log_dir: directory for main log file - :param image_log_dir: directory for image build logs - :param upload_log_dir: directory for upload logs - :param output: open file handle to output sample configuration to - - """ - - loggers_and_handlers = [] - logging.debug("Reading config file %s" % config.name) - config = yaml.load(config) - - # image loggers - for (provider, image) in _get_providers_and_images(config): - loggers_and_handlers.append( - _generate_image_logger_and_handler(image_log_dir, provider, image)) - - # provider loggers (for upload) - for provider in _get_upload_providers(config): - loggers_and_handlers.append( - _generate_upload_logger_and_handler(upload_log_dir, provider)) - - # output all loggers and handlers - logger_titles = [] - handler_titles = [] - image_loggers_and_handlers = "" - for item in loggers_and_handlers: - logger_titles.append(item['logger_title']) - handler_titles.append(item['handler_title']) - image_loggers_and_handlers += item['logger'] + item['handler'] - - # Final output - final_output = _BASIC_FILE % { - 'log_dir': log_dir, - 'image_log_dir': image_log_dir, - 'logger_titles': "\n".join(textwrap.wrap(', '.join(logger_titles), - break_long_words=False, - break_on_hyphens=False, - subsequent_indent=' ')), - 'handler_titles': "\n".join(textwrap.wrap(', '.join(handler_titles), - break_long_words=False, - break_on_hyphens=False, - subsequent_indent=' ')), - 'loggers_and_handlers': image_loggers_and_handlers, - } - - logging.debug("Writing output to %s" % output.name) - output.write(final_output) - output.flush() - logging.debug("Done!") - - -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument('-d', '--debug', action='store_true', - help="Enable debugging") - parser.add_argument('-c', '--config', default=CONFIG_FILE, - help="Config file to read in " - "(default: %s)" % CONFIG_FILE, - type=argparse.FileType('r')) - parser.add_argument('-o', '--output', default=LOGGING_CONFIG_FILE, - help="Output file " - "(default: %s)" % LOGGING_CONFIG_FILE, - type=argparse.FileType('w')) - parser.add_argument('-l', '--log-dir', default=LOG_DIR, - help="Output directory for logs " - "(default: %s)" % LOG_DIR) - parser.add_argument('-i', '--image-log-dir', default=IMAGE_LOG_DIR, - help="Output directory for image logs " - "(default: %s)" % IMAGE_LOG_DIR.replace('%', '%%')) - parser.add_argument('-u', '--upload-log-dir', default=UPLOAD_LOG_DIR, - help="Output directory for upload logs " - "(default: %s)" % UPLOAD_LOG_DIR.replace('%', '%%')) - args = parser.parse_args() - - logging.basicConfig(level=logging.DEBUG if args.debug else logging.INFO) - - generate_log_config(args.config, - args.log_dir, - args.image_log_dir, - args.upload_log_dir, - args.output) - - -if __name__ == "__main__": - main()