From c34f3ef74b7fc0bd49a608d898bf200e88b765df Mon Sep 17 00:00:00 2001 From: Michal Piotrowski Date: Wed, 28 Oct 2020 09:41:44 +0100 Subject: [PATCH] Fixed docker configuration template - Added logging configuration - Added env variables for puglins configuration - Added env variables for Statsd configuration - Removing Hipchat. This service doesn't exist anymore - Removing deprecated ALARM_PROCESSORS env var Change-Id: Ia35c7da1c306cdd1c14d787f1c54504d6df49080 Story: 2003180 Task: 39090 --- docker/Dockerfile | 8 +- docker/README.rst | 28 +- docker/monasca-notification.conf.j2 | 400 +++++++++++++--------------- docker/notification-logging.conf.j2 | 33 +++ docker/start.sh | 2 +- 5 files changed, 231 insertions(+), 240 deletions(-) create mode 100644 docker/notification-logging.conf.j2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 794e759..0318ea6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,11 +11,13 @@ ARG BASE_TAG=master FROM monasca/base:$BASE_TAG # Environment variables used for our service or wait scripts. ENV \ + LOG_LEVEL=WARNING \ + LOG_LEVEL_KAFKA=WARNING \ + LOG_LEVEL_PLUGINS=WARNING \ KAFKA_URI=kafka:9092 \ KAFKA_LEGACY_CLIENT_ENABLED=false \ KAFKA_WAIT_FOR_TOPICS=retry-notifications,alarm-state-transitions,alarm-notifications,60-seconds-notifications \ ZOOKEEPER_URL=zookeeper:2181 \ - ALARM_PROCESSORS=2 \ NOTIFICATION_PROCESSORS=2 \ RETRY_INTERVAL=30 \ RETRY_MAX_ATTEMPTS=5 \ @@ -24,11 +26,11 @@ ENV \ MYSQL_USER=notification \ MYSQL_PASSWORD=password \ MYSQL_DB=mon \ - STATSD_ENABLE=true \ + STATSD_ENABLED=true \ STATSD_HOST=monasca-statsd \ STATSD_PORT=8125 \ STAY_ALIVE_ON_FAILURE="false" # Copy all necessary files to proper locations. -COPY monasca-notification.conf.j2 /etc/monasca/ +COPY monasca-notification.conf.j2 notification-logging.conf.j2 /etc/monasca/ # Implement start script in `start.sh` file. CMD ["/start.sh"] diff --git a/docker/README.rst b/docker/README.rst index 2b524fa..365021c 100644 --- a/docker/README.rst +++ b/docker/README.rst @@ -6,7 +6,7 @@ The Monasca notification image is based on the monasca-base image. Building monasca-base image =========================== -See https://github.com/openstack/monasca-common/tree/master/docker/README.rst +See https://opendev.org/openstack/monasca-common/src/branch/master/docker/README.rst Building Monasca notification image @@ -24,15 +24,16 @@ Environment variables ============================== ================= ================================================ Variable Default Description ============================== ================= ================================================ +LOG_LEVEL WARNING Log level for main logging +LOG_LEVEL_KAFKA WARNING Log level for Kafka +LOG_LEVEL_PLUGINS WARNING Log level for plugins (email, webhook, etc) KAFKA_URI kafka:9092 The host and port for kafka KAFKA_LEGACY_CLIENT_ENABLED false Enable legacy Kafka client ZOOKEEPER_URL zookeeper:2181 URL to Zookeeper -ALARM_PROCESSORS 2 Number of alarm processing threads NOTIFICATION_PROCESSORS 2 Number of notification processing threads RETRY_INTERVAL 30 Retry interval in seconds RETRY_MAX_ATTEMPTS 5 Max number of notification retries -LOG_LEVEL WARN Logging level -STATSD_ENABLE true Monasca agent StatsD enable or disable +STATSD_ENABLED true Monasca agent StatsD enable or disable STATSD_HOST monasca-statsd Monasca agent StatsD host for self-monitoring STATSD_PORT 8125 Monasca agent StatsD port for self-monitoring NF_PLUGINS See below "Notification Plugins" @@ -118,19 +119,6 @@ Options: * NF_PAGERDUTY_URL: PagerDuty Event API endpoint, defaults to official URL -HipChat -------- -Name: hipchat - -Notifies via a HipChat message to some room. Authentication and destination details are configured with the notification. - -Options: - * NF_HIPCHAT_TIMEOUT: timeout in seconds, default: 5 - * NF_HIPCHAT_SSL_CERTS: path to SSL certs, default: system certs - * NF_HIPCHAT_INSECURE: if true, don't verify SSL - * NF_HIPCHAT_PROXY: if set, use the given HTTP(S) proxy server to send notifications - - Slack ----- Name: slack @@ -146,9 +134,9 @@ Options: Provide Configuration templates ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -* notification.yaml.j2 - +* monasca-notification.conf.j2 +* notification-logging.conf.j2 Links ~~~~~ -https://github.com/openstack/monasca-notification/blob/master/README.rst +https://opendev.org/openstack/monasca-notification/src/branch/master/README.rst diff --git a/docker/monasca-notification.conf.j2 b/docker/monasca-notification.conf.j2 index 22e1c9a..ea9b2d5 100644 --- a/docker/monasca-notification.conf.j2 +++ b/docker/monasca-notification.conf.j2 @@ -4,11 +4,6 @@ # From oslo.log # -# If set to true, the logging level will be set to DEBUG instead of the default -# INFO level. (boolean value) -# Note: This option can be changed without restarting. -#debug = false - # The name of a logging configuration file. This file is appended to any # existing logging configuration files. For details about logging configuration # files, see the Python logging module documentation. Note that when logging @@ -17,135 +12,7 @@ # example, log-date-format). (string value) # Note: This option can be changed without restarting. # Deprecated group/name - [DEFAULT]/log_config -#log_config_append = - -# Defines the format string for %%(asctime)s in log records. Default: -# %(default)s . This option is ignored if log_config_append is set. (string -# value) -#log_date_format = %Y-%m-%d %H:%M:%S - -# (Optional) Name of log file to send logging output to. If no default is set, -# logging will go to stderr as defined by use_stderr. This option is ignored if -# log_config_append is set. (string value) -# Deprecated group/name - [DEFAULT]/logfile -#log_file = - -# (Optional) The base directory used for relative log_file paths. This option -# is ignored if log_config_append is set. (string value) -# Deprecated group/name - [DEFAULT]/logdir -#log_dir = - -# Uses logging handler designed to watch file system. When log file is moved or -# removed this handler will open a new log file with specified path -# instantaneously. It makes sense only if log_file option is specified and Linux -# platform is used. This option is ignored if log_config_append is set. (boolean -# value) -#watch_log_file = false - -# Use syslog for logging. Existing syslog format is DEPRECATED and will be -# changed later to honor RFC5424. This option is ignored if log_config_append is -# set. (boolean value) -#use_syslog = false - -# Enable journald for logging. If running in a systemd environment you may wish -# to enable journal support. Doing so will use the journal native protocol which -# includes structured metadata in addition to log messages.This option is -# ignored if log_config_append is set. (boolean value) -#use_journal = false - -# Syslog facility to receive log lines. This option is ignored if -# log_config_append is set. (string value) -#syslog_log_facility = LOG_USER - -# Use JSON formatting for logging. This option is ignored if log_config_append -# is set. (boolean value) -#use_json = false - -# Log output to standard error. This option is ignored if log_config_append is -# set. (boolean value) -#use_stderr = false - -# Log output to Windows Event Log. (boolean value) -#use_eventlog = false - -# The amount of time before the log files are rotated. This option is ignored -# unless log_rotation_type is setto "interval". (integer value) -#log_rotate_interval = 1 - -# Rotation interval type. The time of the last file change (or the time when the -# service was started) is used when scheduling the next rotation. (string value) -# Possible values: -# Seconds - -# Minutes - -# Hours - -# Days - -# Weekday - -# Midnight - -#log_rotate_interval_type = days - -# Maximum number of rotated log files. (integer value) -#max_logfile_count = 30 - -# Log file maximum size in MB. This option is ignored if "log_rotation_type" is -# not set to "size". (integer value) -#max_logfile_size_mb = 200 - -# Log rotation type. (string value) -# Possible values: -# interval - Rotate logs at predefined time intervals. -# size - Rotate logs once they reach a predefined size. -# none - Do not rotate log files. -#log_rotation_type = none - -# Format string to use for log messages with context. Used by -# oslo_log.formatters.ContextFormatter (string value) -#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s - -# Format string to use for log messages when context is undefined. Used by -# oslo_log.formatters.ContextFormatter (string value) -#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s - -# Additional data to append to log message when logging level for the message is -# DEBUG. Used by oslo_log.formatters.ContextFormatter (string value) -#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d - -# Prefix each line of exception output with this format. Used by -# oslo_log.formatters.ContextFormatter (string value) -#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s - -# Defines the format string for %(user_identity)s that is used in -# logging_context_format_string. Used by oslo_log.formatters.ContextFormatter -# (string value) -#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s - -# List of package logging levels in logger=LEVEL pairs. This option is ignored -# if log_config_append is set. (list value) -#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,oslo_policy=INFO,dogpile.core.dogpile=INFO - -# Enables or disables publication of error events. (boolean value) -#publish_errors = false - -# The format for an instance that is passed with the log message. (string value) -#instance_format = "[instance: %(uuid)s] " - -# The format for an instance UUID that is passed with the log message. (string -# value) -#instance_uuid_format = "[instance: %(uuid)s] " - -# Interval, number of seconds, of log rate limiting. (integer value) -#rate_limit_interval = 0 - -# Maximum number of logged messages per rate_limit_interval. (integer value) -#rate_limit_burst = 0 - -# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG or -# empty string. Logs with level greater or equal to rate_limit_except_level are -# not filtered. An empty string means that all levels are filtered. (string -# value) -#rate_limit_except_level = CRITICAL - -# Enables or disables fatal status of deprecations. (boolean value) -#fatal_deprecations = false +log_config_append = /etc/monasca/notification-logging.conf [alarm_processor] @@ -161,7 +28,7 @@ # Its value may be silently ignored in the future. # Reason: Options is not used in the current code and will be removed in future # releases. -number = {{ ALARM_PROCESSORS | default(2) }} +#number = 2 # Alarms older than TTL are not processed by notification engine. (integer # value) @@ -185,14 +52,19 @@ number = {{ ALARM_PROCESSORS | default(2) }} # 'monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo', # 'monasca_notification.common.repositories.orm.orm_repo:OrmRepo']). Configuring # either of them will require presence of one of following sections -# dict_keys(['postgresql', 'mysql', 'orm']) inside configuration file. (plugin -# value) +# dict_keys(['postgresql', 'mysql', 'orm']) inside configuration file. (plugin value) # Possible values: -# postgresql - -# monasca_notification.common.repositories.orm.orm_repo:OrmRepo - -# mysql - -# monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo - # orm - +# mysql - +# monasca_notification.common.repositories.postgres.pgsql_repo:PostgresqlRepo - +# monasca_notification.common.repositories.orm.orm_repo:OrmRepo - +# monasca_notification.common.repositories.mysql.mysql_repo:MysqlRepo - +# postgresql - +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +#repo_driver = mysql + [notification_types] # Group allows to configure available notifiers inside notification engine. @@ -200,10 +72,8 @@ number = {{ ALARM_PROCESSORS | default(2) }} # # From monasca_notification # - # List of enabled notification types. You may specify full class name -# monasca_notification.plugins.hipchat_notifier:HipChatNotifier or shorter label -# hipchat. (list value) +# monasca_notification.plugins.email_notifier:EmailNotifier or shorter label email. (list value) # Advanced Option: intended for advanced users and not used # by the majority of users, and might have a significant # effect on stability and/or performance. @@ -211,66 +81,91 @@ number = {{ ALARM_PROCESSORS | default(2) }} # This option has a sample default set, which means that # its actual default value may vary from the one documented # below. -# enabled = email,pagerduty,webhook,hipchat,slack,jira {% if NF_PLUGINS %} {% set enabled = NF_PLUGINS.split(',')|map('trim')|list %} +enabled = {{ NF_PLUGINS }} +{% else %} + {% set enabled = [] %} +#enabled = email,jira,pagerduty,slack,webhook {% endif %} -# -# From monasca_notification -# + +### EMAIL {% if 'email' in enabled %} - {% set from_addr = NF_EMAIL_FROM_ADDR %} - {% set server = NF_EMAIL_SERVER %} - {% set port = NF_EMAIL_PORT | default(25) %} - {% set timeout = NF_EMAIL_TIMEOUT | default(15) %} - {% set user = NF_EMAIL_USER %} - {% set password = NF_EMAIL_PASSWORD %} - {% set grafana_url = NF_EMAIL_GRAFANA_URL %} -{% endif %} - - -[hipchat_notifier] -# -# From monasca_notification -# - -{% if 'hipchat' in enabled %} - {% set timeout = NF_HIPCHAT_TIMEOUT | default(5) %} - {% set insecure = NF_HIPCHAT_INSECURE | default('false') %} - {% set ca_certs = NF_HIPCHAT_SSL_CERTS | default('/etc/ssl/certs/ca-certificates.crt') %} - {% set proxy = NF_HIPCHAT_PROXY %} - {% set template = NF_HIPCHAT_TEMPLATE %} -{% endif %} - -[pagerduty_notifier] +[email_notifier] # # From monasca_notification # +# (string value) +from_addr = {{ NF_EMAIL_FROM_ADDR }} -{% if 'pagerduty' in enabled %} - {% set timeout = NF_PAGERDUTY_TIMEOUT | default(5) %} - {% set url = NF_PAGERDUTY_URL | default('https://events.pagerduty.com/generic/2010-04-15/create_event.json') %} -{% endif %} +# (host address value) +server = {{ NF_EMAIL_SERVER }} +# (port value) +# Minimum value: 0 +# Maximum value: 65535 +port = {{ NF_EMAIL_PORT | default(25) }} -[webhook_notifier] +# (integer value) +# Minimum value: 1 +timeout = {{ NF_EMAIL_TIMEOUT | default(15) }} + +# (string value) +user = {{ NF_EMAIL_USER }} + +# (string value) +password = {{ NF_EMAIL_PASSWORD }} + +# (string value) +grafana_url = {{ NF_EMAIL_GRAFANA_URL }} +{% else %} +#[email_notifier] # # From monasca_notification # -{% if 'webhook' in enabled %} - {% set timeout = NF_WEBHOOK_TIMEOUT|default(5) %} +# (string value) +#from_addr = + +# (host address value) +#server = + +# (port value) +# Minimum value: 0 +# Maximum value: 65535 +#port = 25 + +# (integer value) +# Minimum value: 1 +#timeout = 5 + +# (string value) +#user = + +# (string value) +#password = + +# (string value) +#grafana_url = {% endif %} +### JIRA +{% if 'jira' in enabled %} [jira_notifier] # # From monasca_notification # +{% else %} +#[jira_notifier] + +# +# From monasca_notification +# # (integer value) # Minimum value: 1 #timeout = 5 @@ -286,6 +181,108 @@ number = {{ ALARM_PROCESSORS | default(2) }} # (string value) #proxy = +{% endif %} + + +### PAGERDUTY +{% if 'pagerduty' in enabled %} +[pagerduty_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +timeout = {{ NF_PAGERDUTY_TIMEOUT | default(5) }} + +# (string value) +url = {{ NF_PAGERDUTY_URL | default('https://events.pagerduty.com/generic/2010-04-15/create_event.json') }} +{% else %} +#[pagerduty_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +#timeout = 5 + +# (string value) +#url = https://events.pagerduty.com/generic/2010-04-15/create_event.json +{% endif %} + + +### SLACK +{% if 'slack' in enabled %} +[slack_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +timeout = {{ NF_SLACK_TIMEOUT | default(5) }} + +# (boolean value) +insecure = {{ NF_SLACK_INSECURE | default('false') }} + +# (string value) +ca_certs = {{ NF_SLACK_CA_CERTS | default('/etc/ssl/certs/ca-certificates.crt') }} + +{% if NF_SLACK_PROXY %} +# (string value) +proxy = {{ NF_SLACK_PROXY }} +{% endif %} +{% else %} +#[slack_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +#timeout = 5 + +# (boolean value) +#insecure = true + +# (string value) +#ca_certs = + +# (string value) +#proxy = + +# (string value) +#message_template = +{% endif %} + + +### WEBHOOK +{% if 'webhook' in enabled %} +[webhook_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +timeout = {{ NF_WEBHOOK_TIMEOUT | default(5) }} +{% else %} +#[webhook_notifier] + +# +# From monasca_notification +# + +# (integer value) +# Minimum value: 1 +#timeout = 5 +{% endif %} [kafka] @@ -295,12 +292,6 @@ number = {{ ALARM_PROCESSORS | default(2) }} # From monasca_notification # -# -# Comma separated list of Kafka broker host:port -# (list value) -#uri = {{ KAFKA_URI }} - - # List of addresses (with ports) pointing at Kafka cluster. (list value) url = {{ KAFKA_URI }} @@ -349,7 +340,7 @@ max_offset_lag = 600 # Advanced Option: intended for advanced users and not used # by the majority of users, and might have a significant # effect on stability and/or performance. -legacy_kafka_client_enabled = {{ KAFKA_LEGACY_CLIENT_ENABLED | default(false) }} +legacy_kafka_client_enabled = {{ KAFKA_LEGACY_CLIENT_ENABLED }} [keystone] @@ -422,7 +413,7 @@ db = {{ MYSQL_DB }} # Number of notification processors to spawn. (integer value) # Minimum value: 1 -number = {{ NOTIFICATION_PROCESSORS | default(2) }} +number = {{ NOTIFICATION_PROCESSORS }} [orm] @@ -433,7 +424,7 @@ number = {{ NOTIFICATION_PROCESSORS | default(2) }} # # Connection string for sqlalchemy. (string value) -# url = +#url = [postgresql] @@ -499,36 +490,13 @@ sent_notifications_size = 50 # Advanced Option: intended for advanced users and not used # by the majority of users, and might have a significant # effect on stability and/or performance. -# interval = {{ RETRY_INTERVAL | default(30) }} +interval = {{ RETRY_INTERVAL }} # How many times should retrying be tried. (integer value) # Advanced Option: intended for advanced users and not used # by the majority of users, and might have a significant # effect on stability and/or performance. -# max_attempts = {{ RETRY_MAX_ATTEMPTS | default(5) }} - - -[slack_notifier] - -# -# From monasca_notification -# - -# (integer value) -# Minimum value: 1 -#timeout = 5 - -# (boolean value) -#insecure = true - -# (string value) -#ca_certs = - -# (string value) -#proxy = - -# (string value) -#message_template = +max_attempts = {{ RETRY_MAX_ATTEMPTS }} [statsd] @@ -540,15 +508,15 @@ sent_notifications_size = 50 # # Enable or disable self monitoring. (boolean value) -#enable = true +enable = {{ STATSD_ENABLED }} # IP address of statsd server. (host address value) -#host = 127.0.0.1 +host = {{ STATSD_HOST }} # Port of statsd server. (port value) # Minimum value: 0 # Maximum value: 65535 -#port = 8125 +port = {{ STATSD_PORT }} [zookeeper] diff --git a/docker/notification-logging.conf.j2 b/docker/notification-logging.conf.j2 new file mode 100644 index 0000000..6ced8a1 --- /dev/null +++ b/docker/notification-logging.conf.j2 @@ -0,0 +1,33 @@ +[loggers] +keys = root, kafka, plugins + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = {{ LOG_LEVEL }} +handlers = console + +[logger_kafka] +qualname = monasca_common.confluent_kafka +level = {{ LOG_LEVEL_KAFKA }} +handlers = console +propagate = 0 + +[logger_plugins] +qualname = monasca_notification.plugins +level = {{ LOG_LEVEL_PLUGINS }} +handlers = console +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stderr,) +level = DEBUG +formatter = generic + +[formatter_generic] +class = oslo_log.formatters.ContextFormatter diff --git a/docker/start.sh b/docker/start.sh index 5391393..12352b5 100644 --- a/docker/start.sh +++ b/docker/start.sh @@ -27,7 +27,7 @@ python3 /mysql_check.py # Read usage examples: https://pypi.org/project/Templer/ echo "Start script: creating config files from templates" templer -v -f /etc/monasca/monasca-notification.conf.j2 /etc/monasca/monasca-notification.conf - +templer -v -f /etc/monasca/notification-logging.conf.j2 /etc/monasca/notification-logging.conf echo "Checking files" cd /etc/monasca/