From c8116864a14a06e12f741b3b02fd087c8b3353d1 Mon Sep 17 00:00:00 2001 From: Christian Brandstetter Date: Wed, 8 Aug 2018 15:16:02 +0200 Subject: [PATCH] Add tooling for building Docker image Change-Id: I5398a5f38525eabce8b8cf4d686ae68c666a6d00 Story: 2001694 Task: 24231 --- docker/Dockerfile | 65 ++++ docker/README.rst | 98 ++++++ docker/api-config.conf.j2 | 631 ++++++++++++++++++++++++++++++++++++ docker/api-config.ini.j2 | 27 ++ docker/api-gunicorn.conf.j2 | 15 + docker/api-logging.conf.j2 | 71 ++++ docker/build_image.sh | 146 +++++++++ docker/health_check.py | 45 +++ docker/start.sh | 46 +++ 9 files changed, 1144 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/README.rst create mode 100644 docker/api-config.conf.j2 create mode 100644 docker/api-config.ini.j2 create mode 100644 docker/api-gunicorn.conf.j2 create mode 100644 docker/api-logging.conf.j2 create mode 100755 docker/build_image.sh create mode 100755 docker/health_check.py create mode 100644 docker/start.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..8a7a142e3 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,65 @@ +ARG DOCKER_IMAGE=monasca/api +ARG APP_REPO=https://git.openstack.org/openstack/monasca-api + +# Branch, tag or git hash to build from. +ARG REPO_VERSION=master +ARG CONSTRAINTS_BRANCH=master + +# Extra Python3 dependencies. +# gevent is not in upper constrains and v1.3.6 is not working with +# older greenlet. +ARG EXTRA_DEPS="gunicorn gevent==1.3.5 python-memcached influxdb" + +# Always start from `monasca-base` image and use specific tag of it. +ARG BASE_TAG=master +FROM monasca/base:$BASE_TAG + +# Environment variables used for our service or wait scripts. +ENV \ + KAFKA_URI=kafka:9092 \ + KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \ + MONASCA_CONTAINER_API_PORT=8070 \ + INFLUX_HOST=influxdb \ + INFLUX_PORT=8086 \ + INFLUX_USER=mon_api \ + INFLUX_PASSWORD=password \ + INFLUX_DB=mon \ + MYSQL_HOST=mysql \ + MYSQL_USER=monapi \ + MYSQL_PASSWORD=password \ + MYSQL_DB=mon \ + MEMCACHED_URI=memcached:11211 \ + KEYSTONE_IDENTITY_URI=http://keystone:35357 \ + KEYSTONE_AUTH_URI=http://keystone:5000 \ + KEYSTONE_ADMIN_USER=admin \ + KEYSTONE_ADMIN_PASSWORD=secretadmin \ + KEYSTONE_ADMIN_TENANT=admin \ + KEYSTONE_ADMIN_DOMAIN=default \ + GUNICORN_WORKERS=9 \ + GUNICORN_WORKER_CLASS=gevent \ + GUNICORN_WORKER_CONNECTIONS=2000 \ + GUNICORN_BACKLOG=1000 \ + GUNICORN_TIMEOUT=10 \ + ADD_ACCESS_LOG=true \ + ACCESS_LOG_FORMAT="%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" \ + ACCESS_LOG_FIELDS='%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' \ + LOG_LEVEL_ROOT=WARN \ + LOG_LEVEL_CONSOLE=INFO \ + LOG_LEVEL_ACCESS=INFO \ + STAY_ALIVE_ON_FAILURE=false + +# Copy all neccessary files to proper locations. +COPY api-* /etc/monasca/ + +# Run here all additionals steps your service need post installation. +# Stay with only one `RUN` and use `&& \` for next steps to don't create +# unnecessary image layers. Clean at the end to conserve space. +#RUN \ +# echo "Some steps to do after main installation." && \ +# echo "Hello when building." + +# Expose port for specific service. +EXPOSE ${MONASCA_CONTAINER_API_PORT} + +# Implement start script in `start.sh` file. +CMD ["/start.sh"] diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 000000000..d85864999 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,98 @@ +============================ +Docker image for Monasca API +============================ +The Monasca API image is based on the monasca-base image. + + +Building monasca-base image +=========================== +See https://github.com/openstack/monasca-common/tree/master/docker/README.rst + + +Building Monasca API image +========================== + +Example: + $ ./build_image.sh + + +Requirements from monasca-base image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +health_check.py + This file will be used for checking the status of the Monasca API + application. + + +Scripts +~~~~~~~ +start.sh + In this starting script provide all steps that lead to the proper service + start. Including usage of wait scripts and templating of configuration + files. You also could provide the ability to allow running container after + service died for easier debugging. + +build_image.sh + Please read detailed build description inside the script. + + +Environment variables +~~~~~~~~~~~~~~~~~~~~~ +============================== ======================================================================= ========================================== +Variable Default Description +============================== ======================================================================= ========================================== +KAFKA_URI kafka:9092 URI to Apache Kafka (distributed streaming platform) +KAFKA_WAIT_FOR_TOPICS alarm-state-transitions,metrics The topics where metric-api streams the metric messages and alarm-states +KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts +KAFKA_WAIT_DELAY 5 Seconds to wait between attempts +MONASCA_CONTAINER_API_PORT 8070 The port from the metric pipeline endpoint +INFLUX_HOST influxdb The host for influxdb +INFLUX_PORT 8086 The port for influxdb +INFLUX_USER mon_api The influx username +INFLUX_PASSWORD password The influx password +INFLUX_DB mon The influx database name +MYSQL_DB_HOST mysql The host for MySQL +MYSQL_DB_PORT 3306 The port for MySQL +MYSQL_DB_USERNAME monapi The MySQL username +MYSQL_DB_PASSWORD password The MySQL password +MYSQL_DB_DATABASE mon The MySQL database name +MYSQL_WAIT_RETRIES 24 Number of MySQL connection attempts +MYSQL_WAIT_DELAY 5 Seconds to wait between attempts +API_MYSQL_DISABLED unset If 'true' do not use a mysql database. Only metric API will work +MEMCACHED_URI memcached:11211 URI to Keystone authentication cache +AUTHORIZED_ROLES admin,domainuser,domainadmin,monasca-user Roles for Monasca users (full API access) +AGENT_AUTHORIZED_ROLES monasca-agent Roles for Monasca agents (sending data only) +READ_ONLY_AUTHORIZED_ROLES monasca-read-only-user Roles for read only users +DELEGATE_AUTHORIZED_ROLES admin Roles allow to read/write cross tenant ID +KEYSTONE_IDENTITY_URI http://keystone:35357 URI to Keystone admin endpoint +KEYSTONE_AUTH_URI http://keystone:5000 URI to Keystone public endpoint +KEYSTONE_ADMIN_USER admin OpenStack administrator user name +KEYSTONE_ADMIN_PASSWORD secretadmin OpenStack administrator user password +KEYSTONE_ADMIN_TENANT admin OpenStack administrator tenant name +KEYSTONE_ADMIN_DOMAIN default OpenStack administrator domain +GUNICORN_WORKERS 9 Number of gunicorn (WSGI-HTTP server) workers +GUNICORN_WORKER_CLASS gevent Used gunicorn worker class +GUNICORN_WORKER_CONNECTIONS 2000 Number of gunicorn worker connections +GUNICORN_BACKLOG 1000 Number of gunicorn backlogs +GUNICORN_TIMEOUT 10 Gunicorn connection timeout +ADD_ACCESS_LOG false Enable gunicorn request/access logging +ACCESS_LOG_FORMAT "%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" Define the logging format +ACCESS_LOG_FIELDS '%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' Define the fields to be logged +LOG_LEVEL_ROOT WARN Log level for root logging +LOG_LEVEL_CONSOLE INFO Log level for console logging +LOG_LEVEL_ACCESS INFO Log level for access logging +STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours after service fail +============================== ======================================================================= ========================================== + + +Provide Configuration templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* api-config.conf.j2 +* api-config.ini.j2 +* api-logging.conf.j2 + + +Links +~~~~~ +https://docs.openstack.org/monasca-api/latest/ + +https://github.com/openstack/monasca-api/blob/master/README.rst diff --git a/docker/api-config.conf.j2 b/docker/api-config.conf.j2 new file mode 100644 index 000000000..ac63dae91 --- /dev/null +++ b/docker/api-config.conf.j2 @@ -0,0 +1,631 @@ +[DEFAULT] + +# +# From monasca_api +# + +# +# Region that API is running in +# (string value) +region = useast + +# +# Valid periods for notification methods +# (list value) +#valid_notification_periods = 0,60 + +# +# From oslo.log +# + +# If set to true, the logging level will be set to DEBUG instead of the default +# INFO level (boolean value) +# Note: This option can be changed without restarting. +#debug = false + +# The name of a logging configuration file. This file is appended to any +# existing logging configuration files. For details about logging configuration +# files, see the Python logging module documentation. Note that when logging +# configuration files are used then all logging configuration is set in the +# configuration file and other logging configuration options are ignored (for +# example, logging_context_format_string) (string value) +# Note: This option can be changed without restarting. +# Deprecated group/name - [DEFAULT]/log_config +log_config_append=/etc/monasca/api-logging.conf + +# Defines the format string for %%(asctime)s in log records. Default: +# %(default)s . This option is ignored if log_config_append is set (string +# value) +#log_date_format = %Y-%m-%d %H:%M:%S + +# (Optional) Name of log file to send logging output to. If no default is set, +# logging will go to stderr as defined by use_stderr. This option is ignored if +# log_config_append is set (string value) +# Deprecated group/name - [DEFAULT]/logfile +#log_file = + +# (Optional) The base directory used for relative log_file paths. This option +# is ignored if log_config_append is set (string value) +# Deprecated group/name - [DEFAULT]/logdir +#log_dir = + +# Uses logging handler designed to watch file system. When log file is moved or +# removed this handler will open a new log file with specified path +# instantaneously. It makes sense only if log_file option is specified and +# Linux platform is used. This option is ignored if log_config_append is set +# (boolean value) +#watch_log_file = false + +# Use syslog for logging. Existing syslog format is DEPRECATED and will be +# changed later to honor RFC5424. This option is ignored if log_config_append +# is set (boolean value) +#use_syslog = false + +# Enable journald for logging. If running in a systemd environment you may wish +# to enable journal support. Doing so will use the journal native protocol +# which includes structured metadata in addition to log messages.This option is +# ignored if log_config_append is set (boolean value) +#use_journal = false + +# Syslog facility to receive log lines. This option is ignored if +# log_config_append is set (string value) +#syslog_log_facility = LOG_USER + +# Use JSON formatting for logging. This option is ignored if log_config_append +# is set (boolean value) +#use_json = false + +# Log output to standard error. This option is ignored if log_config_append is +# set (boolean value) +#use_stderr = false + +# Format string to use for log messages with context (string value) +#logging_context_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [%(request_id)s %(user_identity)s] %(instance)s%(message)s + +# Format string to use for log messages when context is undefined (string +# value) +#logging_default_format_string = %(asctime)s.%(msecs)03d %(process)d %(levelname)s %(name)s [-] %(instance)s%(message)s + +# Additional data to append to log message when logging level for the message +# is DEBUG (string value) +#logging_debug_format_suffix = %(funcName)s %(pathname)s:%(lineno)d + +# Prefix each line of exception output with this format (string value) +#logging_exception_prefix = %(asctime)s.%(msecs)03d %(process)d ERROR %(name)s %(instance)s + +# Defines the format string for %(user_identity)s that is used in +# logging_context_format_string (string value) +#logging_user_identity_format = %(user)s %(tenant)s %(domain)s %(user_domain)s %(project_domain)s + +# List of package logging levels in logger=LEVEL pairs. This option is ignored +# if log_config_append is set (list value) +#default_log_levels = amqp=WARN,amqplib=WARN,boto=WARN,qpid=WARN,sqlalchemy=WARN,suds=INFO,oslo.messaging=INFO,oslo_messaging=INFO,iso8601=WARN,requests.packages.urllib3.connectionpool=WARN,urllib3.connectionpool=WARN,websocket=WARN,requests.packages.urllib3.util.retry=WARN,urllib3.util.retry=WARN,keystonemiddleware=WARN,routes.middleware=WARN,stevedore=WARN,taskflow=WARN,keystoneauth=WARN,oslo.cache=INFO,dogpile.core.dogpile=INFO + +# Enables or disables publication of error events (boolean value) +#publish_errors = false + +# The format for an instance that is passed with the log message (string value) +#instance_format = "[instance: %(uuid)s] " + +# The format for an instance UUID that is passed with the log message (string +# value) +#instance_uuid_format = "[instance: %(uuid)s] " + +# Interval, number of seconds, of log rate limiting (integer value) +#rate_limit_interval = 0 + +# Maximum number of logged messages per rate_limit_interval (integer value) +#rate_limit_burst = 0 + +# Log level name used by rate limiting: CRITICAL, ERROR, INFO, WARNING, DEBUG +# or empty string. Logs with level greater or equal to rate_limit_except_level +# are not filtered. An empty string means that all levels are filtered (string +# value) +#rate_limit_except_level = CRITICAL + +# Enables or disables fatal status of deprecations (boolean value) +#fatal_deprecations = false + + +[cassandra] + +# +# From monasca_api +# + +# +# Comma separated list of Cassandra node IP addresses +# (list value) +#contact_points = 127.0.0.1 + +# +# keyspace where metric are stored +# (string value) +#keyspace = monasca + +# +# Cassandra user for monasca-api service +# (string value) +#user = + +# +# Cassandra user password for monasca-api service +# (string value) +#password = + + +[database] + +# +# From monasca_api +# + +# DEPRECATED: +# The SQLAlchemy connection string to use to connect to the database +# (string value) +# This option is deprecated for removal since 1.6.0. +# Its value may be silently ignored in the future. +# Reason: Please use database.connection option,database.url is scheduled for +# removal in Pike release +#url = $database.connection + +# +# From oslo.db +# + +# If True, SQLite uses synchronous mode (boolean value) +#sqlite_synchronous = true + +# The back end to use for the database (string value) +# Deprecated group/name - [DEFAULT]/db_backend +#backend = sqlalchemy + +# The SQLAlchemy connection string to use to connect to the database (string +# value) +# Deprecated group/name - [DEFAULT]/sql_connection +# Deprecated group/name - [DATABASE]/sql_connection +# Deprecated group/name - [sql]/connection +{% if not ( API_MYSQL_DISABLED is defined and API_MYSQL_DISABLED | lower == 'true' ) %} +connection = "mysql+pymysql://{{ MYSQL_USER }}:{{ MYSQL_PASSWORD }}@{{ MYSQL_HOST }}/{{ MYSQL_DB }}" +{% endif %} + + +# The SQLAlchemy connection string to use to connect to the slave database +# (string value) +#slave_connection = + +# The SQL mode to be used for MySQL sessions. This option, including the +# default, overrides any server-set SQL mode. To use whatever SQL mode is set +# by the server configuration, set this to no value. Example: mysql_sql_mode= +# (string value) +#mysql_sql_mode = TRADITIONAL + +# If True, transparently enables support for handling MySQL Cluster (NDB) +# (boolean value) +#mysql_enable_ndb = false + +# Connections which have been present in the connection pool longer than this +# number of seconds will be replaced with a new one the next time they are +# checked out from the pool (integer value) +# Deprecated group/name - [DATABASE]/idle_timeout +# Deprecated group/name - [database]/idle_timeout +# Deprecated group/name - [DEFAULT]/sql_idle_timeout +# Deprecated group/name - [DATABASE]/sql_idle_timeout +# Deprecated group/name - [sql]/idle_timeout +#connection_recycle_time = 3600 + +# DEPRECATED: Minimum number of SQL connections to keep open in a pool (integer +# value) +# Deprecated group/name - [DEFAULT]/sql_min_pool_size +# Deprecated group/name - [DATABASE]/sql_min_pool_size +# This option is deprecated for removal. +# Its value may be silently ignored in the future. +# Reason: The option to set the minimum pool size is not supported by +# sqlalchemy. +#min_pool_size = 1 + +# Maximum number of SQL connections to keep open in a pool. Setting a value of +# 0 indicates no limit (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_pool_size +# Deprecated group/name - [DATABASE]/sql_max_pool_size +#max_pool_size = 5 + +# Maximum number of database connection retries during startup. Set to -1 to +# specify an infinite retry count (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_retries +# Deprecated group/name - [DATABASE]/sql_max_retries +#max_retries = 10 + +# Interval between retries of opening a SQL connection (integer value) +# Deprecated group/name - [DEFAULT]/sql_retry_interval +# Deprecated group/name - [DATABASE]/reconnect_interval +#retry_interval = 10 + +# If set, use this value for max_overflow with SQLAlchemy (integer value) +# Deprecated group/name - [DEFAULT]/sql_max_overflow +# Deprecated group/name - [DATABASE]/sqlalchemy_max_overflow +#max_overflow = 50 + +# Verbosity of SQL debugging information: 0=None, 100=Everything (integer +# value) +# Minimum value: 0 +# Maximum value: 100 +# Deprecated group/name - [DEFAULT]/sql_connection_debug +#connection_debug = 0 + +# Add Python stack traces to SQL as comment strings (boolean value) +# Deprecated group/name - [DEFAULT]/sql_connection_trace +#connection_trace = false + +# If set, use this value for pool_timeout with SQLAlchemy (integer value) +# Deprecated group/name - [DATABASE]/sqlalchemy_pool_timeout +#pool_timeout = + +# Enable the experimental use of database reconnect on connection lost (boolean +# value) +#use_db_reconnect = false + +# Seconds between retries of a database transaction (integer value) +#db_retry_interval = 1 + +# If True, increases the interval between retries of a database operation up to +# db_max_retry_interval (boolean value) +#db_inc_retry_interval = true + +# If db_inc_retry_interval is set, the maximum seconds between retries of a +# database operation (integer value) +#db_max_retry_interval = 10 + +# Maximum retries in case of connection error or deadlock error before error is +# raised. Set to -1 to specify an infinite retry count (integer value) +#db_max_retries = 20 + +# Optional URL parameters to append onto the connection URL at connect time; +# specify as param1=value1¶m2=value2& (string value) +#connection_parameters = + + +[dispatcher] + +# +# From monasca_api +# + +# Versions controller (string value) +versions = monasca_api.v2.reference.versions:Versions + +# Version 2.0 controller (string value) +version_2_0 = monasca_api.v2.reference.version_2_0:Version2 + +# Metrics controller (string value) +metrics = monasca_api.v2.reference.metrics:Metrics + +# Metrics measurements controller (string value) +metrics_measurements = monasca_api.v2.reference.metrics:MetricsMeasurements + +# Metrics statistics controller (string value) +metrics_statistics = monasca_api.v2.reference.metrics:MetricsStatistics + +# Metrics names controller (string value) +metrics_names = monasca_api.v2.reference.metrics:MetricsNames + +# Alarm definitions controller (string value) +alarm_definitions = monasca_api.v2.reference.alarm_definitions:AlarmDefinitions + +# Alarms controller (string value) +alarms = monasca_api.v2.reference.alarms:Alarms + +# Alarms Count controller (string value) +alarms_count = monasca_api.v2.reference.alarms:AlarmsCount + +# Alarms state history controller (string value) +alarms_state_history = monasca_api.v2.reference.alarms:AlarmsStateHistory + +# Notification Methods controller (string value) +notification_methods = monasca_api.v2.reference.notifications:Notifications + +# Dimension Values controller (string value) +dimension_values = monasca_api.v2.reference.metrics:DimensionValues + +# Dimension Names controller (string value) +dimension_names = monasca_api.v2.reference.metrics:DimensionNames + +# Notifications Type Methods controller (string value) +notification_method_types = monasca_api.v2.reference.notificationstype:NotificationsType + +# Health checks endpoint controller (string value) +healthchecks = monasca_api.healthchecks:HealthChecks + + +[influxdb] + +# +# From monasca_api +# + +# +# Database name where metrics are stored +# (string value) +database_name = {{ INFLUX_DB }} + +# +# IP address to Influxdb server +# (host address value) +ip_address = {{ INFLUX_HOST }} + +# Port to Influxdb server (port value) +# Minimum value: 0 +# Maximum value: 65535 +port = {{ INFLUX_PORT }} + +# +# Influxdb user +# (string value) +user = {{ INFLUX_USER }} + +# +# Influxdb password +# (string value) +password = {{ INFLUX_PASSWORD }} + + +[kafka] + +# +# From monasca_api +# + +# +# Comma separated list of Kafka broker host:port +# (list value) +uri = {{ KAFKA_URI }} + +# +# The topic that metrics will be published to +# (string value) +metrics_topic = metrics + +# +# The topic that events will be published too +# (string value) +#events_topic = events + +# +# The topic that alarm state will be published too +# (string value) +#alarm_state_transitions_topic = alarm-state-transitions + +# +# The group name that this service belongs to +# (string value) +group = api + +# +# The ack time back to kafka. +# (integer value) +#ack_time = 20 + +# +# The number of retry when there is a connection error +# (integer value) +max_retry = 1 + +# +# The type of posting +# (boolean value) +async = true + +# +# Specify if the message received should be parsed. +# If True, message will not be parsed, otherwise +# messages will be parsed +# (boolean value) +compact = true + +# +# The partitions this connection should +# listen for messages on. Currently does not +# support multiple partitions. +# Default is to listen on partition 0 +# (list value) +partitions = 0 + +# +# Specify if received data should be simply dropped. +# This parameter is only for testing purposes +# (boolean value) +#drop_data = false + +# +# The wait time when no messages on kafka queue +# (integer value) +# Minimum value: 1 +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +wait_time = 1 + +# +# Should messages be automatically committed +# (boolean value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +#auto_commit = false + + +[messaging] + +# +# From monasca_api +# + +# +# The message queue driver to use +# (string value) +driver = monasca_api.common.messaging.kafka_publisher:KafkaPublisher + +# DEPRECATED: +# The type of metrics message format to publish to the message queue +# (string value) +# This option is deprecated for removal since 2.1.0. +# Its value may be silently ignored in the future. +# Reason: +# Option is not used anywhere in the codebase +#metrics_message_format = reference + +# DEPRECATED: +# The type of events message format to publish to the message queue +# (string value) +# This option is deprecated for removal since 2.1.0. +# Its value may be silently ignored in the future. +# Reason: +# Option is not used anywhere in the codebase +#events_message_format = reference + + +[oslo_policy] + +# +# From oslo.policy +# + +# This option controls whether or not to enforce scope when evaluating +# policies. If ``True``, the scope of the token used in the request is compared +# to the ``scope_types`` of the policy being enforced. If the scopes do not +# match, an ``InvalidScope`` exception will be raised. If ``False``, a message +# will be logged informing operators that policies are being invoked with +# mismatching scope (boolean value) +#enforce_scope = false + +# The file that defines policies (string value) +#policy_file = policy.json + +# Default rule. Enforced when a requested rule is not found (string value) +#policy_default_rule = default + +# Directories where policy configuration files are stored. They can be relative +# to any directory in the search path defined by the config_dir option, or +# absolute paths. The file defined by policy_file must exist for these +# directories to be searched. Missing or empty directories are ignored (multi +# valued) +#policy_dirs = policy.d + +# Content Type to send and receive data for REST based policy check (string +# value) +# Possible values: +# application/x-www-form-urlencoded - +# application/json - +#remote_content_type = application/x-www-form-urlencoded + +# server identity verification for REST based policy check (boolean value) +#remote_ssl_verify_server_crt = false + +# Absolute path to ca cert file for REST based policy check (string value) +#remote_ssl_ca_crt_file = + +# Absolute path to client cert for REST based policy check (string value) +#remote_ssl_client_crt_file = + +# Absolute path client key file REST based policy check (string value) +#remote_ssl_client_key_file = + + +[repositories] + +# +# From monasca_api +# + +# +# The repository driver to use for metrics +# (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +metrics_driver = monasca_api.common.repositories.influxdb.metrics_repository:MetricsRepository + +# +# The repository driver to use for alarm definitions +# (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +alarm_definitions_driver = monasca_api.common.repositories.sqla.alarm_definitions_repository:AlarmDefinitionsRepository + +# +# The repository driver to use for alarms +# (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +alarms_driver = monasca_api.common.repositories.sqla.alarms_repository:AlarmsRepository + +# +# The repository driver to use for notifications +# (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notifications_driver = monasca_api.common.repositories.sqla.notifications_repository:NotificationsRepository + +# +# The repository driver to use for notifications +# (string value) +# Advanced Option: intended for advanced users and not used +# by the majority of users, and might have a significant +# effect on stability and/or performance. +notification_method_type_driver = monasca_api.common.repositories.sqla.notification_method_type_repository:NotificationMethodTypeRepository + + +[security] + +# +# From monasca_api +# + +# Roles that are allowed to check the health (list value) +#healthcheck_roles = @ + +# Roles that are allowed to check the versions (list value) +#versions_roles = @ + +# +# Roles that are allowed full access to the API +# (list value) +default_authorized_roles = {{ AUTHORIZED_ROLES | default('admin, domainuser, domainadmin, monasca-user') }} + +# +# Roles that are only allowed to POST to the API +# (list value) +agent_authorized_roles = {{ AGENT_AUTHORIZED_ROLES | default('monasca-agent') }} + +# +# Roles that are only allowed to GET from the API +# (list value) +read_only_authorized_roles = {{ READ_ONLY_AUTHORIZED_ROLES | default('monasca-read-only-user') }} + +# +# Roles that are allowed to POST metrics on +# behalf of another tenant +# (list value) +delegate_authorized_roles = {{ DELEGATE_AUTHORIZED_ROLES | default('admin') }} + +[dispatcher] +driver = v2_reference + +[keystone_authtoken] +auth_type = password +auth_url = {{ KEYSTONE_IDENTITY_URI }} +auth_uri = {{ KEYSTONE_AUTH_URI }} +username = {{ KEYSTONE_ADMIN_USER }} +password = {{ KEYSTONE_ADMIN_PASSWORD }} +user_domain_name = Default +project_name = {{ KEYSTONE_ADMIN_TENANT }} +project_domain_name = Default +service_token_roles_required = true +memcached_servers = {{ MEMCACHED_URI }} +insecure = false +cafile = +certfile = +keyfile = diff --git a/docker/api-config.ini.j2 b/docker/api-config.ini.j2 new file mode 100644 index 000000000..e0f3b44dc --- /dev/null +++ b/docker/api-config.ini.j2 @@ -0,0 +1,27 @@ +[DEFAULT] +name = monasca_api + +[pipeline:main] +pipeline = request_id auth api + +[app:api] +paste.app_factory = monasca_api.api.server:launch + +[filter:auth] +paste.filter_factory = monasca_api.healthcheck.keystone_protocol:filter_factory + +[filter:request_id] +paste.filter_factory = oslo_middleware.request_id:RequestId.factory + +[server:main] +use = egg:gunicorn#main +host = 0.0.0.0 +port = {{ MONASCA_CONTAINER_API_PORT }} +workers = 9 +worker-connections = 2000 +worker-class = eventlet +timeout = 30 +backlog = 2048 +keepalive = 2 +proc_name = monasca_api +#loglevel = DEBUG diff --git a/docker/api-gunicorn.conf.j2 b/docker/api-gunicorn.conf.j2 new file mode 100644 index 000000000..d6bca3725 --- /dev/null +++ b/docker/api-gunicorn.conf.j2 @@ -0,0 +1,15 @@ +bind = '0.0.0.0:{{ MONASCA_CONTAINER_API_PORT }}' +proc_name = 'monasca-api' + +backlog = {{ GUNICORN_BACKLOG | int }} +workers = {{ GUNICORN_WORKERS | int }} +worker_class = '{{ GUNICORN_WORKER_CLASS }}' +worker_connections = '{{ GUNICORN_WORKER_CONNECTIONS }}' +timeout = {{ GUNICORN_TIMEOUT | int }} + +{% if ADD_ACCESS_LOG == true %} +accesslog = '-' +{% endif %} +access_log_format = '{{ ACCESS_LOG_FIELDS }}' + +capture_output = True diff --git a/docker/api-logging.conf.j2 b/docker/api-logging.conf.j2 new file mode 100644 index 000000000..517f46dae --- /dev/null +++ b/docker/api-logging.conf.j2 @@ -0,0 +1,71 @@ +[default] +disable_existing_loggers = 0 + +[loggers] +keys = root, gunicorn.access, sqlalchemy, kafka, kafka.consumer, urllib3 + +[handlers] +keys = console, console_access + +[formatters] +keys = context, generic + +[logger_root] +level = {{ LOG_LEVEL_ROOT }} +handlers = console + +[logger_gunicorn.access] +level = INFO +handlers = console_access +propagate = 0 +qualname = gunicorn.access + +[logger_sqlalchemy] +qualname = sqlalchemy.engine +# "level = INFO" logs SQL queries. +# "level = DEBUG" logs SQL queries and results. +# "level = WARN" logs neither. (Recommended for production systems.) +level = ERROR +handlers = console +propagate=0 + +[logger_kafka.consumer] +qualname = kafka.consumer +level = INFO +formatter = default +handlers = console +propagate = 0 + +[logger_kafka] +qualname = monasca_common.kafka_lib +level = INFO +formatter = default +handlers = console +propagate = 0 + +[logger_urllib3] +qualname = urllib3.connectionpool +level = INFO +formatter = default +handlers = console +propagate = 0 + +[handler_console] +class = logging.StreamHandler +args = (sys.stdout,) +level = {{ LOG_LEVEL_CONSOLE }} +formatter = context + +[handler_console_access] +class = logging.StreamHandler +args = (sys.stdout,) +level = {{ LOG_LEVEL_ACCESS }} +formatter = generic + +[formatter_context] +class = oslo_log.formatters.ContextFormatter + +[formatter_generic] +format={{ ACCESS_LOG_FORMAT }} +datefmt=%Y-%m-%d %H:%M:%S +class=logging.Formatter diff --git a/docker/build_image.sh b/docker/build_image.sh new file mode 100755 index 000000000..1dccc4886 --- /dev/null +++ b/docker/build_image.sh @@ -0,0 +1,146 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# TODO(Dobroslaw): move this script to monasca-common/docker folder +# and leave here small script to download it and execute using env variables +# to minimize code duplication. + +set -x # Print each script step. +set -eo pipefail # Exit the script if any statement returns error. + +# This script is used for building Docker image with proper labels +# and proper version of monasca-common. +# +# Example usage: +# $ ./build_image.sh +# +# Everything after `./build_image.sh` is optional and by default configured +# to get versions from `Dockerfile`. +# +# To build from master branch (default): +# $ ./build_image.sh +# To build specific version run this script in the following way: +# $ ./build_image.sh stable/queens +# Building from specific commit: +# $ ./build_image.sh cb7f226 +# When building from a tag monasca-common will be used in version available +# in upper constraint file: +# $ ./build_image.sh 2.5.0 +# To build image from Gerrit patch sets that is targeting branch stable/queens: +# $ ./build_image.sh refs/changes/51/558751/1 stable/queens +# +# If you want to build image with custom monasca-common version you need +# to provide it as in the following example: +# $ ./build_image.sh master master refs/changes/19/595719/3 + +[ -z "$DOCKER_IMAGE" ] && \ + DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=") + +: "${REPO_VERSION:=$1}" +[ -z "$REPO_VERSION" ] && \ + REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=") +# Let's stick to more readable version and disable SC2001 here. +# shellcheck disable=SC2001 +REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g') + +[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=") +GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \ + sed 's/ssh:/https:/') + +if [ -z "$CONSTRAINTS_FILE" ]; then + CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true + : "${CONSTRAINTS_FILE:=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}" +fi + +: "${CONSTRAINTS_BRANCH:=$2}" +[ -z "$CONSTRAINTS_BRANCH" ] && \ + CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=") + +# When using stable version of repository use same stable constraints file. +case "$REPO_VERSION" in + *stable*) + CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION" + # Get monasca-common version from stable upper constraints file. + CONSTRAINTS_TMP_FILE=$(mktemp) + wget --output-document "$CONSTRAINTS_TMP_FILE" \ + "$CONSTRAINTS_FILE"?h="$CONSTRAINTS_BRANCH_CLEAN" + UPPER_COMMON=$(\grep 'monasca-common' "$CONSTRAINTS_TMP_FILE") + # Get only version part from monasca-common. + UPPER_COMMON_VERSION="${UPPER_COMMON##*===}" + rm -rf "$CONSTRAINTS_TMP_FILE" + ;; + *) + CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH" + ;; +esac + +# Monasca-common variables. +if [ -z "$COMMON_REPO" ]; then + COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true + : "${COMMON_REPO:=https://git.openstack.org/openstack/monasca-common}" +fi +: "${COMMON_VERSION:=$3}" +if [ -z "$COMMON_VERSION" ]; then + COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true + if [ "$UPPER_COMMON_VERSION" ]; then + # Common from upper constraints file. + COMMON_VERSION="$UPPER_COMMON_VERSION" + fi +fi + +# Clone project to temporary directory for getting proper commit number from +# branches and tags. We need this for setting proper image labels. +# Docker does not allow to get any data from inside of system when building +# image. +TMP_DIR=$(mktemp -d) +( + cd "$TMP_DIR" + # This many steps are needed to support gerrit patch sets. + git init + git remote add origin "$APP_REPO" + git fetch origin "$REPO_VERSION" + git reset --hard FETCH_HEAD +) +GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse HEAD) +[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 +rm -rf "$TMP_DIR" + +# Do the same for monasca-common. +COMMON_TMP_DIR=$(mktemp -d) +( + cd "$COMMON_TMP_DIR" + # This many steps are needed to support gerrit patch sets. + git init + git remote add origin "$COMMON_REPO" + git fetch origin "$COMMON_VERSION" + git reset --hard FETCH_HEAD +) +COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse HEAD) +[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 +rm -rf "$COMMON_TMP_DIR" + +CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +docker build --no-cache \ + --build-arg CREATION_TIME="$CREATION_TIME" \ + --build-arg GITHUB_REPO="$GITHUB_REPO" \ + --build-arg APP_REPO="$APP_REPO" \ + --build-arg REPO_VERSION="$REPO_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \ + --build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \ + --build-arg COMMON_REPO="$COMMON_REPO" \ + --build-arg COMMON_VERSION="$COMMON_VERSION" \ + --build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \ + --tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" . diff --git a/docker/health_check.py b/docker/health_check.py new file mode 100755 index 000000000..df5c5aa0d --- /dev/null +++ b/docker/health_check.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# coding=utf-8 + +# (C) Copyright 2018 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Health check will returns 0 when service is working properly.""" + +import logging +from urllib import request +import os +import sys + + +LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO')) +logging.basicConfig(level=LOG_LEVEL) +logger = logging.getLogger(__name__) + +API_PORT = os.environ.get('MONASCA_CONTAINER_API_PORT', '8070') +url = "http://localhost:" + API_PORT + "/healthcheck" + + +def main(): + """Send health check request to health check endpoint of Monasca API.""" + logger.debug('Send health check request to %s', url) + try: + request.urlopen(url=url) + except Exception as ex: + logger.error('Exception during request handling: ' + repr(ex)) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/docker/start.sh b/docker/start.sh new file mode 100644 index 000000000..9c8e25262 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,46 @@ +#!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Starting script. +# All checks and configuration templating you need to do before service +# could be safely started should be added in this file. + +set -eo pipefail # Exit the script if any statement returns error. + +# Test services we need before starting our service. +echo "Start script: waiting for needed services" +python3 /kafka_wait_for_topics.py +python3 /mysql_check.py + +# Template all config files before start, it will use env variables. +# Read usage examples: https://pypi.org/project/Templer/ +echo "Start script: creating config files from templates" +templer -v -f /etc/monasca/api-config.conf.j2 /etc/monasca/api-config.conf +templer -v -f /etc/monasca/api-config.ini.j2 /etc/monasca/api-config.ini +templer -v -f /etc/monasca/api-logging.conf.j2 /etc/monasca/api-logging.conf +templer -v -f /etc/monasca/api-gunicorn.conf.j2 /etc/monasca/api-gunicorn.conf + +# Start our service. +echo "Start script: starting container" +gunicorn \ + --config /etc/monasca/api-gunicorn.conf \ + --paste /etc/monasca/api-config.ini + +# Allow server to stay alive in case of failure for 2 hours for debugging. +RESULT=$? +if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then + echo "Service died, waiting 120 min before exiting" + sleep 7200 +fi +exit $RESULT