Add tooling for building Docker image

Story: 2001694
Task: 22630

Change-Id: Icf82a299ee2cd449e9c6bc58fabe471a9292f9df
This commit is contained in:
Christian Brandstetter 2018-06-11 16:44:00 +02:00 committed by Dobroslaw Zybort
parent fc125c3f90
commit 119e4a99a0
9 changed files with 535 additions and 0 deletions

57
docker/Dockerfile Normal file
View File

@ -0,0 +1,57 @@
ARG DOCKER_IMAGE=monasca/log-api
ARG APP_REPO=https://git.openstack.org/openstack/monasca-log-api
# Branch, tag or git hash to build from.
ARG REPO_VERSION=master
ARG CONSTRAINTS_BRANCH=master
# Extra Python3 dependencies.
ARG EXTRA_DEPS="gunicorn python-memcached gevent"
# Always start from `monasca-base` image and use specific tag of it.
ARG BASE_TAG=master
FROM monasca/base:$BASE_TAG
# Environment variables used for our service or wait scripts.
ENV \
KAFKA_URI=kafka:9092 \
KAFKA_WAIT_FOR_TOPICS=log \
MONASCA_CONTAINER_LOG_API_PORT=5607 \
MEMCACHED_URI=memcached:11211 \
AUTHORIZED_ROLES=admin,domainuser,domainadmin,monasca-user \
AGENT_AUTHORIZED_ROLES=monasca-agent \
KEYSTONE_IDENTITY_URI=http://keystone:35357 \
KEYSTONE_AUTH_URI=http://keystone:5000 \
KEYSTONE_ADMIN_USER=admin \
KEYSTONE_ADMIN_PASSWORD=secretadmin \
KEYSTONE_ADMIN_TENANT=admin \
KEYSTONE_ADMIN_DOMAIN=default \
GUNICORN_WORKERS=9 \
GUNICORN_WORKER_CLASS=gevent \
GUNICORN_WORKER_CONNECTIONS=2000 \
GUNICORN_BACKLOG=1000 \
GUNICORN_TIMEOUT=10 \
PYTHONIOENCODING=utf-8 \
ADD_ACCESS_LOG=false \
ACCESS_LOG_FORMAT="%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" \
ACCESS_LOG_FIELDS='%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' \
LOG_LEVEL_ROOT=INFO \
LOG_LEVEL_CONSOLE=INFO \
LOG_LEVEL_ACCESS=INFO \
STAY_ALIVE_ON_FAILURE="false"
# Copy all neccessary files to proper locations.
COPY log-api* /etc/monasca/
# Run here all additionals steps your service need post installation.
# Stay with only one `RUN` and use `&& \` for next steps to don't create
# unnecessary image layers. Clean at the end to conserve space.
#RUN \
# echo "Some steps to do after main installation." && \
# echo "Hello when building."
# Expose port for specific service.
EXPOSE ${MONASCA_CONTAINER_LOG_API_PORT}
# Implement start script in `start.sh` file.
CMD ["/start.sh"]

85
docker/README.rst Normal file
View File

@ -0,0 +1,85 @@
================================
Docker image for Monasca Log API
================================
The Monasca log API image is based on the monasca-base image.
Building monasca-base image
===========================
See https://github.com/openstack/monasca-common/tree/master/docker/README.rst
Building Monasca log API image
==============================
Example:
$ ./build_image.sh <repository_version> <upper_constains_branch> <common_version>
Requirements from monasca-base image
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
health_check.py
This file will be used for checking the status of the Monasca Log API
application.
Scripts
~~~~~~~
start.sh
In this starting script provide all steps that lead to the proper service
start. Including usage of wait scripts and templating of configuration
files. You also could provide the ability to allow running container after
service died for easier debugging.
build_image.sh
Please read detailed build description inside the script.
Environment variables
~~~~~~~~~~~~~~~~~~~~~
============================== ======================================================================= ==========================================
Variable Default Description
============================== ======================================================================= ==========================================
KAFKA_URI kafka:9092 URI to Apache Kafka (distributed streaming platform)
KAFKA_WAIT_FOR_TOPICS log The topic where log-api streams the log messages
KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts
KAFKA_WAIT_DELAY 5 Seconds to wait between attempts
MONASCA_CONTAINER_LOG_API_PORT 5607 The port from the log pipeline endpoint
MEMCACHED_URI memcached:11211 URI to Keystone authentication cache
AUTHORIZED_ROLES admin,domainuser,domainadmin,monasca-user Roles for Monasca users (full API access)
AGENT_AUTHORIZED_ROLES monasca-agent Roles for Monasca agents (sending data only)
KEYSTONE_IDENTITY_URI http://keystone:35357 URI to Keystone admin endpoint
KEYSTONE_AUTH_URI http://keystone:5000 URI to Keystone public endpoint
KEYSTONE_ADMIN_USER admin OpenStack administrator user name
KEYSTONE_ADMIN_PASSWORD secretadmin OpenStack administrator user password
KEYSTONE_ADMIN_TENANT admin OpenStack administrator tenant name
KEYSTONE_ADMIN_DOMAIN default OpenStack administrator domain
GUNICORN_WORKERS 9 Number of gunicorn (WSGI-HTTP server) workers
GUNICORN_WORKER_CLASS gevent Used gunicorn worker class
GUNICORN_WORKER_CONNECTIONS 2000 Number of gunicorn worker connections
GUNICORN_BACKLOG 1000 Number of gunicorn backlogs
GUNICORN_TIMEOUT 10 Gunicorn connection timeout
PYTHONIOENCODING utf-8 Python encoding
ADD_ACCESS_LOG false Enable gunicorn request/access logging
ACCESS_LOG_FORMAT "%(asctime)s [%(process)d] gunicorn.access [%(levelname)s] %(message)s" Define the logging format
ACCESS_LOG_FIELDS '%(h)s %(l)s %(u)s %(t)s %(r)s %(s)s %(b)s "%(f)s" "%(a)s" %(L)s' Define the fields to be logged
LOG_LEVEL_ROOT WARN Log level for root logging
LOG_LEVEL_CONSOLE INFO Log level for console logging
LOG_LEVEL_ACCESS INFO Log level for access logging
STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours after tests fail
============================== ======================================================================= ==========================================
Provide configuration templates
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
* log-api.conf.j2
* log-api-gunicorn.conf.j2
* log-api-logging.conf.j2
* log-api.paste.ini.j2
Links
~~~~~
https://docs.openstack.org/monasca-log-api/latest/configuration/
https://github.com/openstack/monasca-log-api/blob/master/README.rst

147
docker/build_image.sh Executable file
View File

@ -0,0 +1,147 @@
#!/bin/bash
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# TODO(Dobroslaw): move this script to monasca-common/docker folder
# and leave here small script to download it and execute using env variables
# to minimize code duplication.
set -x # Print each script step.
set -eo pipefail # Exit the script if any statement returns error.
# This script is used for building Docker image with proper labels
# and proper version of monasca-common.
#
# Example usage:
# $ ./build_image.sh <repository_version> <upper_constains_branch> <common_version>
#
# Everything after `./build_image.sh` is optional and by default configured
# to get versions from `Dockerfile`.
#
# To build from master branch (default):
# $ ./build_image.sh
# To build specific version run this script in the following way:
# $ ./build_image.sh stable/queens
# Building from specific commit:
# $ ./build_image.sh cb7f226
# When building from a tag monasca-common will be used in version available
# in upper constraint file:
# $ ./build_image.sh 2.5.0
# To build image from Gerrit patch sets that is targeting branch stable/queens:
# $ ./build_image.sh refs/changes/51/558751/1 stable/queens
#
# If you want to build image with custom monasca-common version you need
# to provide it as in the following example:
# $ ./build_image.sh master master refs/changes/19/595719/3
[ -z "$DOCKER_IMAGE" ] && \
DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=")
: "${REPO_VERSION:=$1}"
[ -z "$REPO_VERSION" ] && \
REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=")
# Let's stick to more readable version and disable SC2001 here.
# shellcheck disable=SC2001
REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g')
[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=")
GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \
sed 's/ssh:/https:/')
if [ -z "$CONSTRAINTS_FILE" ]; then
CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true
: "${CONSTRAINTS_FILE:=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}"
fi
: "${CONSTRAINTS_BRANCH:=$2}"
[ -z "$CONSTRAINTS_BRANCH" ] && \
CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=")
# When using stable version of repository use same stable constraints file.
case "$REPO_VERSION" in
*stable*)
CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION"
# Get monasca-common version from stable upper constraints file.
CONSTRAINTS_TMP_FILE=$(mktemp)
wget --output-document "$CONSTRAINTS_TMP_FILE" \
"$CONSTRAINTS_FILE"?h="$CONSTRAINTS_BRANCH_CLEAN"
UPPER_COMMON=$(\grep 'monasca-common' "$CONSTRAINTS_TMP_FILE")
# Get only version part from monasca-common.
UPPER_COMMON_VERSION="${UPPER_COMMON##*===}"
rm -rf "$CONSTRAINTS_TMP_FILE"
;;
*)
CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH"
;;
esac
# Monasca-common variables.
if [ -z "$COMMON_REPO" ]; then
COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true
: "${COMMON_REPO:=https://git.openstack.org/openstack/monasca-common}"
fi
: "${COMMON_VERSION:=$3}"
if [ -z "$COMMON_VERSION" ]; then
COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true
if [ "$UPPER_COMMON_VERSION" ]; then
# Common from upper constraints file.
COMMON_VERSION="$UPPER_COMMON_VERSION"
fi
fi
# Clone project to temporary directory for getting proper commit number from
# branches and tags. We need this for setting proper image labels.
# Docker does not allow to get any data from inside of system when building
# image.
TMP_DIR=$(mktemp -d)
(
cd "$TMP_DIR"
# This many steps are needed to support gerrit patch sets.
git init
git remote add origin "$APP_REPO"
git fetch origin "$REPO_VERSION"
git reset --hard FETCH_HEAD
)
GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse HEAD)
[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
rm -rf "$TMP_DIR"
# Do the same for monasca-common.
COMMON_TMP_DIR=$(mktemp -d)
(
cd "$COMMON_TMP_DIR"
# This many steps are needed to support gerrit patch sets.
git init
git remote add origin "$COMMON_REPO"
git fetch origin "$COMMON_VERSION"
git reset --hard FETCH_HEAD
)
COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse HEAD)
[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
rm -rf "$COMMON_TMP_DIR"
CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
docker build --no-cache \
--build-arg CREATION_TIME="$CREATION_TIME" \
--build-arg GITHUB_REPO="$GITHUB_REPO" \
--build-arg APP_REPO="$APP_REPO" \
--build-arg REPO_VERSION="$REPO_VERSION" \
--build-arg GIT_COMMIT="$GIT_COMMIT" \
--build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \
--build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \
--build-arg COMMON_REPO="$COMMON_REPO" \
--build-arg COMMON_VERSION="$COMMON_VERSION" \
--build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" .

45
docker/health_check.py Executable file
View File

@ -0,0 +1,45 @@
#!/usr/bin/env python
# coding=utf-8
# (C) Copyright 2018 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Health check will returns 0 when service is working properly."""
import logging
from urllib import request
import os
import sys
LOG_LEVEL = logging.getLevelName(os.environ.get('LOG_LEVEL', 'INFO'))
logging.basicConfig(level=LOG_LEVEL)
logger = logging.getLogger(__name__)
API_PORT = os.environ.get('MONASCA_CONTAINER_LOG_API_PORT', '5607')
url = "http://localhost:" + API_PORT + "/healthcheck"
def main():
"""Send health check request to health check endpoint of log API."""
logger.debug('Send health check request to %s', url)
try:
request.urlopen(url=url)
except Exception as ex:
logger.error('Exception during request handling: ' + repr(ex))
sys.exit(1)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,13 @@
bind = '0.0.0.0:{{ MONASCA_CONTAINER_LOG_API_PORT }}'
proc_name = 'monasca-log-api'
backlog = {{ GUNICORN_BACKLOG | int }}
workers = {{ GUNICORN_WORKERS | int }}
worker_class = '{{ GUNICORN_WORKER_CLASS }}'
timeout = {{ GUNICORN_TIMEOUT | int }}
{% if ADD_ACCESS_LOG == true %}
accesslog = '-'
access_log_format = '{{ ACCESS_LOG_FIELDS }}'
{% endif %}
capture_output = True

View File

@ -0,0 +1,47 @@
[default]
disable_existing_loggers = 0
[loggers]
keys = root, gunicorn_access, kafka
[handlers]
keys = console, gunicorn_access
[formatters]
keys = context, gunicorn_access
[logger_root]
level = {{ LOG_LEVEL_ROOT }}
handlers = console
[logger_gunicorn_access]
level = {{ LOG_LEVEL_ACCESS }}
handlers = console
propagate = 0
qualname = gunicorn.access
[logger_kafka]
qualname = kafka
level = DEBUG
handlers = console
propagate = 0
[handler_console]
class = logging.StreamHandler
args = (sys.stdout,)
level = {{ LOG_LEVEL_CONSOLE }}
formatter = context
[handler_gunicorn_access]
class = logging.StreamHandler
args = (sys.stdout,)
level = {{ LOG_LEVEL_ACCESS }}
formatter = gunicorn_access
[formatter_context]
class = oslo_log.formatters.ContextFormatter
[formatter_gunicorn_access]
class = logging.Formatter
format = {{ ACCESS_LOG_FORMAT }}
datefmt = %Y-%m-%d %H:%M:%S

View File

@ -0,0 +1,53 @@
[DEFAULT]
name = monasca_log_api
[composite:main]
use = egg:Paste#urlmap
/: la_version
/healthcheck: la_healthcheck
/v2.0: la_api_v2
/v3.0: la_api_v3
[pipeline:la_version]
pipeline = error_trap versionapp
[pipeline:la_healthcheck]
pipeline = error_trap healthcheckapp
[pipeline:la_api_v2]
pipeline = error_trap request_id auth roles api_v2_app
[pipeline:la_api_v3]
pipeline = error_trap request_id auth roles api_v3_app
[app:versionapp]
paste.app_factory = monasca_log_api.app.api:create_version_app
[app:healthcheckapp]
paste.app_factory = monasca_log_api.app.api:create_healthcheck_app
[app:api_v2_app]
paste.app_factory = monasca_log_api.app.api:create_api_app
set api_version=v2.0
[app:api_v3_app]
paste.app_factory = monasca_log_api.app.api:create_api_app
set api_version=v3.0
[filter:auth]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
[filter:roles]
paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory
[filter:request_id]
paste.filter_factory = oslo_middleware.request_id:RequestId.factory
[filter:debug]
paste.filter_factory = oslo_middleware.debug:Debug.factory
[filter:error_trap]
paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory
[server:main]
use = egg:gunicorn#main

42
docker/log-api.conf.j2 Normal file
View File

@ -0,0 +1,42 @@
[DEFAULT]
log_config_append=/etc/monasca/log-api-logging.conf
[monitoring]
enable = {{ MONITORING_ENABLE | default(False) }}
statsd_host = {{ STATSD_HOST | default('127.0.0.1') }}
statsd_port = {{ STATSD_PORT | default(8125) }}
statsd_buffer = {{ STATSD_BUFFER | default(50) }}
[service]
region = useast
max_log_size = 1048576
[roles_middleware]
path = /v2.0/log,/v3.0/logs
default_roles = {{ AUTHORIZED_ROLES | default('admin, domainuser, domainadmin, monasca-user') }}
agent_roles = {{ AGENT_AUTHORIZED_ROLES | default('monasca-agent') }}
[log_publisher]
topics = log
kafka_url = {{ KAFKA_URI | default('kafka:9092') }}
max_message_size = 1048576
[kafka_healthcheck]
kafka_url = {{ KAFKA_URI | default('kafka:9092') }}
kafka_topics = log
[keystone_authtoken]
auth_type = password
auth_url = {{ KEYSTONE_IDENTITY_URI }}
auth_uri = {{ KEYSTONE_AUTH_URI }}
username = {{ KEYSTONE_ADMIN_USER }}
password = {{ KEYSTONE_ADMIN_PASSWORD }}
user_domain_name = Default
project_name = {{ KEYSTONE_ADMIN_TENANT }}
project_domain_name = Default
service_token_roles_required = true
memcached_servers = {{ MEMCACHED_URI }}
insecure = false
cafile =
certfile =
keyfile =

46
docker/start.sh Normal file
View File

@ -0,0 +1,46 @@
#!/bin/sh
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Starting script.
# All checks and configuration templating you need to do before service
# could be safely started should be added in this file.
set -eo pipefail # Exit the script if any statement returns error.
# Test services we need before starting our service.
echo "Start script: waiting for needed services"
python3 /kafka_wait_for_topics.py
# Template all config files before start, it will use env variables.
# Read usage examples: https://pypi.org/project/Templer/
echo "Start script: creating config files from templates"
templer -v -f /etc/monasca/log-api.conf.j2 /etc/monasca/log-api.conf
templer -v -f /etc/monasca/log-api-gunicorn.conf.j2 /etc/monasca/log-api-gunicorn.conf
templer -v -f /etc/monasca/log-api-logging.conf.j2 /etc/monasca/log-api-logging.conf
templer -v -f /etc/monasca/log-api-paste.ini.j2 /etc/monasca/log-api-paste.ini
# Start our service.
# gunicorn --args
echo "Start script: starting container"
gunicorn \
--config /etc/monasca/log-api-gunicorn.conf \
--paste /etc/monasca/log-api-paste.ini
# Allow server to stay alive in case of failure for 2 hours for debugging.
RESULT=$?
if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then
echo "Service died, waiting 120 min before exiting"
sleep 7200
fi
exit $RESULT