Merge "Add tooling for building Docker image"
This commit is contained in:
Zuul
Gerrit Code Review
commit
256d192f85
|
|
@ -0,0 +1,47 @@
|
|||
ARG DOCKER_IMAGE=monasca-persister
|
||||
ARG APP_REPO=https://git.openstack.org/openstack/monasca-persister
|
||||
|
||||
# Branch, tag or git hash to build from.
|
||||
ARG REPO_VERSION=master
|
||||
ARG CONSTRAINTS_BRANCH=master
|
||||
|
||||
# Extra Python3 dependencies.
|
||||
ARG EXTRA_DEPS="influxdb"
|
||||
|
||||
# Always start from `monasca-base` image and use specific tag of it.
|
||||
ARG BASE_TAG=master
|
||||
FROM monasca/base:$BASE_TAG
|
||||
|
||||
# Environment variables used for our service or wait scripts.
|
||||
ENV \
|
||||
DEBUG=false \
|
||||
VERBOSE=true \
|
||||
ZOOKEEPER_URI=zookeeper:2181 \
|
||||
KAFKA_URI=kafka:9092 \
|
||||
KAFKA_ALARM_HISTORY_BATCH_SIZE=1000 \
|
||||
KAFKA_ALARM_HISTORY_WAIT_TIME=15 \
|
||||
KAFKA_METRICS_BATCH_SIZE=1000 \
|
||||
KAFKA_METRICS_WAIT_TIME=15 \
|
||||
KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \
|
||||
INFLUX_HOST=influxdb \
|
||||
INFLUX_PORT=8086 \
|
||||
INFLUX_USER=mon_persister \
|
||||
INFLUX_PASSWORD=password \
|
||||
INFLUX_DB=mon \
|
||||
STAY_ALIVE_ON_FAILURE="false"
|
||||
|
||||
# Copy all necessary files to proper locations.
|
||||
COPY monasca_persister.conf.j2 /etc/monasca/
|
||||
|
||||
# Run here all additional steps your service need post installation.
|
||||
# Stay with only one `RUN` and use `&& \` for next steps to don't create
|
||||
# unnecessary image layers. Clean at the end to conserve space.
|
||||
#RUN \
|
||||
# echo "Some steps to do after main installation." && \
|
||||
# echo "Hello when building."
|
||||
|
||||
# Expose port for specific service.
|
||||
#EXPOSE 1234
|
||||
|
||||
# Implement start script in `start.sh` file.
|
||||
CMD ["/start.sh"]
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
==================================
|
||||
Docker image for Monasca persister
|
||||
==================================
|
||||
The Monasca persister image is based on the monasca-base image.
|
||||
|
||||
|
||||
Building monasca-base image
|
||||
===========================
|
||||
See https://github.com/openstack/monasca-common/tree/master/docker/README.rst
|
||||
|
||||
|
||||
Building Monasca persister image (child)
|
||||
========================================
|
||||
|
||||
|
||||
Requirements from monasca-base image
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
health_check.py
|
||||
This file will be used for checking the status of the Monasca persister
|
||||
application.
|
||||
|
||||
|
||||
Scripts for child image
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
start.sh
|
||||
In this starting script provide all steps that lead to the proper service
|
||||
start. Including usage of wait scripts and templating of configuration
|
||||
files. You also could provide the ability to allow running container after
|
||||
service died for easier debugging.
|
||||
|
||||
build_image.sh
|
||||
Please read detailed build description inside the script.
|
||||
|
||||
|
||||
Build arguments (child)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~
|
||||
====================== =========================
|
||||
Arguments Occurrence
|
||||
====================== =========================
|
||||
BASE_TAG Dockerfile
|
||||
====================== =========================
|
||||
|
||||
|
||||
Environment variables (child)
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
============================== =============================== ================================================
|
||||
Variable Default Description
|
||||
============================== =============================== ================================================
|
||||
DEBUG false If true, enable debug logging
|
||||
VERBOSE true If true, enable info logging
|
||||
ZOOKEEPER_URI zookeeper:2181 The host and port for zookeeper
|
||||
KAFKA_URI kafka:9092 The host and port for kafka
|
||||
KAFKA_WAIT_FOR_TOPICS alarm-state-transitions,metrics Topics to wait on at startup
|
||||
KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts
|
||||
KAFKA_WAIT_DELAY 5 Seconds to wait between attempts
|
||||
KAFKA_ALARM_HISTORY_BATCH_SIZE 1000 Kafka consumer takes messages in a batch
|
||||
KAFKA_ALARM_HISTORY_WAIT_TIME 15 Seconds to wait if the batch size is not reached
|
||||
KAFKA_METRICS_BATCH_SIZE 1000 Kafka consumer takes messages in a batch
|
||||
KAFKA_METRICS_WAIT_TIME 15 Seconds to wait if the batch size is not reached
|
||||
INFLUX_HOST influxdb The host for influxdb
|
||||
INFLUX_PORT 8086 The port for influxdb
|
||||
INFLUX_USER mon_persister The influx username
|
||||
INFLUX_PASSWORD password The influx password
|
||||
INFLUX_DB mon The influx database name
|
||||
STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours even start fails
|
||||
============================== =============================== ================================================
|
||||
|
||||
|
||||
Provide Configuration templates
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
* persister.conf.j2
|
||||
|
||||
|
||||
Links
|
||||
~~~~~
|
||||
https://github.com/openstack/monasca-persister/tree/master/monasca_persister
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# TODO(Dobroslaw): move this script to monasca-common/docker folder
|
||||
# and leave here small script to download it and execute using env variables
|
||||
# to minimize code duplication.
|
||||
|
||||
set -x # Print each script step.
|
||||
set -eo pipefail # Exit the script if any statement returns error.
|
||||
|
||||
# This script is used for building Docker image with proper labels.
|
||||
#
|
||||
# Example usage:
|
||||
# $ ./build_image.sh <repository_version> <upper_constains_branch>
|
||||
#
|
||||
# To build from master branch (default):
|
||||
# $ ./build_image.sh
|
||||
# To build specific version run this script in the following way:
|
||||
# $ ./build_image.sh stable/queens
|
||||
# Building from specific commit:
|
||||
# $ ./build_image.sh cb7f226
|
||||
# When building from a tag monasca-common will be used in version available
|
||||
# in upper constraint file:
|
||||
# $ ./build_image.sh 2.5.0
|
||||
# To build image from Gerrit patch sets that is targeting branch stable/queens:
|
||||
# $ ./build_image.sh refs/changes/51/558751/1 stable/queens
|
||||
|
||||
[ -z "$DOCKER_IMAGE" ] && \
|
||||
DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=")
|
||||
|
||||
: "${REPO_VERSION:=$1}"
|
||||
[ -z "$REPO_VERSION" ] && \
|
||||
REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=")
|
||||
# Let's stick to more readable version and disable SC2001 here.
|
||||
# shellcheck disable=SC2001
|
||||
REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g')
|
||||
|
||||
[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=")
|
||||
GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \
|
||||
sed 's/ssh:/https:/')
|
||||
|
||||
if [ -z "$CONSTRAINTS_FILE" ]; then
|
||||
CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true
|
||||
: "${CONSTRAINTS_FILE:=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}"
|
||||
fi
|
||||
|
||||
: "${CONSTRAINTS_BRANCH:=$2}"
|
||||
[ -z "$CONSTRAINTS_BRANCH" ] && \
|
||||
CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=")
|
||||
|
||||
# When using stable version of repository use same stable constraints file.
|
||||
case "$REPO_VERSION" in
|
||||
*stable*)
|
||||
CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION"
|
||||
;;
|
||||
*)
|
||||
CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH"
|
||||
;;
|
||||
esac
|
||||
|
||||
# Monasca-common variables.
|
||||
if [ -z "$COMMON_REPO" ]; then
|
||||
COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true
|
||||
: "${COMMON_REPO:=https://git.openstack.org/openstack/monasca-common}"
|
||||
fi
|
||||
if [ -z "$COMMON_VERSION" ]; then
|
||||
COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true
|
||||
: "${COMMON_VERSION:=master}"
|
||||
fi
|
||||
|
||||
# Clone project to temporary directory for getting proper commit number from
|
||||
# branches and tags. We need this for setting proper image labels.
|
||||
# Docker does not allow to get any data from inside of system when building
|
||||
# image.
|
||||
TMP_DIR=$(mktemp -d)
|
||||
(
|
||||
cd "$TMP_DIR"
|
||||
# This many steps are needed to support gerrit patch sets.
|
||||
git init
|
||||
git remote add origin "$APP_REPO"
|
||||
git fetch origin "$REPO_VERSION"
|
||||
git reset --hard FETCH_HEAD
|
||||
)
|
||||
GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse FETCH_HEAD)
|
||||
[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
|
||||
rm -rf "$TMP_DIR"
|
||||
|
||||
# Do the same for monasca-common.
|
||||
COMMON_TMP_DIR=$(mktemp -d)
|
||||
(
|
||||
cd "$COMMON_TMP_DIR"
|
||||
# This many steps are needed to support gerrit patch sets.
|
||||
git init
|
||||
git remote add origin "$COMMON_REPO"
|
||||
git fetch origin "$COMMON_VERSION"
|
||||
git reset --hard FETCH_HEAD
|
||||
)
|
||||
COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse FETCH_HEAD)
|
||||
[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1
|
||||
rm -rf "$COMMON_TMP_DIR"
|
||||
|
||||
CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
docker build --no-cache \
|
||||
--build-arg CREATION_TIME="$CREATION_TIME" \
|
||||
--build-arg GITHUB_REPO="$GITHUB_REPO" \
|
||||
--build-arg APP_REPO="$APP_REPO" \
|
||||
--build-arg REPO_VERSION="$REPO_VERSION" \
|
||||
--build-arg GIT_COMMIT="$GIT_COMMIT" \
|
||||
--build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \
|
||||
--build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \
|
||||
--build-arg COMMON_REPO="$COMMON_REPO" \
|
||||
--build-arg COMMON_VERSION="$COMMON_VERSION" \
|
||||
--build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \
|
||||
--tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" .
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
#!/usr/bin/env python
|
||||
# coding=utf-8
|
||||
|
||||
# (C) Copyright 2018 FUJITSU LIMITED
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""Health check will returns 0 when service is working properly."""
|
||||
|
||||
def main():
|
||||
"""health check for Monasca-persister"""
|
||||
#TODO wait for health check endpoint ...
|
||||
return 0
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
[DEFAULT]
|
||||
#log_file = monasca_persister.log
|
||||
#log_dir = /var/log/monasca_persister
|
||||
|
||||
# Default log level is WARNING
|
||||
# Show debugging output in logs (sets DEBUG log level output)
|
||||
debug = {{ DEBUG }}
|
||||
# Show more verbose log output (sets INFO log level output) if debug is False
|
||||
verbose = {{ VERBOSE }}
|
||||
|
||||
[repositories]
|
||||
# The driver to use for the metrics repository
|
||||
metrics_driver = monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository
|
||||
|
||||
# The driver to use for the alarm state history repository
|
||||
alarm_state_history_driver = monasca_persister.repositories.influxdb.alarm_state_history_repository:AlarmStateHistInfluxdbRepository
|
||||
|
||||
[zookeeper]
|
||||
# Comma separated list of host:port
|
||||
uri = {{ ZOOKEEPER_URI }}
|
||||
partition_interval_recheck_seconds = 15
|
||||
|
||||
[kafka_alarm_history]
|
||||
# Comma separated list of Kafka broker host:port.
|
||||
uri = {{ KAFKA_URI }}
|
||||
group_id = 1_alarm-state-transitions
|
||||
topic = alarm-state-transitions
|
||||
consumer_id = 1
|
||||
client_id = 1
|
||||
database_batch_size = {{ KAFKA_ALARM_HISTORY_BATCH_SIZE }}
|
||||
max_wait_time_seconds = {{ KAFKA_ALARM_HISTORY_WAIT_TIME }}
|
||||
# The following 3 values are set to the kakfa-python defaults
|
||||
fetch_size_bytes = 4096
|
||||
buffer_size = 4096
|
||||
# 8 times buffer size
|
||||
max_buffer_size = 32768
|
||||
# Path in zookeeper for kafka consumer group partitioning algo
|
||||
zookeeper_path = /persister_partitions/alarm-state-transitions
|
||||
num_processors = 1
|
||||
|
||||
[kafka_metrics]
|
||||
# Comma separated list of Kafka broker host:port
|
||||
uri = {{ KAFKA_URI }}
|
||||
group_id = 1_metrics
|
||||
topic = metrics
|
||||
consumer_id = 1
|
||||
client_id = 1
|
||||
database_batch_size = {{ KAFKA_METRICS_BATCH_SIZE }}
|
||||
max_wait_time_seconds = {{ KAFKA_METRICS_WAIT_TIME }}
|
||||
# The following 3 values are set to the kakfa-python defaults
|
||||
fetch_size_bytes = 4096
|
||||
buffer_size = 4096
|
||||
# 8 times buffer size
|
||||
max_buffer_size = 32768
|
||||
# Path in zookeeper for kafka consumer group partitioning algo
|
||||
zookeeper_path = /persister_partitions/metrics
|
||||
num_processors = 1
|
||||
|
||||
[influxdb]
|
||||
database_name = {{ INFLUX_DB }}
|
||||
ip_address = {{ INFLUX_HOST }}
|
||||
port = {{ INFLUX_PORT }}
|
||||
user = {{ INFLUX_USER }}
|
||||
password = {{ INFLUX_PASSWORD }}
|
||||
|
|
@ -0,0 +1,41 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
# Starting script.
|
||||
# All checks and configuration templating you need to do before service
|
||||
# could be safely started should be added in this file.
|
||||
|
||||
set -eo pipefail # Exit the script if any statement returns error.
|
||||
|
||||
# Test services we need before starting our service.
|
||||
echo "Start script: waiting for needed services"
|
||||
python3 /kafka_wait_for_topics.py
|
||||
|
||||
# Template all config files before start, it will use env variables.
|
||||
# Read usage examples: https://pypi.org/project/Templer/
|
||||
echo "Start script: creating config files from templates"
|
||||
templer -v -f /etc/monasca/monasca_persister.conf.j2 /etc/monasca/monasca_persister.conf
|
||||
|
||||
# Start our service.
|
||||
# gunicorn --args
|
||||
echo "Start script: starting container"
|
||||
monasca-persister --config-file /etc/monasca/monasca_persister.conf
|
||||
|
||||
# Allow server to stay alive in case of failure for 2 hours for debugging.
|
||||
RESULT=$?
|
||||
if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then
|
||||
echo "Service died, waiting 120 min before exiting"
|
||||
sleep 7200
|
||||
fi
|
||||
exit $RESULT
|
||||
Loading…
Reference in New Issue