diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..06226b1e --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,47 @@ +ARG DOCKER_IMAGE=monasca-persister +ARG APP_REPO=https://git.openstack.org/openstack/monasca-persister + +# Branch, tag or git hash to build from. +ARG REPO_VERSION=master +ARG CONSTRAINTS_BRANCH=master + +# Extra Python3 dependencies. +ARG EXTRA_DEPS="influxdb" + +# Always start from `monasca-base` image and use specific tag of it. +ARG BASE_TAG=master +FROM monasca/base:$BASE_TAG + +# Environment variables used for our service or wait scripts. +ENV \ + DEBUG=false \ + VERBOSE=true \ + ZOOKEEPER_URI=zookeeper:2181 \ + KAFKA_URI=kafka:9092 \ + KAFKA_ALARM_HISTORY_BATCH_SIZE=1000 \ + KAFKA_ALARM_HISTORY_WAIT_TIME=15 \ + KAFKA_METRICS_BATCH_SIZE=1000 \ + KAFKA_METRICS_WAIT_TIME=15 \ + KAFKA_WAIT_FOR_TOPICS=alarm-state-transitions,metrics \ + INFLUX_HOST=influxdb \ + INFLUX_PORT=8086 \ + INFLUX_USER=mon_persister \ + INFLUX_PASSWORD=password \ + INFLUX_DB=mon \ + STAY_ALIVE_ON_FAILURE="false" + +# Copy all necessary files to proper locations. +COPY monasca_persister.conf.j2 /etc/monasca/ + +# Run here all additional steps your service need post installation. +# Stay with only one `RUN` and use `&& \` for next steps to don't create +# unnecessary image layers. Clean at the end to conserve space. +#RUN \ +# echo "Some steps to do after main installation." && \ +# echo "Hello when building." + +# Expose port for specific service. +#EXPOSE 1234 + +# Implement start script in `start.sh` file. +CMD ["/start.sh"] diff --git a/docker/README.rst b/docker/README.rst new file mode 100644 index 00000000..846c9f79 --- /dev/null +++ b/docker/README.rst @@ -0,0 +1,76 @@ +================================== +Docker image for Monasca persister +================================== +The Monasca persister image is based on the monasca-base image. + + +Building monasca-base image +=========================== +See https://github.com/openstack/monasca-common/tree/master/docker/README.rst + + +Building Monasca persister image (child) +======================================== + + +Requirements from monasca-base image +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +health_check.py + This file will be used for checking the status of the Monasca persister + application. + + +Scripts for child image +~~~~~~~~~~~~~~~~~~~~~~~ +start.sh + In this starting script provide all steps that lead to the proper service + start. Including usage of wait scripts and templating of configuration + files. You also could provide the ability to allow running container after + service died for easier debugging. + +build_image.sh + Please read detailed build description inside the script. + + +Build arguments (child) +~~~~~~~~~~~~~~~~~~~~~~~ +====================== ========================= +Arguments Occurrence +====================== ========================= +BASE_TAG Dockerfile +====================== ========================= + + +Environment variables (child) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +============================== =============================== ================================================ +Variable Default Description +============================== =============================== ================================================ +DEBUG false If true, enable debug logging +VERBOSE true If true, enable info logging +ZOOKEEPER_URI zookeeper:2181 The host and port for zookeeper +KAFKA_URI kafka:9092 The host and port for kafka +KAFKA_WAIT_FOR_TOPICS alarm-state-transitions,metrics Topics to wait on at startup +KAFKA_WAIT_RETRIES 24 Number of kafka connect attempts +KAFKA_WAIT_DELAY 5 Seconds to wait between attempts +KAFKA_ALARM_HISTORY_BATCH_SIZE 1000 Kafka consumer takes messages in a batch +KAFKA_ALARM_HISTORY_WAIT_TIME 15 Seconds to wait if the batch size is not reached +KAFKA_METRICS_BATCH_SIZE 1000 Kafka consumer takes messages in a batch +KAFKA_METRICS_WAIT_TIME 15 Seconds to wait if the batch size is not reached +INFLUX_HOST influxdb The host for influxdb +INFLUX_PORT 8086 The port for influxdb +INFLUX_USER mon_persister The influx username +INFLUX_PASSWORD password The influx password +INFLUX_DB mon The influx database name +STAY_ALIVE_ON_FAILURE false If true, container runs 2 hours even start fails +============================== =============================== ================================================ + + +Provide Configuration templates +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +* persister.conf.j2 + + +Links +~~~~~ +https://github.com/openstack/monasca-persister/tree/master/monasca_persister diff --git a/docker/build_image.sh b/docker/build_image.sh new file mode 100755 index 00000000..b601dfa7 --- /dev/null +++ b/docker/build_image.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# TODO(Dobroslaw): move this script to monasca-common/docker folder +# and leave here small script to download it and execute using env variables +# to minimize code duplication. + +set -x # Print each script step. +set -eo pipefail # Exit the script if any statement returns error. + +# This script is used for building Docker image with proper labels. +# +# Example usage: +# $ ./build_image.sh +# +# To build from master branch (default): +# $ ./build_image.sh +# To build specific version run this script in the following way: +# $ ./build_image.sh stable/queens +# Building from specific commit: +# $ ./build_image.sh cb7f226 +# When building from a tag monasca-common will be used in version available +# in upper constraint file: +# $ ./build_image.sh 2.5.0 +# To build image from Gerrit patch sets that is targeting branch stable/queens: +# $ ./build_image.sh refs/changes/51/558751/1 stable/queens + +[ -z "$DOCKER_IMAGE" ] && \ + DOCKER_IMAGE=$(\grep DOCKER_IMAGE Dockerfile | cut -f2 -d"=") + +: "${REPO_VERSION:=$1}" +[ -z "$REPO_VERSION" ] && \ + REPO_VERSION=$(\grep REPO_VERSION Dockerfile | cut -f2 -d"=") +# Let's stick to more readable version and disable SC2001 here. +# shellcheck disable=SC2001 +REPO_VERSION_CLEAN=$(echo "$REPO_VERSION" | sed 's|/|-|g') + +[ -z "$APP_REPO" ] && APP_REPO=$(\grep APP_REPO Dockerfile | cut -f2 -d"=") +GITHUB_REPO=$(echo "$APP_REPO" | sed 's/git.openstack.org/github.com/' | \ + sed 's/ssh:/https:/') + +if [ -z "$CONSTRAINTS_FILE" ]; then + CONSTRAINTS_FILE=$(\grep CONSTRAINTS_FILE Dockerfile | cut -f2 -d"=") || true + : "${CONSTRAINTS_FILE:=http://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt}" +fi + +: "${CONSTRAINTS_BRANCH:=$2}" +[ -z "$CONSTRAINTS_BRANCH" ] && \ + CONSTRAINTS_BRANCH=$(\grep CONSTRAINTS_BRANCH Dockerfile | cut -f2 -d"=") + +# When using stable version of repository use same stable constraints file. +case "$REPO_VERSION" in + *stable*) + CONSTRAINTS_BRANCH_CLEAN="$REPO_VERSION" + ;; + *) + CONSTRAINTS_BRANCH_CLEAN="$CONSTRAINTS_BRANCH" + ;; +esac + +# Monasca-common variables. +if [ -z "$COMMON_REPO" ]; then + COMMON_REPO=$(\grep COMMON_REPO Dockerfile | cut -f2 -d"=") || true + : "${COMMON_REPO:=https://git.openstack.org/openstack/monasca-common}" +fi +if [ -z "$COMMON_VERSION" ]; then + COMMON_VERSION=$(\grep COMMON_VERSION Dockerfile | cut -f2 -d"=") || true + : "${COMMON_VERSION:=master}" +fi + +# Clone project to temporary directory for getting proper commit number from +# branches and tags. We need this for setting proper image labels. +# Docker does not allow to get any data from inside of system when building +# image. +TMP_DIR=$(mktemp -d) +( + cd "$TMP_DIR" + # This many steps are needed to support gerrit patch sets. + git init + git remote add origin "$APP_REPO" + git fetch origin "$REPO_VERSION" + git reset --hard FETCH_HEAD +) +GIT_COMMIT=$(git -C "$TMP_DIR" rev-parse FETCH_HEAD) +[ -z "${GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 +rm -rf "$TMP_DIR" + +# Do the same for monasca-common. +COMMON_TMP_DIR=$(mktemp -d) +( + cd "$COMMON_TMP_DIR" + # This many steps are needed to support gerrit patch sets. + git init + git remote add origin "$COMMON_REPO" + git fetch origin "$COMMON_VERSION" + git reset --hard FETCH_HEAD +) +COMMON_GIT_COMMIT=$(git -C "$COMMON_TMP_DIR" rev-parse FETCH_HEAD) +[ -z "${COMMON_GIT_COMMIT}" ] && echo "No git commit hash found" && exit 1 +rm -rf "$COMMON_TMP_DIR" + +CREATION_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ") + +docker build --no-cache \ + --build-arg CREATION_TIME="$CREATION_TIME" \ + --build-arg GITHUB_REPO="$GITHUB_REPO" \ + --build-arg APP_REPO="$APP_REPO" \ + --build-arg REPO_VERSION="$REPO_VERSION" \ + --build-arg GIT_COMMIT="$GIT_COMMIT" \ + --build-arg CONSTRAINTS_FILE="$CONSTRAINTS_FILE" \ + --build-arg CONSTRAINTS_BRANCH="$CONSTRAINTS_BRANCH_CLEAN" \ + --build-arg COMMON_REPO="$COMMON_REPO" \ + --build-arg COMMON_VERSION="$COMMON_VERSION" \ + --build-arg COMMON_GIT_COMMIT="$COMMON_GIT_COMMIT" \ + --tag "$DOCKER_IMAGE":"$REPO_VERSION_CLEAN" . diff --git a/docker/health_check.py b/docker/health_check.py new file mode 100644 index 00000000..7d38b641 --- /dev/null +++ b/docker/health_check.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python +# coding=utf-8 + +# (C) Copyright 2018 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Health check will returns 0 when service is working properly.""" + +def main(): + """health check for Monasca-persister""" + #TODO wait for health check endpoint ... + return 0 + +if __name__ == '__main__': + main() diff --git a/docker/monasca_persister.conf.j2 b/docker/monasca_persister.conf.j2 new file mode 100644 index 00000000..bc9f4f64 --- /dev/null +++ b/docker/monasca_persister.conf.j2 @@ -0,0 +1,64 @@ +[DEFAULT] +#log_file = monasca_persister.log +#log_dir = /var/log/monasca_persister + +# Default log level is WARNING +# Show debugging output in logs (sets DEBUG log level output) +debug = {{ DEBUG }} +# Show more verbose log output (sets INFO log level output) if debug is False +verbose = {{ VERBOSE }} + +[repositories] +# The driver to use for the metrics repository +metrics_driver = monasca_persister.repositories.influxdb.metrics_repository:MetricInfluxdbRepository + +# The driver to use for the alarm state history repository +alarm_state_history_driver = monasca_persister.repositories.influxdb.alarm_state_history_repository:AlarmStateHistInfluxdbRepository + +[zookeeper] +# Comma separated list of host:port +uri = {{ ZOOKEEPER_URI }} +partition_interval_recheck_seconds = 15 + +[kafka_alarm_history] +# Comma separated list of Kafka broker host:port. +uri = {{ KAFKA_URI }} +group_id = 1_alarm-state-transitions +topic = alarm-state-transitions +consumer_id = 1 +client_id = 1 +database_batch_size = {{ KAFKA_ALARM_HISTORY_BATCH_SIZE }} +max_wait_time_seconds = {{ KAFKA_ALARM_HISTORY_WAIT_TIME }} +# The following 3 values are set to the kakfa-python defaults +fetch_size_bytes = 4096 +buffer_size = 4096 +# 8 times buffer size +max_buffer_size = 32768 +# Path in zookeeper for kafka consumer group partitioning algo +zookeeper_path = /persister_partitions/alarm-state-transitions +num_processors = 1 + +[kafka_metrics] +# Comma separated list of Kafka broker host:port +uri = {{ KAFKA_URI }} +group_id = 1_metrics +topic = metrics +consumer_id = 1 +client_id = 1 +database_batch_size = {{ KAFKA_METRICS_BATCH_SIZE }} +max_wait_time_seconds = {{ KAFKA_METRICS_WAIT_TIME }} +# The following 3 values are set to the kakfa-python defaults +fetch_size_bytes = 4096 +buffer_size = 4096 +# 8 times buffer size +max_buffer_size = 32768 +# Path in zookeeper for kafka consumer group partitioning algo +zookeeper_path = /persister_partitions/metrics +num_processors = 1 + +[influxdb] +database_name = {{ INFLUX_DB }} +ip_address = {{ INFLUX_HOST }} +port = {{ INFLUX_PORT }} +user = {{ INFLUX_USER }} +password = {{ INFLUX_PASSWORD }} diff --git a/docker/start.sh b/docker/start.sh new file mode 100644 index 00000000..a43fc3b4 --- /dev/null +++ b/docker/start.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Starting script. +# All checks and configuration templating you need to do before service +# could be safely started should be added in this file. + +set -eo pipefail # Exit the script if any statement returns error. + +# Test services we need before starting our service. +echo "Start script: waiting for needed services" +python3 /kafka_wait_for_topics.py + +# Template all config files before start, it will use env variables. +# Read usage examples: https://pypi.org/project/Templer/ +echo "Start script: creating config files from templates" +templer -v -f /etc/monasca/monasca_persister.conf.j2 /etc/monasca/monasca_persister.conf + +# Start our service. +# gunicorn --args +echo "Start script: starting container" +monasca-persister --config-file /etc/monasca/monasca_persister.conf + +# Allow server to stay alive in case of failure for 2 hours for debugging. +RESULT=$? +if [ $RESULT != 0 ] && [ "$STAY_ALIVE_ON_FAILURE" = "true" ]; then + echo "Service died, waiting 120 min before exiting" + sleep 7200 +fi +exit $RESULT