From 893e66360caf3bcf0578d4541b3c17d089c33b02 Mon Sep 17 00:00:00 2001 From: Dean Troyer Date: Fri, 13 Sep 2013 15:05:51 -0500 Subject: [PATCH] Add meta-config via local.conf This defines a new local.conf file that is designed to take the place of all of the 'pass-through'[1] configuration options that have been defined in DevStack. * new local.conf file can contain multiple config file settings to be merged in to existing project config files * localrc can be embedded into local.conf and will auto-extract if localrc does not exist * Adds functions get_meta_section(), get_meta_section_files(), merge_config_file() and merge_config_group() * Adds EXTRA_OPTS, EXTRA_BAREMETAL_OPTS, Q_DHCP_EXTRA_DEFAULT_OPTS and Q_SRV_EXTRA_DEFAULT_OPTS to the deprecated warning list at the end of stack.sh [1] Pass-through options are those that do not configure or change DevStack's behaviour but simply set a value in a project config file. This includes most of the EXTRA_XXX_OPTS configuration variables. Change-Id: I367cadc86116621e9574ac203aafdab483d810d3 --- README.md | 39 ++++++++++ functions | 16 ++++ lib/config | 130 +++++++++++++++++++++++++++++++ stack.sh | 102 ++++++++++++++++++++++++ stackrc | 6 +- tests/test_config.sh | 179 +++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 471 insertions(+), 1 deletion(-) create mode 100644 lib/config create mode 100755 tests/test_config.sh diff --git a/README.md b/README.md index 99e983887e..329b94d96f 100644 --- a/README.md +++ b/README.md @@ -244,3 +244,42 @@ To setup a cells environment add the following to your `localrc`: enable_service n-cell Be aware that there are some features currently missing in cells, one notable one being security groups. The exercises have been patched to disable functionality not supported by cells. + + +# Local Configuration + +Historically DevStack has used ``localrc`` to contain all local configuration and customizations. More and more of the configuration variables available for DevStack are passed-through to the individual project configuration files. The old mechanism for this required specific code for each file and did not scale well. This is handled now by a master local configuration file. + +# local.conf + +The new config file ``local.conf`` is an extended-INI format that introduces a new meta-section header that provides some additional information such as a phase name and destination config filename: + + [[ | ]] + +where is one of a set of phase names defined by ``stack.sh`` and is the project config filename. The filename is eval'ed in the stack.sh context so all environment variables are available and may be used. Using the project config file variables in the header is strongly suggested (see example of NOVA_CONF below). If the path of the config file does not exist it is skipped. + +The defined phases are: + +* local - extracts ``localrc`` from ``local.conf`` before ``stackrc`` is sourced +* post-config - runs after the layer 2 services are configured and before they are started +* extra - runs after services are started and before any files in ``extra.d`` are executes + +The file is processed strictly in sequence; meta-sections may be specified more than once but if any settings are duplicated the last to appear in the file will be used. + + [[post-config|$NOVA_CONF]] + [DEFAULT] + use_syslog = True + + [osapi_v3] + enabled = False + +A specific meta-section ``local:localrc`` is used to provide a default localrc file. This allows all custom settings for DevStack to be contained in a single file. ``localrc`` is not overwritten if it exists to preserve compatability. + + [[local|localrc]] + FIXED_RANGE=10.254.1.0/24 + ADMIN_PASSWORD=speciale + LOGFILE=$DEST/logs/stack.sh.log + +Note that ``Q_PLUGIN_CONF_FILE`` is unique in that it is assumed to _NOT_ start with a ``/`` (slash) character. A slash will need to be added: + + [[post-config|/$Q_PLUGIN_CONF_FILE]] diff --git a/functions b/functions index f996ba89ab..87586eb17c 100644 --- a/functions +++ b/functions @@ -155,6 +155,22 @@ function err_if_not_set() { } +# Prints line number and "message" in warning format +# warn $LINENO "message" +function warn() { + local exitcode=$? + errXTRACE=$(set +o | grep xtrace) + set +o xtrace + local msg="[WARNING] ${BASH_SOURCE[2]}:$1 $2" + echo $msg 1>&2; + if [[ -n ${SCREEN_LOGDIR} ]]; then + echo $msg >> "${SCREEN_LOGDIR}/error.log" + fi + $errXTRACE + return $exitcode +} + + # HTTP and HTTPS proxy servers are supported via the usual environment variables [1] # ``http_proxy``, ``https_proxy`` and ``no_proxy``. They can be set in # ``localrc`` or on the command line if necessary:: diff --git a/lib/config b/lib/config new file mode 100644 index 0000000000..6f686e9b5d --- /dev/null +++ b/lib/config @@ -0,0 +1,130 @@ +# lib/config - Configuration file manipulation functions + +# These functions have no external dependencies and the following side-effects: +# +# CONFIG_AWK_CMD is defined, default is ``awk`` + +# Meta-config files contain multiple INI-style configuration files +# using a specific new section header to delimit them: +# +# [[group-name|file-name]] +# +# group-name refers to the group of configuration file changes to be processed +# at a particular time. These are called phases in ``stack.sh`` but +# group here as these functions are not DevStack-specific. +# +# file-name is the destination of the config file + +# Save trace setting +C_XTRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Allow the awk command to be overridden on legacy platforms +CONFIG_AWK_CMD=${CONFIG_AWK_CMD:-awk} + +# Get the section for the specific group and config file +# get_meta_section infile group configfile +function get_meta_section() { + local file=$1 + local matchgroup=$2 + local configfile=$3 + + [[ -r $file ]] || return 0 + [[ -z $configfile ]] && return 0 + + $CONFIG_AWK_CMD -v matchgroup=$matchgroup -v configfile=$configfile ' + BEGIN { group = "" } + /^\[\[.+|.*\]\]/ { + if (group == "") { + gsub("[][]", "", $1); + split($1, a, "|"); + if (a[1] == matchgroup && a[2] == configfile) { + group=a[1] + } + } else { + group="" + } + next + } + { + if (group != "") + print $0 + } + ' $file +} + + +# Get a list of config files for a specific group +# get_meta_section_files infile group +function get_meta_section_files() { + local file=$1 + local matchgroup=$2 + + [[ -r $file ]] || return 0 + + $CONFIG_AWK_CMD -v matchgroup=$matchgroup ' + /^\[\[.+\|.*\]\]/ { + gsub("[][]", "", $1); + split($1, a, "|"); + if (a[1] == matchgroup) + print a[2] + } + ' $file +} + + +# Merge the contents of a meta-config file into its destination config file +# If configfile does not exist it will be created. +# merge_config_file infile group configfile +function merge_config_file() { + local file=$1 + local matchgroup=$2 + local configfile=$3 + + [[ -r $configfile ]] || touch $configfile + + get_meta_section $file $matchgroup $configfile | \ + $CONFIG_AWK_CMD -v configfile=$configfile ' + BEGIN { section = "" } + /^\[.+\]/ { + gsub("[][]", "", $1); + section=$1 + next + } + /^ *\#/ { + next + } + /^.+/ { + split($0, d, " *= *") + print "iniset " configfile " " section " " d[1] " \"" d[2] "\"" + } + ' | while read a; do eval "$a"; done + +} + + +# Merge all of the files specified by group +# merge_config_group infile group [group ...] +function merge_config_group() { + local localfile=$1; shift + local matchgroups=$@ + + [[ -r $localfile ]] || return 0 + + for group in $matchgroups; do + for configfile in $(get_meta_section_files $localfile $group); do + if [[ -d $(dirname $configfile) ]]; then + merge_config_file $localfile $group $configfile + fi + done + done +} + + +# Restore xtrace +$C_XTRACE + +# Local variables: +# mode: shell-script +# End: diff --git a/stack.sh b/stack.sh index be04bedade..f6ab4c49f6 100755 --- a/stack.sh +++ b/stack.sh @@ -29,6 +29,9 @@ TOP_DIR=$(cd $(dirname "$0") && pwd) # Import common functions source $TOP_DIR/functions +# Import config functions +source $TOP_DIR/lib/config + # Determine what system we are running on. This provides ``os_VENDOR``, # ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME`` # and ``DISTRO`` @@ -38,6 +41,25 @@ GetDistro # Global Settings # =============== +# Check for a ``localrc`` section embedded in ``local.conf`` and extract if +# ``localrc`` does not already exist + +# Phase: local +rm -f $TOP_DIR/.localrc.auto +if [[ -r $TOP_DIR/local.conf ]]; then + LRC=$(get_meta_section_files $TOP_DIR/local.conf local) + for lfile in $LRC; do + if [[ "$lfile" == "localrc" ]]; then + if [[ -r $TOP_DIR/localrc ]]; then + warn $LINENO "localrc and local.conf:[[local]] both exist, using localrc" + else + echo "# Generated file, do not exit" >$TOP_DIR/.localrc.auto + get_meta_section $TOP_DIR/local.conf local $lfile >>$TOP_DIR/.localrc.auto + fi + fi + done +fi + # ``stack.sh`` is customizable by setting environment variables. Override a # default setting via export:: # @@ -842,6 +864,9 @@ if is_service_enabled sysstat;then fi +# Start Services +# ============== + # Keystone # -------- @@ -1153,6 +1178,14 @@ if is_service_enabled nova && is_baremetal; then fi +# Local Configuration +# =================== + +# Apply configuration from local.conf if it exists for layer 2 services +# Phase: post-config +merge_config_group $TOP_DIR/local.conf post-config + + # Launch Services # =============== @@ -1348,6 +1381,14 @@ for i in BASE_SQL_CONN ENABLED_SERVICES HOST_IP LOGFILE \ done +# Local Configuration +# =================== + +# Apply configuration from local.conf if it exists for layer 2 services +# Phase: extra +merge_config_group $TOP_DIR/local.conf extra + + # Run extras # ========== @@ -1420,5 +1461,66 @@ if [[ -n "$DEPRECATED_TEXT" ]]; then echo_summary "WARNING: $DEPRECATED_TEXT" fi +# Specific warning for deprecated configs +if [[ -n "$EXTRA_OPTS" ]]; then + echo "" + echo_summary "WARNING: EXTRA_OPTS is used" + echo "You are using EXTRA_OPTS to pass configuration into nova.conf." + echo "Please convert that configuration in localrc to a nova.conf section in local.conf:" + echo " +[[post-config|\$NOVA_CONF]] +[DEFAULT] +" + for I in "${EXTRA_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$EXTRA_BAREMETAL_OPTS" ]]; then + echo "" + echo_summary "WARNING: EXTRA_OPTS is used" + echo "You are using EXTRA_OPTS to pass configuration into nova.conf." + echo "Please convert that configuration in localrc to a nova.conf section in local.conf:" + echo " +[[post-config|\$NOVA_CONF]] +[baremetal] +" + for I in "${EXTRA_BAREMETAL_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$Q_DHCP_EXTRA_DEFAULT_OPTS" ]]; then + echo "" + echo_summary "WARNING: Q_DHCP_EXTRA_DEFAULT_OPTS is used" + echo "You are using Q_DHCP_EXTRA_DEFAULT_OPTS to pass configuration into $Q_DHCP_CONF_FILE." + echo "Please convert that configuration in localrc to a $Q_DHCP_CONF_FILE section in local.conf:" + echo " +[[post-config|\$Q_DHCP_CONF_FILE]] +[DEFAULT] +" + for I in "${Q_DHCP_EXTRA_DEFAULT_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + +if [[ -n "$Q_SRV_EXTRA_DEFAULT_OPTS" ]]; then + echo "" + echo_summary "WARNING: Q_SRV_EXTRA_DEFAULT_OPTS is used" + echo "You are using Q_SRV_EXTRA_DEFAULT_OPTS to pass configuration into $NEUTRON_CONF." + echo "Please convert that configuration in localrc to a $NEUTRON_CONF section in local.conf:" + echo " +[[post-config|\$NEUTRON_CONF]] +[DEFAULT] +" + for I in "${Q_SRV_EXTRA_DEFAULT_OPTS[@]}"; do + # Replace the first '=' with ' ' for iniset syntax + echo ${I} + done +fi + # Indicate how long this took to run (bash maintained variable ``SECONDS``) echo_summary "stack.sh completed in $SECONDS seconds." diff --git a/stackrc b/stackrc index 3a338d16f2..e4a96160d1 100644 --- a/stackrc +++ b/stackrc @@ -48,8 +48,12 @@ IDENTITY_API_VERSION=2.0 USE_SCREEN=True # allow local overrides of env variables, including repo config -if [ -f $RC_DIR/localrc ]; then +if [[ -f $RC_DIR/localrc ]]; then + # Old-style user-supplied config source $RC_DIR/localrc +elif [[ -f $RC_DIR/.localrc.auto ]]; then + # New-style user-supplied config extracted from local.conf + source $RC_DIR/.localrc.auto fi diff --git a/tests/test_config.sh b/tests/test_config.sh new file mode 100755 index 0000000000..fed2e7d477 --- /dev/null +++ b/tests/test_config.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash + +# Tests for DevStack meta-config functions + +TOP=$(cd $(dirname "$0")/.. && pwd) + +# Import common functions +source $TOP/functions + +# Import config functions +source $TOP/lib/config + +# check_result() tests and reports the result values +# check_result "actual" "expected" +function check_result() { + local actual=$1 + local expected=$2 + if [[ "$actual" == "$expected" ]]; then + echo "OK" + else + echo -e "failed: $actual != $expected\n" + fi +} + +TEST_1C_ADD="[eee] +type=new +multi = foo2" + +function create_test1c() { + cat >test1c.conf <test2a.conf <test.conf <