From 95611f4adfdd73d846e9cdc040afe4ce1a7bb66a Mon Sep 17 00:00:00 2001 From: zhipengl Date: Wed, 15 Jul 2020 01:47:53 +0800 Subject: [PATCH] Add --config-file argument for docker image build script build-stx-base.sh and build-stx-images.sh need to accept a new argument (--config-file ) specifying a path to a config file. These commands automatically look for the config files at the predetermined location, and apply them if found. If not found, continue without error. If optional argument --config-file is added, These commands will use specified configfile instead of predetermined one. Test pass on local build enviroment. Closes-Bug: 1886062 Change-Id: I476e38064d1515d3d0a0d49c789f1d472d8ca1cc Signed-off-by: zhipengl (cherry picked from commit 4d6af626b1976987e614487a9c7a9293e5427113) --- .../base-image-build-centos-dev.cfg | 2 + .../base-image-build-centos-stable.cfg | 2 + .../build-docker-images/build-stx-base.sh | 63 +++++++++++- .../build-docker-images/build-stx-images.sh | 97 ++++++++++++++++--- .../docker-image-build-centos-dev.cfg | 2 + .../docker-image-build-centos-stable.cfg | 2 + .../docker-image-build.cfg | 2 - 7 files changed, 151 insertions(+), 19 deletions(-) create mode 100644 build-tools/build-docker-images/base-image-build-centos-dev.cfg create mode 100644 build-tools/build-docker-images/base-image-build-centos-stable.cfg create mode 100644 build-tools/build-docker-images/docker-image-build-centos-dev.cfg create mode 100644 build-tools/build-docker-images/docker-image-build-centos-stable.cfg delete mode 100644 build-tools/build-docker-images/docker-image-build.cfg diff --git a/build-tools/build-docker-images/base-image-build-centos-dev.cfg b/build-tools/build-docker-images/base-image-build-centos-dev.cfg new file mode 100644 index 00000000..3471b02d --- /dev/null +++ b/build-tools/build-docker-images/base-image-build-centos-dev.cfg @@ -0,0 +1,2 @@ +repo=ussuri-ceph,http://build.starlingx.cengn.ca:80/mirror/centos/download.ceph.com/rpm-mimic/el7/x86_64/ +repo=ussuri-wsgi,http://build.starlingx.cengn.ca:80/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/base-image-build-centos-stable.cfg b/build-tools/build-docker-images/base-image-build-centos-stable.cfg new file mode 100644 index 00000000..3471b02d --- /dev/null +++ b/build-tools/build-docker-images/base-image-build-centos-stable.cfg @@ -0,0 +1,2 @@ +repo=ussuri-ceph,http://build.starlingx.cengn.ca:80/mirror/centos/download.ceph.com/rpm-mimic/el7/x86_64/ +repo=ussuri-wsgi,http://build.starlingx.cengn.ca:80/mirror/centos/centos/mirror.centos.org/centos/7/sclo/x86_64/rh/ diff --git a/build-tools/build-docker-images/build-stx-base.sh b/build-tools/build-docker-images/build-stx-base.sh index ac53e17f..a3cbe6bc 100755 --- a/build-tools/build-docker-images/build-stx-base.sh +++ b/build-tools/build-docker-images/build-stx-base.sh @@ -24,6 +24,9 @@ BUILD_STREAM=stable IMAGE_VERSION= PUSH=no PROXY="" +CONFIG_FILE="" +DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images" +DEFAULT_CONFIG_FILE_PREFIX="base-image-build" DOCKER_USER=${USER} DOCKER_REGISTRY= declare -a REPO_LIST @@ -56,11 +59,50 @@ Options: --clean: Remove image(s) from local registry --hostname: build repo host --attempts: Max attempts, in case of failure (default: 1) + --config-file:Specify a path to a config file which will specify additional arguments to be passed into the command EOF } -OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts: -- "$@") +function get_args_from_file { + # get additional args from specified file. + local -a config_items + + echo "Get args from file: $1" + for i in $(cat $1) + do + config_items=($(echo $i | sed s/=/\ /g)) + echo "--${config_items[0]} ${config_items[1]}" + case ${config_items[0]} in + version) + if [ -z "${IMAGE_VERSION}" ]; then + IMAGE_VERSION=${config_items[1]} + fi + ;; + user) + if [ -z "${DOCKER_USER}" ]; then + DOCKER_USER=${config_items[1]} + fi + ;; + proxy) + if [ -z "${PROXY}" ]; then + PROXY=${config_items[1]} + fi + ;; + registry) + if [ -z "${DOCKER_REGISTRY}" ]; then + # Add a trailing / if needed + DOCKER_REGISTRY="${config_items[1]%/}/" + fi + ;; + repo) + REPO_LIST+=(${config_items[1]}) + ;; + esac + done +} + +OPTS=$(getopt -o h -l help,os:,os-version:,version:,stream:,release:,repo:,push,proxy:,latest,latest-tag:,user:,registry:,local,clean,hostname:,attempts:,config-file: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -140,6 +182,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --config-file) + CONFIG_FILE=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -169,6 +215,21 @@ if [ -z "${IMAGE_VERSION}" ]; then IMAGE_VERSION=${OS_VERSION} fi +DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg" + +# Read additional auguments from config file if it exists. +if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then + CONFIG_FILE=${DEFAULT_CONFIG_FILE} +fi +if [[ ! -z ${CONFIG_FILE} ]]; then + if [[ -f ${CONFIG_FILE} ]]; then + get_args_from_file ${CONFIG_FILE} + else + echo "Config file not found: ${CONFIG_FILE}" + exit 1 + fi +fi + if [ ${#REPO_LIST[@]} -eq 0 ]; then # Either --repo or --local must be specified if [ "${LOCAL}" = "yes" ]; then diff --git a/build-tools/build-docker-images/build-stx-images.sh b/build-tools/build-docker-images/build-stx-images.sh index ca33b96e..7c82d4c1 100755 --- a/build-tools/build-docker-images/build-stx-images.sh +++ b/build-tools/build-docker-images/build-stx-images.sh @@ -27,12 +27,14 @@ PREFIX=dev LATEST_PREFIX="" PUSH=no PROXY="" +CONFIG_FILE="" DOCKER_USER=${USER} DOCKER_REGISTRY= BASE= WHEELS= WHEELS_ALTERNATE= -IMAGE_BUILD_CFG_FILE="docker-image-build.cfg" +DEFAULT_CONFIG_FILE_DIR="${MY_REPO}/build-tools/build-docker-images" +DEFAULT_CONFIG_FILE_PREFIX="docker-image-build" CLEAN=no TAG_LATEST=no TAG_LIST_FILE= @@ -69,6 +71,7 @@ Options: can be specified with a comma-separated list, or with multiple --skip arguments. --attempts: Max attempts, in case of failure (default: 1) + --config-file:Specify a path to a config file which will specify additional arguments to be passed into the the command EOF @@ -90,6 +93,63 @@ function is_empty { test $# -eq 0 } +function get_args_from_file { + # get additional build args from specified file. + local -a config_items + + echo "Get args from file: $1" + for i in $(cat $1) + do + config_items=($(echo $i | sed s/=/\ /g)) + echo "--${config_items[0]} ${config_items[1]}" + case ${config_items[0]} in + base) + if [ -z "${BASE}" ]; then + BASE=${config_items[1]} + fi + ;; + user) + if [ -z "${DOCKER_USER}" ]; then + DOCKER_USER=${config_items[1]} + fi + ;; + proxy) + if [ -z "${PROXY}" ]; then + PROXY=${config_items[1]} + fi + ;; + registry) + if [ -z "${DOCKER_REGISTRY}" ]; then + # Add a trailing / if needed + DOCKER_REGISTRY="${config_items[1]%/}/" + fi + ;; + only) + # Read comma-separated values into array + if [ -z "${ONLY}" ]; then + # Read comma-separated values into array + ONLY=(`echo ${config_items[1]} | sed s/,/\ /g`) + fi + ;; + wheels) + if [ -z "${WHEELS}" ]; then + WHEELS=${config_items[1]} + fi + ;; + wheels_alternate) + if [ -z "${WHEELS_ALTERNATE}" ]; then + WHEELS_ALTERNATE=${config_items[1]} + echo "WHEELS_ALTERNATE: ${WHEELS_ALTERNATE}" >&2 + fi + ;; + services_alternate) + SERVICES_ALTERNATE=(`echo ${config_items[1]} | sed s/,/\ /g`) + echo "SERVICES_ALTERNATE: ${SERVICES_ALTERNATE[@]}" >&2 + ;; + esac + done +} + # # get_git: Clones a git into a subdirectory of ${WORKDIR}, and # leaves you in that directory. On error the directory @@ -621,7 +681,7 @@ function build_image { esac } -OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts: -- "$@") +OPTS=$(getopt -o h -l help,os:,version:,release:,stream:,push,proxy:,user:,registry:,base:,wheels:,wheels-alternate:,only:,skip:,prefix:,latest,latest-prefix:,clean,attempts:,config-file: -- "$@") if [ $? -ne 0 ]; then usage exit 1 @@ -711,6 +771,10 @@ while true; do MAX_ATTEMPTS=$2 shift 2 ;; + --config-file) + CONFIG_FILE=$2 + shift 2 + ;; -h | --help ) usage exit 1 @@ -736,25 +800,26 @@ if [ ${VALID_OS} -ne 0 ]; then exit 1 fi +DEFAULT_CONFIG_FILE="${DEFAULT_CONFIG_FILE_DIR}/${DEFAULT_CONFIG_FILE_PREFIX}-${OS}-${BUILD_STREAM}.cfg" + +# Read additional arguments from config file if it exists. +if [[ -z "$CONFIG_FILE" ]] && [[ -f ${DEFAULT_CONFIG_FILE} ]]; then + CONFIG_FILE=${DEFAULT_CONFIG_FILE} +fi +if [[ ! -z ${CONFIG_FILE} ]]; then + if [[ -f ${CONFIG_FILE} ]]; then + get_args_from_file ${CONFIG_FILE} + else + echo "Config file not found: ${CONFIG_FILE}" + exit 1 + fi +fi + if [ -z "${WHEELS}" ]; then echo "Path to wheels tarball must be specified with --wheels option." >&2 exit 1 fi -# Get python2 based services from config file -if [ -f ${MY_REPO}/build-tools/build-docker-images/${IMAGE_BUILD_CFG_FILE} ]; then - SERVICES_ALTERNATE=(`source ${MY_REPO}/build-tools/build-docker-images/$IMAGE_BUILD_CFG_FILE \ - && echo ${SERVICES_ALTERNATE} | sed s/,/\ /g`) - if [ -z "${WHEELS_ALTERNATE}" ]; then - WHEELS_ALTERNATE=$(source ${MY_REPO}/build-tools/build-docker-images/$IMAGE_BUILD_CFG_FILE \ - && echo ${WHEELS_ALTERNATE}) - fi - echo "SERVICES_ALTERNATE: ${SERVICES_ALTERNATE[@]}" >&2 - echo "WHEELS_ALTERNATE: ${WHEELS_ALTERNATE}" >&2 -else - echo "IMAGE_BUILD_CFG_FILE not found!" >&2 -fi - if [ ${#SERVICES_ALTERNATE[@]} -ne 0 ] && [ -z "${WHEELS_ALTERNATE}" ]; then echo "Path to wheels-alternate tarball must be specified with --wheels-alternate option"\ "if python2 based services need to be build!" >&2 diff --git a/build-tools/build-docker-images/docker-image-build-centos-dev.cfg b/build-tools/build-docker-images/docker-image-build-centos-dev.cfg new file mode 100644 index 00000000..be8199e3 --- /dev/null +++ b/build-tools/build-docker-images/docker-image-build-centos-dev.cfg @@ -0,0 +1,2 @@ +services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients +wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/rc/4.0/centos/stx-centos-py2_dev-wheels.tar diff --git a/build-tools/build-docker-images/docker-image-build-centos-stable.cfg b/build-tools/build-docker-images/docker-image-build-centos-stable.cfg new file mode 100644 index 00000000..7f55b422 --- /dev/null +++ b/build-tools/build-docker-images/docker-image-build-centos-stable.cfg @@ -0,0 +1,2 @@ +services_alternate=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients +wheels_alternate=http://mirror.starlingx.cengn.ca/mirror/starlingx/rc/4.0/centos/stx-centos-py2_stable-wheels.tar diff --git a/build-tools/build-docker-images/docker-image-build.cfg b/build-tools/build-docker-images/docker-image-build.cfg deleted file mode 100644 index da0a1f81..00000000 --- a/build-tools/build-docker-images/docker-image-build.cfg +++ /dev/null @@ -1,2 +0,0 @@ -SERVICES_ALTERNATE=stx-fm-rest-api,stx-keystone-api-proxy,stx-nova-api-proxy,stx-platformclients -WHEELS_ALTERNATE=http://mirror.starlingx.cengn.ca/mirror/starlingx/master/centos/stx-centos-py2_stable-wheels.tar