openstack
/
tripleo-ci
Code Issues Proposed changes

Move toci BASH variables to ansible 

This review contains the following changes and references work from [1]:

DRY_RUN, STATS_* and COLLECT_CONF vars:

	The dryrun ansible var will not be defined by default,
	only manually defined at reproducer and similar.

        - PLAYBOOK_DRY_RUN -> <job config>:dryrun

	The STATS_ vars will be re-implemented after TOCI refactoring

featureset related:

	It does the following move to ansible:
	- FEATURESET_FILE -> common.yaml:job_featureset_file
	- FEATURESET_CONF -> common.yaml:featureset_conf (job_featureset_file +
                                         featureset-multinode-common.yml)
                          -> ovb.yaml:featureset_conf (job_featureset_file)
                                      overrides common.yaml

	Also add additional ansible variables:
	- multinode_featureset_file: to point to tq featureset-multinode-common.yaml
	- tripleo_quickstart_dir: to point to tq dir at tripleo_root
	- common_featureset_conf to have the --extra-vars for the featurset_file

Replace toci ENV_VARS by ansible env_vars:

	This move the ENV_VARS to the different
	environment_type yaml config files.

	It also implements there the check if it's running in RDO,
	to add the extra config for it.

	Also note that RHCLOUD env variable has being removed,
	and harcoded the use with "rdocloud".

Move nodes_file and nodes_args to common.yaml:

	As part of [1] task, we do the following move
	from toci bash environment variables:
	- NODES_FILE -> common.yaml:nodes_file
	- NODES_ARGS -> common.yaml:nodes_args

        NODES_FILE exported for the reproducer (will be addressed in a
        separate change)

[1] https://trello.com/c/AerVRkBH/887-replace-bash-environment-variables-in-tociquickstartsh-with-ansible-variables-from-zuul-inventory-or-zuul-job-config-s17

Change-Id: I786ef968953e2bff34939c15c16a01f10130d2aa
This commit is contained in:
Quique Llorente 2018-07-31 10:45:44 +02:00 committed by Ronelle Landy
parent 51548c7635
commit debdb4b961
9 changed files with 68 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
playbooks/tripleo-ci/templates/oooq_common_functions.sh.j2
View File

@ -114,7 +114,7 @@ function create_collect_logs_script {
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
export NODES_FILE=${NODES_FILE:-''}
export NODES_FILE={{ nodes_file }}
export TOCI_JOBTYPE=$TOCI_JOBTYPE
export STABLE_RELEASE=${STABLE_RELEASE:-''}
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}

70
playbooks/tripleo-ci/templates/toci_gate_test.sh.j2
View File

@ -19,10 +19,8 @@ if [ -f /etc/nodepool/provider ] ; then
# source variables common across all the scripts.
source /etc/ci/mirror_info.sh
export RHCLOUD=''
if [ ${NODEPOOL_PROVIDER:-''} == 'rdo-cloud-tripleo' ]; then
RHCLOUD='rdocloud'
source $(dirname $0)/scripts/$RHCLOUD.env
source $(dirname $0)/scripts/rdocloud.env
# In order to save space remove the cached git repositories, at this point in
# CI the ones we are interested in have been cloned to /opt/stack/new. We
@ -58,13 +56,6 @@ export PERIODIC=0
export QUICKSTART_RELEASE="${STABLE_RELEASE:-master}"
# Stores OVB undercloud instance id
export UCINSTANCEID="null"
# Define environment variables file
export ENV_VARS=""
# Define file with set of features to test
export FEATURESET_FILE=""
export FEATURESET_CONF=""
# Define file with nodes topology
export NODES_FILE=""
# Set the number of overcloud nodes
export NODECOUNT=0
# Set the overcloud hosts for multinode
@ -74,12 +65,8 @@ export SUBNODES_SSH_KEY=
TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60))
export EXTRA_VARS=${EXTRA_VARS:-""}
export VXLAN_VARS=${VXLAN_VARS:-""}
export NODES_ARGS=""
export EXTRANODE=""
export EMIT_RELEASES_EXTRA_ARGS=""
# Set playbook execution status
export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0}
export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml"
LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
LWD=$LOCAL_WORKING_DIR
@ -91,49 +78,24 @@ case {{ environment_type | default('standalone') }} in
ovb)
OVB=1
UCINSTANCEID=$(http_proxy= curl http://169.254.169.254/openstack/2015-10-15/meta_data.json | python -c 'import json, sys; print json.load(sys.stdin)["uuid"]')
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb.yml"
if [[ -f "$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/ovb-$RHCLOUD.yml"
fi
;;
multinode)
SUBNODES_SSH_KEY=/etc/nodepool/id_rsa
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
CONTROLLER_HOSTS=$(sed -n 1,1p /etc/nodepool/sub_nodes_private)
OVERCLOUD_HOSTS=$(cat /etc/nodepool/sub_nodes_private)
;;
singlenode)
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
;;
standalone)
FEATURESET_CONF=" --extra-vars @$LWD/config/general_config/featureset-multinode-common.yml $FEATURESET_CONF"
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode.yml"
if [[ $NODEPOOL_PROVIDER == "rdo-cloud-tripleo" ]]; then
ENV_VARS="$ENV_VARS --extra-vars @$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/testenv/multinode-rdocloud.yml"
fi
;;
esac
EXTRA_TAGS=""
FEATURESET_FILE="{{ tripleo_root}}/tripleo-quickstart/config/general_config/featureset{{ featureset }}.yml"
FEATURESET_CONF="$FEATURESET_CONF --extra-vars @$FEATURESET_FILE"
MIXED_UPGRADE_TYPE=''
# Order matters. ffu featureset has both mixed version and ffu_overcloud_upgrade.
if is_featureset ffu_overcloud_upgrade "$FEATURESET_FILE"; then
if is_featureset ffu_overcloud_upgrade "{{ job_featureset_file }}"; then
MIXED_UPGRADE_TYPE='ffu_upgrade'
elif is_featureset mixed_upgrade "$FEATURESET_FILE"; then
elif is_featureset mixed_upgrade "{{ job_featureset_file }}"; then
MIXED_UPGRADE_TYPE='mixed_upgrade'
elif is_featureset overcloud_update "$FEATURESET_FILE"; then
elif is_featureset overcloud_update "{{ job_featureset_file }}"; then
EXTRA_TAGS=",overcloud-update"
elif is_featureset undercloud_upgrade "$FEATURESET_FILE"; then
elif is_featureset undercloud_upgrade "{{ job_featureset_file }}"; then
EXTRA_TAGS=",undercloud-upgrade"
export UPGRADE_RELEASE=$QUICKSTART_RELEASE
export QUICKSTART_RELEASE=$(previous_release_mixed_upgrade_case "${UPGRADE_RELEASE}")
@ -147,43 +109,35 @@ fi
export EXTRA_TAGS
{% if nodes is defined %}
NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/{{ nodes }}.yml"
{% endif %}
{% if periodic|bool %}
PERIODIC=1
QUICKSTART_RELEASE="promotion-testing-hash-${QUICKSTART_RELEASE}"
EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --is-periodic"
{% endif %}
{% if dryrun|default(false)|bool %}
#TODO(rfolco): Remove PLAYBOOK_DRY_RUN when bash code is translated to ansible playbooks
PLAYBOOK_DRY_RUN=1
{% endif %}
if [[ -f "$RELEASES_SCRIPT" ]] && [[ $FEATURESET_FILE =~ '037' || $FEATURESET_FILE =~ '050' || $FEATURESET_FILE =~ '010' || $FEATURESET_FILE =~ '011' ]]; then
if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 037|050|010|011 ]]; then
python $RELEASES_SCRIPT \
--stable-release ${STABLE_RELEASE:-"master"} \
--featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename $FEATURESET_FILE) \
--featureset-file $TRIPLEO_ROOT/tripleo-quickstart/config/general_config/$(basename {{ job_featureset_file }}) \
--output-file $RELEASES_FILE_OUTPUT \
--log-file $RELEASES_SCRIPT_LOGFILE \
$EMIT_RELEASES_EXTRA_ARGS
fi
if [[ ! -z $NODES_FILE ]]; then
{% if nodes is defined %}
if [[ ! -z {{ nodes_file }} ]]; then
pushd $TRIPLEO_ROOT/tripleo-quickstart
NODECOUNT=$(shyaml get-value node_count < $NODES_FILE)
NODECOUNT=$(shyaml get-value node_count < {{ nodes_file }})
popd
NODES_ARGS="--extra-vars @$NODES_FILE"
for PART in $(sed 's/_/ /g' <<< "$NODES_FILE") ; do
for PART in $(sed 's/_/ /g' <<< "{{ nodes }}") ; do
if [[ "$PART" == *"supp"* ]]; then
EXTRANODE=" --extra-nodes ${PART//[!0-9]/} "
fi;
done
fi
{% endif %}
# Import gated tripleo-upgrade in oooq for upgrades/updates jobs
if [[ -d $TRIPLEO_ROOT/tripleo-upgrade ]]; then

32
playbooks/tripleo-ci/templates/toci_quickstart.sh.j2
View File

@ -2,8 +2,6 @@
set -eux
set -o pipefail
export ANSIBLE_NOCOLOR=1
[[ -n ${STATS_TESTENV:-''} ]] && export STATS_TESTENV=$(( $(date +%s) - STATS_TESTENV ))
export STATS_OOOQ=$(date +%s)
LOCAL_WORKING_DIR="{{ workspace }}/.quickstart"
WORKING_DIR="$HOME"
@ -49,11 +47,11 @@ QUICKSTART_COLLECTLOGS_CMD="$LOCAL_WORKING_DIR/bin/ansible-playbook \
$LOCAL_WORKING_DIR/playbooks/collect-logs.yml \
-vv \
--extra-vars @$LOCAL_WORKING_DIR/config/release/tripleo-ci/$QUICKSTART_RELEASE.yml \
$FEATURESET_CONF \
$ENV_VARS \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
$DEFAULT_ARGS \
--extra-vars @$COLLECT_CONF \
--extra-vars @{{ tripleo_root }}/tripleo-ci/toci-quickstart/config/collect-logs.yml \
--extra-vars artcl_collect_dir=$LOGS_DIR \
--tags all \
--skip-tags teardown-all \
@ -112,22 +110,21 @@ set +u
source $LOCAL_WORKING_DIR/bin/activate
set -u
source $OOOQ_DIR/ansible_ssh_env.sh
[[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ ))
# Debug step capture env variables
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
{% if dryrun|default(false)|bool %}
echo "-- Capture Environment Variables Used ---------"
echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log
declare -p | tee -a $LOGS_DIR/toci_env_args_output.log
fi
{% endif %}
echo "-- Playbooks Output --------------------------"
for playbook in {{ " ".join(playbooks) }}; do
echo "$QUICKSTART_INSTALL_CMD \
${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \
$NODES_ARGS \
$FEATURESET_CONF \
$ENV_VARS \
{{ nodes_args }} \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
{{ vxlan_vars }} \
$DEFAULT_ARGS \
@ -138,18 +135,19 @@ for playbook in {{ " ".join(playbooks) }}; do
| tee -a $LOGS_DIR/playbook_executions.log
done
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
{% if dryrun|default(false)|bool %}
# Bypassing execution, dryrun was set
exit_value=0
else
{% else %}
## LOGS COLLECTION PREPARE
create_collect_logs_script
for playbook in {{ " ".join(playbooks) }}; do
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
$NODES_ARGS \
$FEATURESET_CONF \
$ENV_VARS \
{{ nodes_args }} \
{{ featureset_conf }} \
{{ env_vars }} \
{{ extra_vars }} \
{{ vxlan_vars }} \
$DEFAULT_ARGS \
@ -172,7 +170,7 @@ done
fi
fi
fi
{% endif %}
popd

38
playbooks/tripleo-ci/vars/common.yaml
View File

@ -1,10 +1,38 @@
workspace: "{{ ansible_user_dir }}/workspace/"
tripleo_root: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/"
workspace: "{{ ansible_user_dir }}/workspace"
tripleo_root: "{{ ansible_user_dir }}/src/git.openstack.org/openstack"
tripleo_ci_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack-infra/tripleo-ci'].src_dir }}"
tripleo_quickstart_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-quickstart'].src_dir }}"
vxlan_vars: "-e toci_vxlan_networking='false' -e vxlan_networking='false'"
extra_vars: ""
job_featureset_file: "{{ tripleo_quickstart_dir }}/config/general_config/featureset{{ featureset }}.yml"
featureset_conf: >-
{% if common_featureset is defined -%}
--extra-vars @{{ tripleo_quickstart_dir }}/config/general_config/{{ common_featureset }} {% endif -%}
--extra-vars @{{ job_featureset_file }}
periodic: >-
{%- if 'periodic' in zuul.pipeline -%}
{% if 'periodic' in zuul.pipeline -%}
true
{%- else -%}
{% else %}
false
{%- endif -%}
{%- endif %}
nodepool_provider: "{{ hostvars[inventory_hostname].nodepool.provider }}"
nodes_file: >-
{% if nodes is defined -%}
{{ tripleo_quickstart_dir }}/config/nodes/{{ nodes }}.yml
{%- endif %}
nodes_args: >-
{% if nodes is defined -%}
--extra-vars @{{ nodes_file }}
{%- endif %}
rdo_env_vars: >-
{% if nodepool_provider == 'rdo-cloud-tripleo' -%}
--extra-vars @{{ tripleo_ci_dir }}/toci-quickstart/config/testenv/{{ environment_type }}-rdocloud.yml
{%- endif %}
env_vars: "--extra-vars @{{ tripleo_ci_dir }}/toci-quickstart/config/testenv/{{ environment_type }}.yml {{ rdo_env_vars }}"

1
toci-quickstart/config/testenv/singlenode-rdocloud.yml Symbolic link
View File

@ -0,0 +1 @@
multinode-rdocloud.yml

1
toci-quickstart/config/testenv/singlenode.yml Symbolic link
View File

@ -0,0 +1 @@
multinode.yml

1
toci-quickstart/config/testenv/standalone-rdocloud.yml Symbolic link
View File

@ -0,0 +1 @@
multinode-rdocloud.yml

1
toci-quickstart/config/testenv/standalone.yml Symbolic link
View File

@ -0,0 +1 @@
multinode.yml

3
zuul.d/base.yaml
View File

@ -57,6 +57,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-undercloud.yml
@ -108,6 +109,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-undercloud.yml
@ -134,6 +136,7 @@
vars:
undercloud: 127.0.0.2
environment_infra: osinfra
common_featureset: featureset-multinode-common.yml
playbooks:
- quickstart.yml
- multinode-standalone.yml