openstack
/
tripleo-ci
Code Issues Proposed changes

Support running toci_gate_test-oooq.sh locally 

These are various improvements to tripleo-ci (squashed into a single
commit to save CI resources) that enable running the tripleo-ci entry
point script which is now toci_gate_test-oooq.sh locally in a standalone
environment of vm's not spawned by nodepool.

This is useful to developers trying to reproduce CI locally for
development and testing.

The changes are:
- Default $REMAINING_TIME in common_vars.bash
- Don't hardcode the jenkins user everywhere
- Improved initial package handling:
    Before blindly yum installing the packages needed by these scripts in a
    few spots, check if the packages are already installed with just the rpm
    command. This can end up saving a decent amount of time when iterating
    on running these scripts locally. Also installs wget.
- Reinstall python-requests
    When locally running toci_gate_test.sh, python-requests needs to be
    reinstalled since toci_gate_test.sh removes the python module as it
    was pip installed on the infra provided image. Also updates the
    logic to handle the python-requests -> python2-requests transition.

Change-Id: I48cd1e5f53b86455992b83248d57f5338c29a7ca
This commit is contained in:
James Slagle 2017-03-29 15:47:37 -04:00 committed by Sagi Shnaidman
parent 7050f811a6
commit bb430f3011
13 changed files with 66 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scripts/bootstrap-overcloud-full-minimal.sh
View File

@ -17,11 +17,10 @@ sudo yum erase -y epel-release || :
# Copied from toci_gate_test.sh...need to apply this fix on subnodes as well
# TODO(pabelanger): Why is python-requests installed from pip?
sudo rm -rf /usr/lib/python2.7/site-packages/requests
# Reinstall python-requests if it was already installed, otherwise it will be
# installed later when other packages are installed.
if rpm -q python-requests; then
sudo yum reinstall -y python-requests
fi
# Need to reinstall requests since it's been rm'd
sudo rpm -e --nodeps python-requests || :
sudo rpm -e --nodeps python2-requests || :
sudo yum -y install python-requests
# Clear out any puppet modules on the node placed their by infra configuration
sudo rm -rf /etc/puppet/modules/*

6
scripts/bootstrap-overcloud-full.sh
View File

@ -19,9 +19,9 @@ sudo yum erase -y epel-release || :
sudo rm -rf /usr/lib/python2.7/site-packages/requests
# Reinstall python-requests if it was already installed, otherwise it will be
# installed later when other packages are installed.
if rpm -q python-requests; then
sudo yum reinstall -y python-requests
fi
sudo rpm -e --nodeps python-requests || :
sudo rpm -e --nodeps python2-requests || :
sudo yum -y install python-requests
# Remove the anything on the infra image template that might interfere with CI
# Note for tripleo-quickstart: this task is already managed in tripleo-ci-setup-playbook.yml

4
scripts/common_vars.bash
View File

@ -31,6 +31,10 @@ export PATH=/sbin:/usr/sbin:$PATH
export UNDERCLOUD_VM_NAME=instack
export REMAINING_TIME=${REMAINING_TIME:-180}
export NODEPOOL_DOCKER_REGISTRY_PROXY=${NODEPOOL_DOCKER_REGISTRY_PROXY:-""}
export CENTOS_MIRROR=${CENTOS_MIRROR:-""}
# post ci chores to run at the end of ci
SSH_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32'

40
toci-quickstart/config/collect-logs.yml
View File

@ -8,26 +8,26 @@ artcl_rsync_logs: false
artcl_collect_list:
- /etc/
- /var/log/
- /home/jenkins/.instack/install-undercloud.log
- /home/jenkins/stackrc
- /home/jenkins/overcloudrc
- /home/jenkins/*.log
- /home/jenkins/*.txt
- /home/jenkins/*.json
- /home/jenkins/*.conf
- /home/jenkins/*.yml
- /home/jenkins/*.yaml
- /home/jenkins/*.sh
- /home/jenkins/deploy-overcloudrc
- /home/jenkins/network-environment.yaml
- /home/jenkins/skip_file
- /home/jenkins/tempest/*.xml
- /home/jenkins/tempest/*.html
- /home/jenkins/tempest/*.log
- /home/jenkins/tempest/*.subunit
- /home/jenkins/tempest/*.sh
- /home/jenkins/tempest/etc/*.conf
- /home/jenkins/local_tht/
- /home/{{ undercloud_user }}/.instack/install-undercloud.log
- /home/{{ undercloud_user }}/stackrc
- /home/{{ undercloud_user }}/overcloudrc
- /home/{{ undercloud_user }}/*.log
- /home/{{ undercloud_user }}/*.txt
- /home/{{ undercloud_user }}/*.json
- /home/{{ undercloud_user }}/*.conf
- /home/{{ undercloud_user }}/*.yml
- /home/{{ undercloud_user }}/*.yaml
- /home/{{ undercloud_user }}/*.sh
- /home/{{ undercloud_user }}/deploy-overcloudrc
- /home/{{ undercloud_user }}/network-environment.yaml
- /home/{{ undercloud_user }}/skip_file
- /home/{{ undercloud_user }}/tempest/*.xml
- /home/{{ undercloud_user }}/tempest/*.html
- /home/{{ undercloud_user }}/tempest/*.log
- /home/{{ undercloud_user }}/tempest/*.subunit
- /home/{{ undercloud_user }}/tempest/*.sh
- /home/{{ undercloud_user }}/tempest/etc/*.conf
- /home/{{ undercloud_user }}/local_tht/
- /tmp/*.yml
- /tmp/*.yaml
artcl_exclude_list:

6
toci-quickstart/config/testenv/multinode.yml
View File

@ -1,11 +1,11 @@
# TRIPLEO-CI environment settings
undercloud_user: jenkins
non_root_user: jenkins
undercloud_user: "{{ lookup('env','USER') }}"
non_root_user: "{{ undercloud_user }}"
extra_args: >
-e {{ tripleo_root }}/tripleo-ci/test-environments/worker-config.yaml
-e /usr/share/openstack-tripleo-heat-templates/environments/debug.yaml
--compute-scale 0
--overcloud-ssh-user {{ lookup('env','USER') }}
--overcloud-ssh-user {{ undercloud_user }}
toci_vxlan_networking: true
toci_vxlan_networking_multinode: true

4
toci-quickstart/config/testenv/osinfra_hosts
View File

@ -1,8 +1,8 @@
localhost ansible_connection=local
127.0.0.2 ansible_host=127.0.0.2 ansible_user=jenkins ansible_private_key_file=/home/jenkins/.ssh/id_rsa undercloud_ip=127.0.0.2
127.0.0.2 ansible_host=127.0.0.2 ansible_user="{{ undercloud_user }}" ansible_private_key_file="{{ lookup('env','HOME') }}"/.ssh/id_rsa undercloud_ip=127.0.0.2
undercloud ansible_host=127.0.0.2 ansible_user=jenkins ansible_private_key_file=/home/jenkins/.ssh/id_rsa undercloud_ip=127.0.0.2
undercloud ansible_host=127.0.0.2 ansible_user="{{ undercloud_user }}" ansible_private_key_file="{{ lookup('env','HOME') }}"/.ssh/id_rsa undercloud_ip=127.0.0.2
[virthost]
127.0.0.2

6
toci-quickstart/config/testenv/ovb.yml
View File

@ -1,6 +1,6 @@
undercloud_user: jenkins
non_root_user: jenkins
ssh_user: jenkins
undercloud_user: "{{ lookup('env','USER') }}"
non_root_user: "{{ undercloud_user }}"
ssh_user: "{{ undercloud_user }}"
enable_vbmc: false
gating_repo_enabled: true

4
toci-quickstart/config/testenv/ovb_hosts
View File

@ -1,8 +1,8 @@
localhost ansible_connection=local
127.0.0.2 ansible_host=127.0.0.2 ansible_user=jenkins ansible_private_key_file=/home/jenkins/.ssh/id_rsa undercloud_ip=127.0.0.2
127.0.0.2 ansible_host=127.0.0.2 ansible_user="{{ undercloud_user }}" ansible_private_key_file="{{ lookup('env','HOME') }}"/.ssh/id_rsa undercloud_ip=127.0.0.2
undercloud ansible_host=undercloud ansible_user=jenkins ansible_private_key_file=/home/jenkins/.ssh/id_rsa undercloud_ip=127.0.0.2
undercloud ansible_host=undercloud ansible_user="{{ undercloud_user }}" ansible_private_key_file="{{ lookup('env','HOME') }}"/.ssh/id_rsa undercloud_ip=127.0.0.2
[virthost]
127.0.0.2

4
toci-quickstart/playbooks/containers_minimal.yml
View File

@ -1,5 +1,5 @@
# set the working_dir on the undercloud
working_dir: /home/jenkins
working_dir: {{ lookup('env','HOME') }}
# Define a single controller node and a single compute node.
overcloud_nodes:
@ -43,7 +43,7 @@ tempest_workers: 4
#overcloud_templates_refspec:
# Config for custom tripleo-common
#overcloud_tripleo_common_path: /home/jenkins/tripleo-common
#overcloud_tripleo_common_path: {{ lookup('env','HOME') }}/tripleo-common
#overcloud_tripleo_common_repo: https://git.openstack.org/openstack/tripleo-common
# use overcloud_tripleo_common _branch when not testing with an unmerged review
#overcloud_tripleo_common_branch: master

2
toci-quickstart/playbooks/multinode.yml
View File

@ -34,7 +34,7 @@
- undercloud-setup
environment:
WORKSPACE: "{{ lookup('env','WORKSPACE') }}"
SSH_OPTIONS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32 -l jenkins"
SSH_OPTIONS: "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=Verbose -o PasswordAuthentication=no -o ConnectionAttempts=32 -l {{ undercloud_user }}"
- name: Run DLRN gate role and install repo
hosts: undercloud

19
toci_gate_test-oooq.sh
View File

@ -39,17 +39,24 @@ if [ $NODEPOOL_CLOUD == 'tripleo-test-cloud-rh1' ]; then
sudo rm -rf /opt/git /opt/stack/cache/files/mysql.qcow2 /opt/stack/cache/files/ubuntu-12.04-x86_64.tar.gz
fi
# Remove epel, either by epel-release, or unpackaged repo files
rpm -q epel-release && sudo yum -y erase epel-release
sudo rm -f /etc/yum.repos.d/epel*
# Clean any cached yum metadata, it maybe stale
sudo rm /etc/yum.repos.d/epel*
sudo yum clean all
# Install additional packages
sudo yum install -y \
qemu-img # used by multinode to create empty image
rpm -q qemu-img || \
sudo yum install -y \
qemu-img # used by multinode to create empty image
# NOTE(pabelanger): Current hack to make centos-7 dib work.
# TODO(pabelanger): Why is python-requests installed from pip?
sudo rm -rf /usr/lib/python2.7/site-packages/requests
sudo rpm -e --nodeps python-requests || :
sudo rpm -e --nodeps python2-requests || :
sudo yum -y install python-requests
# JOB_NAME used to be available from jenkins, we need to create it ourselves until
# we remove our reliance on it.
@ -142,6 +149,8 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
esac
done
command -v pip || \
(curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"; sudo python get-pip.py)
sudo pip install shyaml
if [[ ! -z $NODES_FILE ]]; then
pushd $TRIPLEO_ROOT/tripleo-quickstart
@ -173,7 +182,7 @@ else
# Clear out any puppet modules on the node placed their by infra configuration
sudo rm -rf /etc/puppet/modules/*
# Copy nodepool keys to jenkins user
# Copy nodepool keys to current user
sudo cp /etc/nodepool/id_rsa* $HOME/.ssh/
sudo chown $USER:$USER $HOME/.ssh/id_rsa*
chmod 0600 $HOME/.ssh/id_rsa*
@ -197,7 +206,7 @@ else
undercloud_haproxy_admin_ip=$undercloud_net_range"3"
export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1
qemu-img create -f qcow2 /home/jenkins/overcloud-full.qcow2 1G
qemu-img create -f qcow2 $HOME/overcloud-full.qcow2 1G
# multinode bootstrap script
export BOOTSTRAP_SUBNODES_MINIMAL=0

6
toci_gate_test-orig.sh
View File

@ -28,6 +28,9 @@ sudo yum clean all
# NOTE(pabelanger): Current hack to make centos-7 dib work.
# TODO(pabelanger): Why is python-requests installed from pip?
sudo rm -rf /usr/lib/python2.7/site-packages/requests
sudo rpm -e --nodeps python-requests || :
sudo rpm -e --nodeps python2-requests || :
sudo yum -y install python-requests
# Remove metrics from a previous run
rm -f /tmp/metric-start-times /tmp/metrics-data
@ -144,13 +147,14 @@ elif [[ "$TOCI_JOBTYPE" =~ "periodic" && "$TOCI_JOBTYPE" =~ "-nonha" ]]; then
fi
# Test version of ssh package for bug https://bugzilla.redhat.com/show_bug.cgi?id=1415218
rpm -q wget || sudo yum install -y wget
http_proxy= wget -P /tmp -T 60 --tries=3 --progress=dot:mega http://66.187.229.139/test/openssh-6.6.1p1-33.el7.x86_64.rpm
http_proxy= wget -P /tmp -T 60 --tries=3 --progress=dot:mega http://66.187.229.139/test/openssh-server-6.6.1p1-33.el7.x86_64.rpm
sudo rpm -ivh --force /tmp/openssh-6.6.1p1-33.el7.x86_64.rpm /tmp/openssh-server-6.6.1p1-33.el7.x86_64.rpm
# start dstat early
# TODO add it to the gate image building
sudo yum install -y dstat nmap-ncat #nc is for metrics
rpm -q dstat nmap-ncat || sudo yum install -y dstat nmap-ncat #nc is for metrics
mkdir -p "$WORKSPACE/logs"
dstat -tcmndrylpg --top-cpu-adv --top-io-adv --nocolor | tee --append $WORKSPACE/logs/dstat.log > /dev/null &
disown

5
toci_instack_osinfra.sh
View File

@ -25,8 +25,8 @@ undercloud_haproxy_public_ip=$undercloud_net_range"2"
undercloud_haproxy_admin_ip=$undercloud_net_range"3"
export no_proxy=$undercloud_services_ip,$undercloud_haproxy_public_ip,$undercloud_haproxy_admin_ip,$MY_IP,$MY_IP_eth1
# Copy nodepool ssh keys for the jenkins user because apparently id_rsa.pub is
# missing from /home/jenkins/.ssh
# Copy nodepool ssh keys for the current user because apparently id_rsa.pub is
# missing from ~/.ssh
cp /etc/nodepool/id_rsa ~/.ssh/
cp /etc/nodepool/id_rsa.pub ~/.ssh/
@ -47,6 +47,7 @@ $TRIPLEO_ROOT/tripleo-ci/scripts/tripleo.sh --repo-setup
sudo yum -y install wget python-simplejson yum-plugin-priorities
# Need to reinstall requests since it's rm'd in toci_gate_test.sh
sudo rpm -e --nodeps python-requests || :
sudo rpm -e --nodeps python2-requests || :
sudo yum -y install python-requests
# Open up port for delorean yum repo server
sudo iptables -I INPUT -p tcp --dport 8766 -j ACCEPT