Add Zuul v3 tests for stable branch

Change-Id: I6ba35ea9e2080012f098569efcd65e1145829d91
This commit is contained in:
Dean Troyer 2017-11-06 14:20:56 -06:00
parent a5c77a91b9
commit e720c3c051
36 changed files with 695 additions and 0 deletions

84
.zuul.yaml Normal file
View File

@ -0,0 +1,84 @@
# - nodeset:
# name: openstack-single-node
# nodes:
# - name: controller
# label: ubuntu-xenial
# groups:
# - name: tempest
# nodes:
# - controller
# - nodeset:
# name: openstack-two-node
# nodes:
# - name: controller
# label: ubuntu-xenial
# - name: compute1
# label: ubuntu-xenial
# groups:
# - name: tempest
# nodes:
# - controller
# - name: compute
# nodes:
# - controller
# - compute1
- job:
name: devstack
parent: multinode
description: Base devstack job
nodeset: openstack-single-node
required-projects:
- openstack-dev/devstack
- openstack/cinder
- openstack/glance
- openstack/keystone
- openstack/neutron
- openstack/nova
- openstack/requirements
- openstack/swift
roles:
- zuul: openstack-infra/openstack-zuul-jobs
timeout: 7200
vars:
devstack_localrc:
DATABASE_PASSWORD: secretdatabase
RABBIT_PASSWORD: secretrabbit
ADMIN_PASSWORD: secretadmin
SERVICE_PASSWORD: secretservice
NETWORK_GATEWAY: 10.1.0.1
Q_USE_DEBUG_COMMAND: True
FIXED_RANGE: 10.1.0.0/20
IPV4_ADDRS_SAFE_TO_USE: 10.1.0.0/20
FLOATING_RANGE: 172.24.5.0/24
PUBLIC_NETWORK_GATEWAY: 172.24.5.1
FLOATING_HOST_PREFIX: 172.24.4
FLOATING_HOST_MASK: 23
SWIFT_REPLICAS: 1
SWIFT_START_ALL_SERVICES: False
LOGFILE: /opt/stack/logs/devstacklog.txt
LOG_COLOR: False
VERBOSE: True
NETWORK_GATEWAY: 10.1.0.1
NOVNC_FROM_PACKAGE: True
ERROR_ON_CLONE: True
# Gate jobs can't deal with nested virt. Disable it.
LIBVIRT_TYPE: qemu
# NOTE(dims): etcd 3.x is not available in debian/ubuntu
# etc. As a stop gap measure, devstack uses wget to download
# from the location below for all the CI jobs.
ETCD_DOWNLOAD_URL: "http://tarballs.openstack.org/etcd/"
devstack_services:
horizon: False
tempest: False
pre-run: playbooks/pre
run: playbooks/devstack
post-run: playbooks/post
- project:
name: openstack-dev/devstack
check:
jobs:
- devstack

3
playbooks/devstack.yaml Normal file
View File

@ -0,0 +1,3 @@
- hosts: all
roles:
- run-devstack

4
playbooks/post.yaml Normal file
View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- export-devstack-journal
- fetch-devstack-log-dir

22
playbooks/pre.yaml Normal file
View File

@ -0,0 +1,22 @@
- hosts: all
roles:
- configure-swap
- setup-stack-user
- setup-tempest-user
- setup-devstack-source-dirs
- setup-devstack-log-dir
- setup-devstack-cache
- start-fresh-logging
- write-devstack-local-conf
# TODO(jeblair): remove when configure-mirrors is fixed
tasks:
- name: Hack mirror_info
shell:
_raw_params: |
mkdir /etc/ci
cat << "EOF" > /etc/ci/mirror_info.sh
export NODEPOOL_UCA_MIRROR=http://mirror.dfw.rax.openstack.org/ubuntu-cloud-archive
EOF
args:
executable: /bin/bash
become: true

View File

@ -0,0 +1,15 @@
Export journal files from devstack services
Export the systemd journal for every devstack service in native
journal format as well as text. Also, export a syslog-style file with
kernal and sudo messages.
Writes the output to the ``logs/`` subdirectory of
``devstack_base_dir``.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,29 @@
# TODO: convert this to ansible
- name: Export journal files
become: true
shell:
cmd: |
u=""
name=""
for u in `systemctl list-unit-files | grep devstack | awk '{print $1}'`; do
name=$(echo $u | sed 's/devstack@/screen-/' | sed 's/\.service//')
journalctl -o short-precise --unit $u | tee {{ devstack_base_dir }}/logs/$name.txt > /dev/null
done
# Export the journal in export format to make it downloadable
# for later searching. It can then be rewritten to a journal native
# format locally using systemd-journal-remote. This makes a class of
# debugging much easier. We don't do the native conversion here as
# some distros do not package that tooling.
journalctl -u 'devstack@*' -o export | \
xz --threads=0 - > {{ devstack_base_dir }}/logs/devstack.journal.xz
# The journal contains everything running under systemd, we'll
# build an old school version of the syslog with just the
# kernel and sudo messages.
journalctl \
-t kernel \
-t sudo \
--no-pager \
--since="$(cat {{ devstack_base_dir }}/log-start-timestamp.txt)" \
| tee {{ devstack_base_dir }}/logs/syslog.txt > /dev/null

View File

@ -0,0 +1,10 @@
Fetch content from the devstack log directory
Copy logs from every host back to the zuul executor.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,5 @@
- name: Collect devstack logs
synchronize:
dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"
mode: pull
src: "{{ devstack_base_dir }}/logs"

View File

@ -0,0 +1,8 @@
Run devstack
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,6 @@
- name: Run devstack
command: ./stack.sh
args:
chdir: "{{devstack_base_dir}}/devstack"
become: true
become_user: stack

View File

@ -0,0 +1,15 @@
Set up the devstack cache directory
If the node has a cache of devstack image files, copy it into place.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.
.. zuul:rolevar:: devstack_cache_dir
:default: /opt/cache
The directory with the cached files.

View File

@ -0,0 +1,2 @@
devstack_base_dir: /opt/stack
devstack_cache_dir: /opt/cache

View File

@ -0,0 +1,14 @@
- name: Copy cached devstack files
# This uses hard links to avoid using extra space.
command: "find {{ devstack_cache_dir }}/files -mindepth 1 -maxdepth 1 -exec cp -l {} {{ devstack_base_dir }}/devstack/files/ ;"
become: true
- name: Set ownership of cached files
file:
path: '{{ devstack_base_dir }}/devstack/files'
state: directory
recurse: true
owner: stack
group: stack
mode: a+r
become: yes

View File

@ -0,0 +1,11 @@
Set up the devstack log directory
Create a log directory on the ephemeral disk partition to save space
on the root device.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,5 @@
- name: Create logs directory
file:
path: '{{ devstack_base_dir }}/logs'
state: directory
become: yes

View File

@ -0,0 +1,11 @@
Set up the devstack source directories
Ensure that the base directory exists, and then move the source repos
into it.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,22 @@
- name: Find all source repos used by this job
find:
paths:
- src/git.openstack.org/openstack
- src/git.openstack.org/openstack-dev
- src/git.openstack.org/openstack-infra
file_type: directory
register: found_repos
- name: Copy Zuul repos into devstack working directory
command: rsync -a {{ item.path }} {{ devstack_base_dir }}
with_items: '{{ found_repos.files }}'
become: yes
- name: Set ownership of repos
file:
path: '{{ devstack_base_dir }}'
state: directory
recurse: true
owner: stack
group: stack
become: yes

View File

@ -0,0 +1,16 @@
Set up the `stack` user
Create the stack user, set up its home directory, and allow it to
sudo.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.
.. zuul:rolevar:: devstack_stack_home_dir
:default: {{ devstack_base_dir }}
The home directory for the stack user.

View File

@ -0,0 +1,2 @@
devstack_base_dir: /opt/stack
devstack_stack_home_dir: '{{ devstack_base_dir }}'

View File

@ -0,0 +1 @@
stack ALL=(root) NOPASSWD:ALL

View File

@ -0,0 +1,45 @@
- name: Create stack group
group:
name: stack
become: yes
# NOTE(andreaf) Create a user home_dir is not safe via
# the user module since it will fail if the containing
# folder does not exists. If the folder does exists and
# it's empty, the skeleton is setup and ownership set.
- name: Create the stack user home folder
file:
path: '{{ devstack_stack_home_dir }}'
state: directory
become: yes
- name: Create stack user
user:
name: stack
shell: /bin/bash
home: '{{ devstack_stack_home_dir }}'
group: stack
become: yes
- name: Set stack user home directory permissions
file:
path: '{{ devstack_stack_home_dir }}'
mode: 0755
become: yes
- name: Copy 50_stack_sh file to /etc/sudoers.d
copy:
src: 50_stack_sh
dest: /etc/sudoers.d
mode: 0440
owner: root
group: root
become: yes
- name: Create new/.cache folder within BASE
file:
path: '{{ devstack_stack_home_dir }}/.cache'
state: directory
owner: stack
group: stack
become: yes

View File

@ -0,0 +1,10 @@
Set up the `tempest` user
Create the tempest user and allow it to sudo.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1,3 @@
tempest ALL=(root) NOPASSWD:/sbin/ip
tempest ALL=(root) NOPASSWD:/sbin/iptables
tempest ALL=(root) NOPASSWD:/usr/bin/ovsdb-client

View File

@ -0,0 +1,20 @@
- name: Create tempest group
group:
name: tempest
become: yes
- name: Create tempest user
user:
name: tempest
shell: /bin/bash
group: tempest
become: yes
- name: Copy 51_tempest_sh to /etc/sudoers.d
copy:
src: 51_tempest_sh
dest: /etc/sudoers.d
owner: root
group: root
mode: 0440
become: yes

View File

@ -0,0 +1,11 @@
Restart logging on all hosts
Restart syslog so that the system logs only include output from the
job.
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.

View File

@ -0,0 +1 @@
devstack_base_dir: /opt/stack

View File

@ -0,0 +1,56 @@
- name: Check for /bin/journalctl file
command: which journalctl
changed_when: False
failed_when: False
register: which_out
- block:
- name: Get current date
command: date +"%Y-%m-%d %H:%M:%S"
register: date_out
- name: Copy current date to log-start-timestamp.txt
copy:
dest: "{{ devstack_base_dir }}/log-start-timestamp.txt"
content: "{{ date_out.stdout }}"
when: which_out.rc == 0
become: yes
- block:
- name: Stop rsyslog
service: name=rsyslog state=stopped
- name: Save syslog file prior to devstack run
command: mv /var/log/syslog /var/log/syslog-pre-devstack
- name: Save kern.log file prior to devstack run
command: mv /var/log/kern.log /var/log/kern_log-pre-devstack
- name: Recreate syslog file
file: name=/var/log/syslog state=touch
- name: Recreate syslog file owner and group
command: chown /var/log/syslog --ref /var/log/syslog-pre-devstack
- name: Recreate syslog file permissions
command: chmod /var/log/syslog --ref /var/log/syslog-pre-devstack
- name: Add read permissions to all on syslog file
file: name=/var/log/syslog mode=a+r
- name: Recreate kern.log file
file: name=/var/log/kern.log state=touch
- name: Recreate kern.log file owner and group
command: chown /var/log/kern.log --ref /var/log/kern_log-pre-devstack
- name: Recreate kern.log file permissions
command: chmod /var/log/kern.log --ref /var/log/kern_log-pre-devstack
- name: Add read permissions to all on kern.log file
file: name=/var/log/kern.log mode=a+r
- name: Start rsyslog
service: name=rsyslog state=started
when: which_out.rc == 1
become: yes

View File

@ -0,0 +1,63 @@
Write the local.conf file for use by devstack
**Role Variables**
.. zuul:rolevar:: devstack_base_dir
:default: /opt/stack
The devstack base directory.
.. zuul:rolevar:: devstack_local_conf_path
:default: {{ devstack_base_dir }}/devstack/local.conf
The path of the local.conf file.
.. zuul:rolevar:: devstack_localrc
:type: dict
A dictionary of variables that should be written to the localrc
section of local.conf. The values (which are strings) may contain
bash shell variables, and will be ordered so that variables used by
later entries appear first.
.. zuul:rolevar:: devstack_local_conf
:type: dict
A complex argument consisting of nested dictionaries which combine
to form the meta-sections of the local_conf file. The top level is
a dictionary of phases, followed by dictionaries of filenames, then
sections, which finally contain key-value pairs for the INI file
entries in those sections.
The keys in this dictionary are the devstack phases.
.. zuul:rolevar:: [phase]
:type: dict
The keys in this dictionary are the filenames for this phase.
.. zuul:rolevar:: [filename]
:type: dict
The keys in this dictionary are the INI sections in this file.
.. zuul:rolevar:: [section]
:type: dict
This is a dictionary of key-value pairs which comprise
this section of the INI file.
.. zuul:rolevar:: devstack_services
:type: dict
A dictionary mapping service names to boolean values. If the
boolean value is ``false``, a ``disable_service`` line will be
emitted for the service name. If it is ``true``, then
``enable_service`` will be emitted. All other values are ignored.
.. zuul:rolevar:: devstack_plugins
:type: dict
A dictionary mapping a plugin name to a git repo location. If the
location is a non-empty string, then an ``enable_plugin`` line will
be emmitted for the plugin name.

View File

@ -0,0 +1,2 @@
devstack_base_dir: /opt/stack
devstack_local_conf_path: "{{ devstack_base_dir }}/devstack/local.conf"

View File

@ -0,0 +1,185 @@
# Copyright (C) 2017 Red Hat, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
#
# See the License for the specific language governing permissions and
# limitations under the License.
import re
class VarGraph(object):
# This is based on the JobGraph from Zuul.
def __init__(self, vars):
self.vars = {}
self._varnames = set()
self._dependencies = {} # dependent_var_name -> set(parent_var_names)
for k, v in vars.items():
self._varnames.add(k)
for k, v in vars.items():
self._addVar(k, str(v))
bash_var_re = re.compile(r'\$\{?(\w+)')
def getDependencies(self, value):
return self.bash_var_re.findall(value)
def _addVar(self, key, value):
if key in self.vars:
raise Exception("Variable {} already added".format(key))
self.vars[key] = value
# Append the dependency information
self._dependencies.setdefault(key, set())
try:
for dependency in self.getDependencies(value):
if dependency == key:
# A variable is allowed to reference itself; no
# dependency link needed in that case.
continue
if dependency not in self._varnames:
# It's not necessary to create a link for an
# external variable.
continue
# Make sure a circular dependency is never created
ancestor_vars = self._getParentVarNamesRecursively(
dependency, soft=True)
ancestor_vars.add(dependency)
if any((key == anc_var) for anc_var in ancestor_vars):
raise Exception("Dependency cycle detected in var {}".
format(key))
self._dependencies[key].add(dependency)
except Exception:
del self.vars[key]
del self._dependencies[key]
raise
def getVars(self):
ret = []
keys = sorted(self.vars.keys())
seen = set()
for key in keys:
dependencies = self.getDependentVarsRecursively(key)
for var in dependencies + [key]:
if var not in seen:
ret.append((var, self.vars[var]))
seen.add(var)
return ret
def getDependentVarsRecursively(self, parent_var):
dependent_vars = []
current_dependent_vars = self._dependencies[parent_var]
for current_var in current_dependent_vars:
if current_var not in dependent_vars:
dependent_vars.append(current_var)
for dep in self.getDependentVarsRecursively(current_var):
if dep not in dependent_vars:
dependent_vars.append(dep)
return dependent_vars
def _getParentVarNamesRecursively(self, dependent_var, soft=False):
all_parent_vars = set()
vars_to_iterate = set([dependent_var])
while len(vars_to_iterate) > 0:
current_var = vars_to_iterate.pop()
current_parent_vars = self._dependencies.get(current_var)
if current_parent_vars is None:
if soft:
current_parent_vars = set()
else:
raise Exception("Dependent var {} not found: ".format(
dependent_var))
new_parent_vars = current_parent_vars - all_parent_vars
vars_to_iterate |= new_parent_vars
all_parent_vars |= new_parent_vars
return all_parent_vars
class LocalConf(object):
def __init__(self, localrc, localconf, services, plugins):
self.localrc = []
self.meta_sections = {}
if plugins:
self.handle_plugins(plugins)
if services:
self.handle_services(services)
if localrc:
self.handle_localrc(localrc)
if localconf:
self.handle_localconf(localconf)
def handle_plugins(self, plugins):
for k, v in plugins.items():
if v:
self.localrc.append('enable_plugin {} {}'.format(k, v))
def handle_services(self, services):
for k, v in services.items():
if v is False:
self.localrc.append('disable_service {}'.format(k))
elif v is True:
self.localrc.append('enable_service {}'.format(k))
def handle_localrc(self, localrc):
vg = VarGraph(localrc)
for k, v in vg.getVars():
self.localrc.append('{}={}'.format(k, v))
def handle_localconf(self, localconf):
for phase, phase_data in localconf.items():
for fn, fn_data in phase_data.items():
ms_name = '[[{}|{}]]'.format(phase, fn)
ms_data = []
for section, section_data in fn_data.items():
ms_data.append('[{}]'.format(section))
for k, v in section_data.items():
ms_data.append('{} = {}'.format(k, v))
ms_data.append('')
self.meta_sections[ms_name] = ms_data
def write(self, path):
with open(path, 'w') as f:
f.write('[[local|localrc]]\n')
f.write('\n'.join(self.localrc))
f.write('\n\n')
for section, lines in self.meta_sections.items():
f.write('{}\n'.format(section))
f.write('\n'.join(lines))
def main():
module = AnsibleModule(
argument_spec=dict(
plugins=dict(type='dict'),
services=dict(type='dict'),
localrc=dict(type='dict'),
local_conf=dict(type='dict'),
path=dict(type='str'),
)
)
p = module.params
lc = LocalConf(p.get('localrc'),
p.get('local_conf'),
p.get('services'),
p.get('plugins'))
lc.write(p['path'])
module.exit_json()
from ansible.module_utils.basic import * # noqa
from ansible.module_utils.basic import AnsibleModule
if __name__ == '__main__':
main()

View File

@ -0,0 +1,9 @@
- name: Write a job-specific local_conf file
become: true
become_user: stack
devstack_local_conf:
path: "{{ devstack_local_conf_path }}"
plugins: "{{ devstack_plugins|default(omit) }}"
services: "{{ devstack_services|default(omit) }}"
localrc: "{{ devstack_localrc|default(omit) }}"
local_conf: "{{ devstack_local_conf|default(omit) }}"