From cd5365db2fcdd8281866d1578e649dadf959533d Mon Sep 17 00:00:00 2001 From: Gorka Eguileor Date: Thu, 21 Feb 2019 16:36:04 +0100 Subject: [PATCH] Add zuul jobs This patch adds the following Zuul CI jobs: - Python 2.7 unit tests - Python 3.6 unit tests - Functional tests with LVM - Functional tests with Ceph - Publis documentation - Release notes Change-Id: I2f3f34c1db4716b323c48908f51500898c2c6242 --- .gitignore | 3 + .stestr.conf | 3 + .zuul.yaml | 40 ++++++++++ cinderlib/tests/functional/ceph.yaml | 11 +++ .../tests/functional}/lvm.yaml | 0 doc/source/_extra/.placeholder | 0 doc/source/_static/.placeholder | 1 + playbooks/setup-ceph.yaml | 76 +++++++++++++++++++ playbooks/setup-lvm.yaml | 75 ++++++++++++++++++ requirements_dev.txt | 14 ---- setup.cfg | 2 +- test-requirements.txt | 1 + tools/cinder-cfg-to-python.py | 14 ++++ tox.ini | 21 +++-- 14 files changed, 238 insertions(+), 23 deletions(-) create mode 100644 .stestr.conf create mode 100644 .zuul.yaml create mode 100644 cinderlib/tests/functional/ceph.yaml rename {tools => cinderlib/tests/functional}/lvm.yaml (100%) create mode 100644 doc/source/_extra/.placeholder create mode 100644 doc/source/_static/.placeholder create mode 100644 playbooks/setup-ceph.yaml create mode 100644 playbooks/setup-lvm.yaml delete mode 100644 requirements_dev.txt diff --git a/.gitignore b/.gitignore index fba0c04..fa31ea1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # Byte-compiled / optimized / DLL files .* !.gitignore +!.testr.conf +!.stestr.conf +!.zuul.yaml !.travis.yml .*.sw? __pycache__/ diff --git a/.stestr.conf b/.stestr.conf new file mode 100644 index 0000000..7473632 --- /dev/null +++ b/.stestr.conf @@ -0,0 +1,3 @@ +[DEFAULT] +test_path=${OS_TEST_PATH:-./cinderlib/tests/unit} +top_dir=./ diff --git a/.zuul.yaml b/.zuul.yaml new file mode 100644 index 0000000..35bbdec --- /dev/null +++ b/.zuul.yaml @@ -0,0 +1,40 @@ +- project: + templates: + # TODO(geguileo): Uncomment publish-to-pypi once we release Cinder on + # PyPi and we can add it to test-requirements instead of the git repo. + # - publish-to-pypi + # PEP8 + PY27 + Post branch-tarball + - openstack-python-jobs + - openstack-python36-jobs + - publish-openstack-docs-pti + - release-notes-jobs-python3 + check: + queue: integrated + jobs: + - cinderlib-lvm-functional + - cinderlib-ceph-functional + gate: + queue: integrated + jobs: + - cinderlib-lvm-functional + - cinderlib-ceph-functional + +- job: + name: cinderlib-lvm-functional + parent: openstack-tox-functional-with-sudo + pre-run: playbooks/setup-lvm.yaml + nodeset: centos-7 + +- job: + name: cinderlib-ceph-functional + parent: openstack-tox-functional-with-sudo + pre-run: playbooks/setup-ceph.yaml + nodeset: centos-7 + vars: + tox_environment: + CL_FTEST_CFG: "cinderlib/tests/functional/ceph.yaml" + CL_FTEST_ROOT_HELPER: sudo + # These come from great-great-grandparent tox job + NOSE_WITH_HTML_OUTPUT: 1 + NOSE_HTML_OUT_FILE: nose_results.html + NOSE_WITH_XUNIT: 1 diff --git a/cinderlib/tests/functional/ceph.yaml b/cinderlib/tests/functional/ceph.yaml new file mode 100644 index 0000000..d47c36f --- /dev/null +++ b/cinderlib/tests/functional/ceph.yaml @@ -0,0 +1,11 @@ +# Logs are way too verbose, so we disable them +logs: false + +# We only define one backend +backends: + - volume_backend_name: ceph + volume_driver: cinder.volume.drivers.rbd.RBDDriver + rbd_user: admin + rbd_pool: rbd + rbd_ceph_conf: /etc/ceph/ceph.conf + rbd_keyring_conf: /etc/ceph/ceph.client.admin.keyring diff --git a/tools/lvm.yaml b/cinderlib/tests/functional/lvm.yaml similarity index 100% rename from tools/lvm.yaml rename to cinderlib/tests/functional/lvm.yaml diff --git a/doc/source/_extra/.placeholder b/doc/source/_extra/.placeholder new file mode 100644 index 0000000..e69de29 diff --git a/doc/source/_static/.placeholder b/doc/source/_static/.placeholder new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/doc/source/_static/.placeholder @@ -0,0 +1 @@ + diff --git a/playbooks/setup-ceph.yaml b/playbooks/setup-ceph.yaml new file mode 100644 index 0000000..46db2b0 --- /dev/null +++ b/playbooks/setup-ceph.yaml @@ -0,0 +1,76 @@ +# Copyright (c) 2018, Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +--- +#------------------------------------------------------------------------------ +# Setup an Ceph cluster that will be used by cinderlib's functional tests +#------------------------------------------------------------------------------ +- hosts: all + vars: + ansible_become: yes + tasks: + # Tox job has in its pre.yaml the ensure-tox role, which installs tox + # from pip, which brings six from pip. This conflicts with ceph-common's + # python-six dependency. + - name: Remove pip's six from the system + pip: + name: six + state: absent + + - name: Install ceph-common and epel-release + yum: + name: ['epel-release', 'ceph-common'] + state: present + + - name: Install Docker from epel + yum: + name: 'docker' + state: present + + - name: Start Docker + service: + name: docker + state: started + + - name: Start Ceph demo + command: | + docker run -d + --name ceph-demo + -e MON_IP=127.0.0.1 + -e CEPH_PUBLIC_NETWORK=127.0.0.1/0 + -e DEMO_DAEMONS="osd mds" + --net=host + --volume /etc/ceph:/etc/ceph + --privileged + ceph/daemon:latest-luminous + demo + + - name: Wait for ceph.conf + wait_for: + path: /etc/ceph/ceph.conf + search_regex: '[global]' + + - name: Set ceph features in config + lineinfile: + path: /etc/ceph/ceph.conf + insertafter: '[global]' + line: 'rbd default features = 3' + state: present + + - name: Set ceph keyring mode + file: + path: /etc/ceph/ceph.client.admin.keyring + mode: 0644 diff --git a/playbooks/setup-lvm.yaml b/playbooks/setup-lvm.yaml new file mode 100644 index 0000000..7264902 --- /dev/null +++ b/playbooks/setup-lvm.yaml @@ -0,0 +1,75 @@ +# Copyright (c) 2018, Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +--- +#------------------------------------------------------------------------------ +# Setup an LVM VG that will be used by cinderlib's functional tests +#------------------------------------------------------------------------------ +- hosts: all + vars: + cldir: . + vg: cinder-volumes + ansible_become: yes + tasks: + - name: Install LVM package + package: + name: lvm2 + state: present + + - name: Start LVM metadata + service: + name: lvm2-lvmetad + state: started + + - name: Create LVM backing file + command: "truncate -s 10G {{vg}}" + args: + creates: "{{cldir}}/{{vg}}" + + - name: Check if VG already exists + shell: "losetup -l | awk '/{{vg}}/ {print $1}'" + changed_when: false + register: existing_loop_device + + - name: "Create loopback device {{vg}}" + command: "losetup --show -f {{cldir}}/{{vg}}" + register: new_loop_device + when: existing_loop_device.stdout == '' + # Workaround because Ansible destroys registers when skipped + - set_fact: loop_device="{{ new_loop_device.stdout if new_loop_device.changed else existing_loop_device.stdout }}" + + - name: "Create VG {{vg}}" + shell: "vgcreate {{vg}} {{loop_device}} && touch {{cldir}}/lvm.vgcreate" + args: + creates: "{{cldir}}/lvm.vgcreate" + + - command: "vgscan --cache" + changed_when: false + + - name: Install iSCSI package + package: + name: iscsi-initiator-utils + state: present + + - name: Create initiator name + shell: echo InitiatorName=`iscsi-iname` > /etc/iscsi/initiatorname.iscsi + args: + creates: /etc/iscsi/initiatorname.iscsi + + - name: Start iSCSI initiator + service: + name: iscsid + state: started diff --git a/requirements_dev.txt b/requirements_dev.txt deleted file mode 100644 index 7ad7c08..0000000 --- a/requirements_dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -ddt>=1.0.1 -unittest2 -pyyaml -pip==8.1.2 -bumpversion==0.5.3 -wheel==0.31.1 -watchdog==0.8.3 -flake8==2.6.0 -tox==2.3.1 -coverage==4.1 -Sphinx==1.6.5 -mock==2.0.0 -readme_renderer==21.0 -git+https://github.com/openstack/cinder.git diff --git a/setup.cfg b/setup.cfg index 5fc94c2..0db80dc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ universal = 1 [flake8] -exclude = docs +exclude = .git,.venv,.tox,dist,doc,*egg,build [metadata] name = cinderlib diff --git a/test-requirements.txt b/test-requirements.txt index bb2e461..0119a5f 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -2,6 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. +pbr!=2.1.0,>=2.0.0 # Apache-2.0 hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0 coverage!=4.4,>=4.0 # Apache-2.0 ddt>=1.0.1 # MIT diff --git a/tools/cinder-cfg-to-python.py b/tools/cinder-cfg-to-python.py index 145d861..9460b80 100755 --- a/tools/cinder-cfg-to-python.py +++ b/tools/cinder-cfg-to-python.py @@ -1,4 +1,18 @@ #!/bin/env python +# Copyright (c) 2017, Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. """Generate Python code to initialize cinderlib based on Cinder config file This tool generates Python code to instantiate backends using a cinder.conf diff --git a/tox.ini b/tox.ini index 94f7e06..ddba9ef 100644 --- a/tox.ini +++ b/tox.ini @@ -1,15 +1,15 @@ [tox] minversion = 2.0 -envlist = py27, py36, flake8 +envlist = py27, py36, pep8 skipsdist = True setenv = VIRTUAL_ENV={envdir} -usedevelop=True [testenv] setenv = OS_STDOUT_CAPTURE=1 OS_STDERR_CAPTURE=1 OS_TEST_TIMEOUT=60 OS_TEST_PATH=./cinderlib/tests/unit +usedevelop=True install_command = pip install {opts} {packages} deps= -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} -r{toxinidir}/test-requirements.txt @@ -25,10 +25,13 @@ whitelist_externals = passenv = *_proxy *_PROXY [testenv:functional] +usedevelop=True basepython=python2.7 setenv = OS_TEST_PATH=./cinderlib/tests/functional - CL_FTEST_CFG={env:CL_FTEST_CFG:{toxinidir}/tools/lvm.yaml} + CL_FTEST_CFG={toxinidir}/{env:CL_FTEST_CFG:cinderlib/tests/functional/lvm.yaml} CL_FTEST_ROOT_HELPER={env:CL_FTEST_ROOT_HELPER:{toxinidir}/tools/virtualenv-sudo.sh} + ADD_PATHS=/usr/local/sbin:/usr/sbin + sitepackages = True # Not reusing py27's env due to https://github.com/tox-dev/tox/issues/477 # envdir = {toxworkdir}/py27 @@ -36,18 +39,20 @@ sitepackages = True # Must run serially or test_stats_with_creation may fail occasionally commands = find . -ignore_readdir_race -type f -name "*.pyc" -delete - stestr run --serial {posargs} + # Tox has a bug and it ignores the PATH set in setenv, so we work around it + bash -i -c "PATH=$PATH:$ADD_PATHS stestr run --serial {posargs}" stestr slowest whitelist_externals = bash find -[testenv:functional-py35] +[testenv:functional-py36] +usedevelop=True setenv = {[testenv:functional]setenv} sitepackages = True -basepython=python3.5 +basepython=python3.6 # Not reusing py35's env due to https://github.com/tox-dev/tox/issues/477 # envdir = {toxworkdir}/py35 commands = {[testenv:functional]commands} @@ -95,9 +100,9 @@ commands = coverage html -d cover coverage xml -o cover/coverage.xml -[testenv:flake8] +[testenv:pep8] basepython=python3 -commands=flake8 cinderlib +commands=flake8 {posargs} . deps= flake8 -r{toxinidir}/test-requirements.txt