add jjb fold

Change-Id: I145e88a2b835dc02f47e24c3514fe8b067772988
This commit is contained in:
Alexey Elagin 2015-12-11 18:37:05 +03:00
parent e1c97daa61
commit 4b8480f97f
6 changed files with 667 additions and 0 deletions

10
jjb/build-plugins.project.yaml Executable file
View File

@ -0,0 +1,10 @@
- project:
name: build
email_to: ''
tag: 'refs/tags/1.0.0'
plugin_name:
- testplugin:
plugin_repo: fuel-plugin-testplugin
jobs:
- 'fuel-plugin.{plugin_name}.build'

19
jjb/builders/build-plugin.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash
set -ex
find . -name '*.erb' -print0 | xargs -0 -P1 -L1 -I '%' erb -P -x -T '-' % | ruby -c
find . -name '*.pp' -print0 | xargs -0 -P1 -L1 puppet parser validate --verbose
find . -name '*.pp' -print0 | xargs -0 -r -P1 -L1 puppet-lint \
--fail-on-warnings \
--with-context \
--with-filename \
--no-80chars-check \
--no-variable_scope-check \
--no-nested_classes_or_defines-check \
--no-autoloader_layout-check \
--no-class_inherits_from_params_class-check \
--no-documentation-check \
--no-arrow_alignment-check
fpb --check ./
fpb --build ./

View File

@ -0,0 +1,11 @@
#!/bin/bash
set -ex
LOGS="${WORKSPACE}/logs/"
rm -rf "${LOGS}"
mkdir -p "${LOGS}"
wget --no-check-certificate "${REPORTED_JOB_URL}/consoleText" -O "${LOGS}/consoleText.txt"

View File

@ -0,0 +1,113 @@
#!/bin/bash -e
# activate bash xtrace for script
[[ "${DEBUG}" == "true" ]] && set -x || set +x
# for manually run of this job
[ -z $ISO_FILE ] && export ISO_FILE=${ISO_FILE}
#remove old logs and test data
rm -f nosetests.xml
rm -rf logs/*
export ISO_VERSION=$(cut -d'-' -f3-3<<< $ISO_FILE)
echo iso build number is $ISO_VERSION
export REQUIRED_FREE_SPACE=200
export ISO_PATH="${ISO_STORAGE}/${ISO_FILE}"
export FUEL_RELEASE=$(cut -d'-' -f2-2 <<< $ISO_FILE | tr -d '.')
export VENV_PATH="${HOME}/${FUEL_RELEASE}-venv"
echo iso-version: $ISO_VERSION
echo fuel-release: $FUEL_RELEASE
echo virtual-env: $VENV_PATH
## For plugins we should get a valid version of requrements of python-venv
## This requirements could be got from the github repo
## but for each branch of a plugin we should map specific branch of the fuel-qa repo
## the fuel-qa branch is determined by a fuel-iso name.
case "${FUEL_RELEASE}" in
*70* ) export REQS_BRANCH="stable/7.0" ;;
*61* ) export REQS_BRANCH="stable/6.1" ;;
* ) export REQS_BRANCH="master"
esac
REQS_PATH="https://raw.githubusercontent.com/openstack/fuel-qa/${REQS_BRANCH}/fuelweb_test/requirements.txt"
###############################################################################
## We have limited disk resources, so before run of system tests a lab
## may have many deployed and runned envs, those may cause errors during test
function delete_envs {
[ -z $VIRTUAL_ENV ] && exit 1
dos.py sync
env_list=$(dos.py list | tail -n +3)
if [[ ! -z "${env_list}" ]]; then
for env in $env_list; do dos.py erase $env; done
fi
}
## We have limited cpu resources, because we use two hypervisors with heavy VMs, so
## we should poweroff all unused envs, if there're exist.
function destroy_envs {
[ -z $VIRTUAL_ENV ] && exit 1
dos.py sync
env_list=$(dos.py list | tail -n +3)
if [[ ! -z "${env_list}" ]]; then
for env in $env_list; do dos.py destroy $env; done
fi
}
## Delete all systest envs except the env with the same version of a fuel-build
## if it exists. This behaviour is needed to use restoring from snapshots.
function delete_systest_envs {
[ -z $VIRTUAL_ENV ] && exit 1
dos.py sync
for env in $(dos.py list | tail -n +3 | grep $ENV_PREFIX); do
[[ $env == *"$ENV_NAME"* ]] && continue || dos.py erase $env
done
}
function prepare_venv {
#rm -rf "${VENV_PATH}"
[ ! -d $VENV_PATH ] && virtualenv "${VENV_PATH}" || echo "${VENV_PATH} already exist"
source "${VENV_PATH}/bin/activate"
pip --version
[ $? -ne 0 ] && easy_install -U pip
pip install -r "${REQS_PATH}" --upgrade > /dev/null 2>/dev/null
django-admin.py syncdb --settings=devops.settings --noinput
django-admin.py migrate devops --settings=devops.settings --noinput
deactivate
}
function fix_logger {
config_path="${HOME}/.devops/log.yaml"
echo devops config path $config_path
sed -i '/disable_existing_loggers.*/d' $config_path
echo disable_existing_loggers: False >> $config_path
}
####################################################################################
prepare_venv
fix_logger
# determine free space before run the cleaner
free_space_exist=false
free_space=$(df -h | grep '/$' | awk '{print $4}' | tr -d G)
(( $free_space > $REQUIRED_FREE_SPACE )) && export free_space_exist=true
# activate a python virtual env
source "$VENV_PATH/bin/activate"
# free space
[ $free_space_exist ] && delete_systest_envs || delete_envs
# poweroff all envs
destroy_envs

34
jjb/builders/testplugin.test.sh Executable file
View File

@ -0,0 +1,34 @@
#!/bin/bash -e
# activate bash xtrace for script
[[ "${DEBUG}" == "true" ]] && set -x || set +x
[ -z $PLUGIN_VERSION ] && exit 1 || echo testplugin version is $PLUGIN_VERSION
export ISO_PATH="${ISO_STORAGE}/${ISO_FILE}"
export ISO_VERSION=$(cut -d'-' -f3-3 <<< $ISO_FILE)
export FUEL_RELEASE=$(cut -d'-' -f2-2 <<< $ISO_FILE | tr -d '.')
export ENV_NAME="${ENV_PREFIX}.${ISO_VERSION}"
export VENV_PATH="${HOME}/${FUEL_RELEASE}-venv"
[[ -z ${PLUGIN_PATH} ]] && export PLUGIN_PATH=$(ls ${WORKSPACE}/testplugin*.rpm) \
|| echo PLUGIN_PATH=$PLUGIN_PATH
source $VENV_PATH/bin/activate
systest_parameters=''
[[ $USE_SNAPSHOTS == "true" ]] && systest_parameters+=' -k' || echo new env will be created
[[ $ERASE_AFTER == "true" ]] && echo the env will be erased after test || systest_parameters+=' -K'
echo test-group: $TEST_GROUP
echo env-name: $ENV_NAME
echo use-snapshots: $USE_SNAPSHOTS
echo fuel-release: $FUEL_RELEASE
echo venv-path: $VENV_PATH
echo env-name: $ENV_NAME
echo iso-path: $ISO_PATH
echo plugin-path: $PLUGIN_PATH
echo plugin-checksum: $(md5sum -b $PLUGIN_PATH)
./plugin_test/utils/jenkins/system_tests.sh -t test ${systest_parameters} -i ${ISO_PATH} -j ${JOB_NAME} -o --group=${TEST_GROUP}

480
jjb/install_plugin.yaml Executable file
View File

@ -0,0 +1,480 @@
## common git settings to get sources
- common-scm: &common-scm
name: 'common-scm'
scm:
- git:
name: ''
url: 'https://review.openstack.org/openstack/{gerrit-repo}'
refspec: $GERRIT_REFSPEC
branches:
- $GERRIT_BRANCH
choosing-strategy: gerrit
submodule:
disable: false
tracking: true
recursive: true
clean:
# we don't clean and re-initialize the repo
before: false
# we don't clean a workspace, so we need to remove rpms manually
wipe-workspace: false
## list of gerrit events to trigger build job
- build-gerrit-events: &build-gerrit-events
name: 'build-gerrit-events'
trigger-on:
- change-merged-event
- draft-published-event
- patchset-created-event:
exclude-trivial-rebase: true
exclude-no-code-change: true
## configuration of gerrit event for the smoke multijob
## smoke multijob should run on patchset, draft events
- smoke-gerrit-events: &smoke-gerrit-events
name: 'smoke-gerrit-events'
trigger-on:
- draft-published-event
- patchset-created-event:
exclude-trivial-rebase: true
exclude-no-code-change: true
## configuration of gerrit event for the bvt multijob
## bvt multijob should run only on merge event
- bvt-gerrit-events: &bvt-gerrit-events
name: 'bvt-gerrit-events'
trigger-on:
- change-merged-event
## the main part of gerrit section as yaml anchor
- generic-gerrit-projects: &generic-gerrit-projects
name: 'generic-gerrit-projects'
projects:
- project-compare-type: 'PLAIN'
project-pattern: 'openstack/{gerrit-repo}'
branches:
- branch-compare-type: 'ANT'
branch-pattern: '{gerrit-branch}'
forbidden-file-paths:
- compare-type: 'ANT'
pattern: 'docs/**'
- compare-type: 'ANT'
pattern: 'specs/**'
silent: false
override-votes: true
server-name: 'review.openstack.org'
custom-url: '* $JOB_NAME $BUILD_URL'
escape-quotes: true
readable-message: true
skip-vote:
successfull: false
failed: false
unstable: true
notbuilt: true
## properties for smoke, bvt and nightly multijobs
- runner-properties: &runner-properties
name: 'runner-properties'
properties:
- heavy-job:
weight: 1
- build-blocker:
use-build-blocker: true
blocking-jobs:
- '{build-name}'
block-level: 'GLOBAL'
queue-scanning: 'BUILDABLE'
## properties for test jobs
- test-properties: &test-properties
name: 'test-properties'
properties:
- heavy-job:
weight: 1
- throttle:
max-per-node: 0
max-total: 0
categories:
- testplugin
option: category
## parameters for smoke, bvt and nightly multijobs
- runner-parameters: &runner-parameters
name: 'runner-parameters'
parameters:
- bool:
name: DEBUG
default: true
description: "Set -x (xtrace) for jobs' bash scripts"
- bool:
name: UPDATE_MASTER
default: true
description: 'turns on update fuel master node to maintenance update'
- string:
name: MIRROR
default: 'http://mirror.seed-cz1.fuel-infra.org'
description: 'mirror for package repositories. this mirror is optimized for CZ'
- string:
name: UPDATE_FUEL_MIRROR
default: "${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/security/x86_64/ ${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/updates/x86_64/"
description: 'repositories to update fuel master node'
- string:
name: MIRROR_UBUNTU
default: 'deb ${{MIRROR}}/pkgs/ubuntu/ trusty main universe multiverse|deb ${{MIRROR}}/pkgs/ubuntu/ trusty-updates main universe multiverse'
description: 'proposed repositories to update ubuntu cluster'
- string:
name: DEB_UPDATES
default: "mos-updates,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-updates main restricted"
description: 'ubuntu-updates repositories for master ui'
- string:
name: DEB_SECURITY
default: "mos-security,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-security main restricted"
description: 'ubuntu-security repositories for master ui'
- string:
name: PLUGIN_VERSION
default: '1.0'
description: 'There is a version of plugin stored in common sotrage'
- string:
name: PLUGIN_PATH
default: ''
description: 'The path to the plugin package on storage'
- string:
name: GERRIT_REFSPEC
default: 'refs/heads/{gerrit-branch}'
description: 'Refspecs for commits in fuel-qa gerrit separated with spaces'
- string:
name: GERRIT_BRANCH
default: 'origin/{gerrit-branch}'
description: 'The branch for fuel-qa gerrit'
- string:
name: ENV_PREFIX
default: '{fuel-release}.{plugin-name}'
description: 'The name of devops env. Needed to properly work of existing mode of devops'
- bool:
name: BONDING
default: false
- bool:
name: USE_SNAPSHOTS
default: false
descrition: 'Will be used existing environment'
## parameters for jobs created per test-group and custom test job
- test-parameters: &test-parameters
name: 'test-parameters'
parameters:
- bool:
name: DEBUG
default: true
description: "Set -x (xtrace) for jobs' bash scripts"
- string:
name: TEST_GROUP
default: '{testgroup}'
- string:
name: ISO_FILE
default: '{iso-file}'
description: 'ISO file name that is on the tpi-s1 in /storage/downloads'
- string:
name: ISO_STORAGE
default: '/storage/downloads'
description: 'Storage for iso files'
- string:
name: ISO_VERSION
description: 'Contrainer for storing an iso build number to output it as job name'
- string:
name: PLUGIN_DISTRIBUTION
default: ''
- string:
name: PLUGIN_VERSION
default: '{plugin-version}'
description: 'There is a version of plugin packages stored in common storage'
- string:
name: PLUGIN_PATH
default: ''
- bool:
name: UPDATE_MASTER
default: true
description: 'turns on update fuel master node to maintenance update'
- string:
name: MIRROR
default: 'http://mirror.seed-cz1.fuel-infra.org'
description: 'mirror for package repositories. this mirror is optimized for CZ'
- string:
name: UPDATE_FUEL_MIRROR
default: "${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/security/x86_64/ ${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/updates/x86_64/"
description: 'repositories to update fuel master node'
- string:
name: MIRROR_UBUNTU
default: 'deb ${{MIRROR}}/pkgs/ubuntu/ trusty main universe multiverse|deb ${{MIRROR}}/pkgs/ubuntu/ trusty-updates main universe multiverse'
description: 'proposed repositories to update ubuntu cluster'
- string:
name: DEB_UPDATES
default: "mos-updates,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-updates main restricted"
description: 'ubuntu-updates repositories for master ui'
- string:
name: DEB_SECURITY
default: "mos-security,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-security main restricted"
description: 'ubuntu-security repositories for master ui'
- string:
name: OPENSTACK_RELEASE
default: 'Ubuntu'
description: 'Openstack release (CentOS, Ubuntu)'
- string:
name: GERRIT_REFSPEC
default: 'refs/heads/{gerrit-branch}'
description: 'Refspecs for commits in fuel-qa gerrit separated with spaces'
- string:
name: GERRIT_BRANCH
default: 'origin/{gerrit-branch}'
description: 'The branch for fuel-qa gerrit'
- string:
name: NODE_VOLUME_SIZE
default: '512'
- string:
name: NODES_COUNT
default: '10'
description: 'Amount of nodes in the test lab'
- string:
name: ADMIN_NODE_MEMORY
default: '4096'
description: 'Amount of vitrual RAM for admin node'
- string:
name: SLAVE_NODE_MEMORY
default: '4096'
description: 'Amount of vitrual RAM per slave node'
- string:
name: ADMIN_NODE_CPU
default: '4'
description: 'Amount of vitrual CPUs for admin node'
- string:
name: SLAVE_NODE_CPU
default: '4'
description: 'Amount of vitrual CPUs per slave node'
- string:
name: ENV_PREFIX
default: '{fuel-release}.{plugin-name}'
description: 'The name of devops env. Needed to properly work of existing mode of devops'
- bool:
name: BONDING
default: false
- bool:
name: USE_SNAPSHOTS
default: false
descrition: 'Will be used existing environment'
- project:
name: 'predefined_parameters'
fuel-release: '7.0'
plugin-version: '1.0'
plugin-name: 'testplugin'
build-name: '{fuel-release}.{plugin-name}.{plugin-version}.build'
smoke-name: '{fuel-release}.{plugin-name}.{plugin-version}.smoke'
bvt-name: '{fuel-release}.{plugin-name}.{plugin-version}.bvt'
regression-name: '{fuel-release}.{plugin-name}.{plugin-version}.regression'
nightly-name: '{fuel-release}.{plugin-name}.{plugin-version}.nightly'
custom-name: '{fuel-release}.{plugin-name}.{plugin-version}.custom'
nightly-timer: 'H 21 * * *'
regression-timer: 'H 19 * * *'
iso-file: 'fuel-7.0-301-2015-09-22_20-01-53.iso'
gerrit-repo: 'fuel-plugin-testplugin'
gerrit-branch: 'master'
email-to: 'devops@testcompany.com'
released-plugin-path: '/storage/testplugin/released/testplugin-1.0.noarch.rpm'
testgroup:
- testplugin_bvt
- testplugin_smoke
- install_testplugin
jobs:
- '{build-name}'
- '{custom-name}'
- '{smoke-name}'
- '{bvt-name}'
- defaults:
name: global
disabled: false
node: 'testplugin'
logrotate:
daysToKeep: 7
numToKeep: 10
artifactDaysToKeep: 7
artifactNumToKeep: 10
<<: *test-properties
<<: *test-parameters
<<: *common-scm
builders:
- copyartifact:
project: '{build-name}'
which-build: last-successful
- shell:
!include-raw-escape builders/testplugin.prepare.sh
- shell:
!include-raw-escape builders/testplugin.test.sh
wrappers:
- ansicolor:
colormap: xterm
- timeout:
fail: true
timeout: 240
publishers:
- postbuildscript:
builders:
- shell: env > properties
# need to delete packages, because we don't wipe workspace,
# but packages could be duplicated
- shell: rm -f .*rpm.*
script-only-if-succeeded: False
- archive:
artifacts: 'build.properties'
allow-empty: false
- archive:
artifacts: 'properties'
allow-empty: false
- archive:
artifacts: '**/nosetests.xml'
allow-empty: true
fingerprint: true
- archive:
artifacts: 'logs/*'
allow-empty: true
fingerprint: true
- xunit:
types:
- junit:
pattern: '**/nosetest.xml'
skip-if-no-test-files: true
- email:
recipients: '{email-to}'
# job for building plugin package
- job-template:
name: '{build-name}'
node: 'runner'
concurrent: true
disabled: false
description: |
'<a href=https://github.com/openstack/{gerrit-repo}>
Build {plugin-name} plugin from fuel-plugins project</a>'
<<: *common-scm
triggers:
- gerrit:
<<: *build-gerrit-events
<<: *generic-gerrit-projects
parameters:
- string:
name: 'GERRIT_REFSPEC'
default: 'refs/heads/{gerrit-branch}'
properties:
- heavy-job:
weight: 1
builders:
- shell:
!include-raw-escape './builders/build-plugin.sh'
- shell:
!include-raw-escape './builders/rpm-check.sh'
publishers:
- postbuildscript:
builders:
- shell: env > build.properties
script-only-if-succeeded: False
- archive:
artifacts: '*.rpm'
allow-empty: false
- archive:
artifacts: 'build.properties'
allow-empty: false
- email:
recipients: '{email-to}'
# jobs for system tests
- job-template:
name: '{custom-name}'
description: 'The custom test for {fuel-release}.{plugin-name}.{plugin-version}'
concurrent: true
- job-template:
name: '{smoke-name}'
disabled: false
description: 'The Smoke test for {fuel-release}.{plugin-name}.{plugin-version}'
concurrent: true
project-type: multijob
node: runner
<<: *runner-parameters
<<: *runner-properties
scm: []
triggers:
- gerrit:
<<: *smoke-gerrit-events
<<: *generic-gerrit-projects
builders:
- copyartifact:
project: '{build-name}'
which-build: last-successful
stable: true
- multijob:
name: 'Smoke tests for open contrail'
condition: SUCCESSFUL
projects:
- name: '{fuel-release}.{plugin-name}.{plugin-version}.testplugin_smoke'
current-parameters: true
kill-phase-on: NEVER
publishers:
- postbuildscript:
script-only-if-succeeded: False
builders:
- shell: env > smoke.properties
- archive:
artifacts: 'build.properties'
allow-empty: false
- archive:
artifacts: 'smoke.properties'
allow-empty: false
- email:
recipients: '{email-to}'
- job-template:
name: '{bvt-name}'
description: 'The Smoke test for {fuel-release}.{plugin-name}.{plugin-version}'
disabled: false
concurrent: true
project-type: multijob
node: runner
<<: *runner-parameters
<<: *runner-properties
scm: []
triggers:
- gerrit:
<<: *bvt-gerrit-events
<<: *generic-gerrit-projects
builders:
- copyartifact:
project: '{build-name}'
which-build: last-successful
stable: true
- multijob:
name: 'Smoke tests for testplugin'
condition: SUCCESSFUL
projects:
- name: '{fuel-release}.{plugin-name}.{plugin-version}.testplugin'
current-parameters: true
kill-phase-on: NEVER
publishers:
- postbuildscript:
script-only-if-succeeded: False
builders:
- shell: env > bvt.properties
- archive:
artifacts: '*.rpm'
allow-empty: false
- archive:
artifacts: 'build.properties'
allow-empty: false
- archive:
artifacts: 'bvt.properties'
allow-empty: false
- email:
recipients: '{email-to}'