diff --git a/jjb/build-plugins.project.yaml b/jjb/build-plugins.project.yaml new file mode 100755 index 0000000..3ee80ab --- /dev/null +++ b/jjb/build-plugins.project.yaml @@ -0,0 +1,10 @@ +- project: + name: build + email_to: '' + tag: 'refs/tags/1.0.0' + plugin_name: + - testplugin: + plugin_repo: fuel-plugin-testplugin + jobs: + - 'fuel-plugin.{plugin_name}.build' + diff --git a/jjb/builders/build-plugin.sh b/jjb/builders/build-plugin.sh new file mode 100755 index 0000000..0d696be --- /dev/null +++ b/jjb/builders/build-plugin.sh @@ -0,0 +1,19 @@ +#!/bin/bash +set -ex + +find . -name '*.erb' -print0 | xargs -0 -P1 -L1 -I '%' erb -P -x -T '-' % | ruby -c +find . -name '*.pp' -print0 | xargs -0 -P1 -L1 puppet parser validate --verbose +find . -name '*.pp' -print0 | xargs -0 -r -P1 -L1 puppet-lint \ + --fail-on-warnings \ + --with-context \ + --with-filename \ + --no-80chars-check \ + --no-variable_scope-check \ + --no-nested_classes_or_defines-check \ + --no-autoloader_layout-check \ + --no-class_inherits_from_params_class-check \ + --no-documentation-check \ + --no-arrow_alignment-check + +fpb --check ./ +fpb --build ./ diff --git a/jjb/builders/publish_build_to_external.sh b/jjb/builders/publish_build_to_external.sh new file mode 100755 index 0000000..71fcb5a --- /dev/null +++ b/jjb/builders/publish_build_to_external.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +LOGS="${WORKSPACE}/logs/" + +rm -rf "${LOGS}" + +mkdir -p "${LOGS}" + +wget --no-check-certificate "${REPORTED_JOB_URL}/consoleText" -O "${LOGS}/consoleText.txt" diff --git a/jjb/builders/testplugin.prepare.sh b/jjb/builders/testplugin.prepare.sh new file mode 100755 index 0000000..4183bfc --- /dev/null +++ b/jjb/builders/testplugin.prepare.sh @@ -0,0 +1,113 @@ +#!/bin/bash -e + +# activate bash xtrace for script +[[ "${DEBUG}" == "true" ]] && set -x || set +x + +# for manually run of this job +[ -z $ISO_FILE ] && export ISO_FILE=${ISO_FILE} + +#remove old logs and test data +rm -f nosetests.xml +rm -rf logs/* + +export ISO_VERSION=$(cut -d'-' -f3-3<<< $ISO_FILE) +echo iso build number is $ISO_VERSION +export REQUIRED_FREE_SPACE=200 +export ISO_PATH="${ISO_STORAGE}/${ISO_FILE}" +export FUEL_RELEASE=$(cut -d'-' -f2-2 <<< $ISO_FILE | tr -d '.') +export VENV_PATH="${HOME}/${FUEL_RELEASE}-venv" + +echo iso-version: $ISO_VERSION +echo fuel-release: $FUEL_RELEASE +echo virtual-env: $VENV_PATH + +## For plugins we should get a valid version of requrements of python-venv +## This requirements could be got from the github repo +## but for each branch of a plugin we should map specific branch of the fuel-qa repo +## the fuel-qa branch is determined by a fuel-iso name. + +case "${FUEL_RELEASE}" in + *70* ) export REQS_BRANCH="stable/7.0" ;; + *61* ) export REQS_BRANCH="stable/6.1" ;; + * ) export REQS_BRANCH="master" +esac + +REQS_PATH="https://raw.githubusercontent.com/openstack/fuel-qa/${REQS_BRANCH}/fuelweb_test/requirements.txt" + +############################################################################### + +## We have limited disk resources, so before run of system tests a lab +## may have many deployed and runned envs, those may cause errors during test + +function delete_envs { + [ -z $VIRTUAL_ENV ] && exit 1 + dos.py sync + env_list=$(dos.py list | tail -n +3) + if [[ ! -z "${env_list}" ]]; then + for env in $env_list; do dos.py erase $env; done + fi +} + +## We have limited cpu resources, because we use two hypervisors with heavy VMs, so +## we should poweroff all unused envs, if there're exist. + +function destroy_envs { + [ -z $VIRTUAL_ENV ] && exit 1 + dos.py sync + env_list=$(dos.py list | tail -n +3) + if [[ ! -z "${env_list}" ]]; then + for env in $env_list; do dos.py destroy $env; done + fi +} + +## Delete all systest envs except the env with the same version of a fuel-build +## if it exists. This behaviour is needed to use restoring from snapshots. + +function delete_systest_envs { + [ -z $VIRTUAL_ENV ] && exit 1 + dos.py sync + for env in $(dos.py list | tail -n +3 | grep $ENV_PREFIX); do + [[ $env == *"$ENV_NAME"* ]] && continue || dos.py erase $env + done +} + +function prepare_venv { + #rm -rf "${VENV_PATH}" + [ ! -d $VENV_PATH ] && virtualenv "${VENV_PATH}" || echo "${VENV_PATH} already exist" + source "${VENV_PATH}/bin/activate" + pip --version + [ $? -ne 0 ] && easy_install -U pip + pip install -r "${REQS_PATH}" --upgrade > /dev/null 2>/dev/null + django-admin.py syncdb --settings=devops.settings --noinput + django-admin.py migrate devops --settings=devops.settings --noinput + deactivate +} + +function fix_logger { + config_path="${HOME}/.devops/log.yaml" + echo devops config path $config_path + sed -i '/disable_existing_loggers.*/d' $config_path + echo disable_existing_loggers: False >> $config_path +} + + +#################################################################################### + +prepare_venv +fix_logger + +# determine free space before run the cleaner +free_space_exist=false +free_space=$(df -h | grep '/$' | awk '{print $4}' | tr -d G) + +(( $free_space > $REQUIRED_FREE_SPACE )) && export free_space_exist=true + +# activate a python virtual env +source "$VENV_PATH/bin/activate" + +# free space +[ $free_space_exist ] && delete_systest_envs || delete_envs + +# poweroff all envs +destroy_envs + diff --git a/jjb/builders/testplugin.test.sh b/jjb/builders/testplugin.test.sh new file mode 100755 index 0000000..ce9b53b --- /dev/null +++ b/jjb/builders/testplugin.test.sh @@ -0,0 +1,34 @@ +#!/bin/bash -e + +# activate bash xtrace for script +[[ "${DEBUG}" == "true" ]] && set -x || set +x + +[ -z $PLUGIN_VERSION ] && exit 1 || echo testplugin version is $PLUGIN_VERSION + +export ISO_PATH="${ISO_STORAGE}/${ISO_FILE}" +export ISO_VERSION=$(cut -d'-' -f3-3 <<< $ISO_FILE) +export FUEL_RELEASE=$(cut -d'-' -f2-2 <<< $ISO_FILE | tr -d '.') + +export ENV_NAME="${ENV_PREFIX}.${ISO_VERSION}" +export VENV_PATH="${HOME}/${FUEL_RELEASE}-venv" + +[[ -z ${PLUGIN_PATH} ]] && export PLUGIN_PATH=$(ls ${WORKSPACE}/testplugin*.rpm) \ + || echo PLUGIN_PATH=$PLUGIN_PATH + +source $VENV_PATH/bin/activate + +systest_parameters='' +[[ $USE_SNAPSHOTS == "true" ]] && systest_parameters+=' -k' || echo new env will be created +[[ $ERASE_AFTER == "true" ]] && echo the env will be erased after test || systest_parameters+=' -K' + +echo test-group: $TEST_GROUP +echo env-name: $ENV_NAME +echo use-snapshots: $USE_SNAPSHOTS +echo fuel-release: $FUEL_RELEASE +echo venv-path: $VENV_PATH +echo env-name: $ENV_NAME +echo iso-path: $ISO_PATH +echo plugin-path: $PLUGIN_PATH +echo plugin-checksum: $(md5sum -b $PLUGIN_PATH) + +./plugin_test/utils/jenkins/system_tests.sh -t test ${systest_parameters} -i ${ISO_PATH} -j ${JOB_NAME} -o --group=${TEST_GROUP} diff --git a/jjb/install_plugin.yaml b/jjb/install_plugin.yaml new file mode 100755 index 0000000..445cdb4 --- /dev/null +++ b/jjb/install_plugin.yaml @@ -0,0 +1,480 @@ +## common git settings to get sources +- common-scm: &common-scm + name: 'common-scm' + scm: + - git: + name: '' + url: 'https://review.openstack.org/openstack/{gerrit-repo}' + refspec: $GERRIT_REFSPEC + branches: + - $GERRIT_BRANCH + choosing-strategy: gerrit + submodule: + disable: false + tracking: true + recursive: true + clean: + # we don't clean and re-initialize the repo + before: false + # we don't clean a workspace, so we need to remove rpms manually + wipe-workspace: false + +## list of gerrit events to trigger build job +- build-gerrit-events: &build-gerrit-events + name: 'build-gerrit-events' + trigger-on: + - change-merged-event + - draft-published-event + - patchset-created-event: + exclude-trivial-rebase: true + exclude-no-code-change: true + +## configuration of gerrit event for the smoke multijob +## smoke multijob should run on patchset, draft events +- smoke-gerrit-events: &smoke-gerrit-events + name: 'smoke-gerrit-events' + trigger-on: + - draft-published-event + - patchset-created-event: + exclude-trivial-rebase: true + exclude-no-code-change: true + +## configuration of gerrit event for the bvt multijob +## bvt multijob should run only on merge event +- bvt-gerrit-events: &bvt-gerrit-events + name: 'bvt-gerrit-events' + trigger-on: + - change-merged-event + +## the main part of gerrit section as yaml anchor +- generic-gerrit-projects: &generic-gerrit-projects + name: 'generic-gerrit-projects' + projects: + - project-compare-type: 'PLAIN' + project-pattern: 'openstack/{gerrit-repo}' + branches: + - branch-compare-type: 'ANT' + branch-pattern: '{gerrit-branch}' + forbidden-file-paths: + - compare-type: 'ANT' + pattern: 'docs/**' + - compare-type: 'ANT' + pattern: 'specs/**' + silent: false + override-votes: true + server-name: 'review.openstack.org' + custom-url: '* $JOB_NAME $BUILD_URL' + escape-quotes: true + readable-message: true + skip-vote: + successfull: false + failed: false + unstable: true + notbuilt: true + +## properties for smoke, bvt and nightly multijobs +- runner-properties: &runner-properties + name: 'runner-properties' + properties: + - heavy-job: + weight: 1 + - build-blocker: + use-build-blocker: true + blocking-jobs: + - '{build-name}' + block-level: 'GLOBAL' + queue-scanning: 'BUILDABLE' + +## properties for test jobs +- test-properties: &test-properties + name: 'test-properties' + properties: + - heavy-job: + weight: 1 + - throttle: + max-per-node: 0 + max-total: 0 + categories: + - testplugin + option: category + +## parameters for smoke, bvt and nightly multijobs +- runner-parameters: &runner-parameters + name: 'runner-parameters' + parameters: + - bool: + name: DEBUG + default: true + description: "Set -x (xtrace) for jobs' bash scripts" + - bool: + name: UPDATE_MASTER + default: true + description: 'turns on update fuel master node to maintenance update' + - string: + name: MIRROR + default: 'http://mirror.seed-cz1.fuel-infra.org' + description: 'mirror for package repositories. this mirror is optimized for CZ' + - string: + name: UPDATE_FUEL_MIRROR + default: "${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/security/x86_64/ ${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/updates/x86_64/" + description: 'repositories to update fuel master node' + - string: + name: MIRROR_UBUNTU + default: 'deb ${{MIRROR}}/pkgs/ubuntu/ trusty main universe multiverse|deb ${{MIRROR}}/pkgs/ubuntu/ trusty-updates main universe multiverse' + description: 'proposed repositories to update ubuntu cluster' + - string: + name: DEB_UPDATES + default: "mos-updates,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-updates main restricted" + description: 'ubuntu-updates repositories for master ui' + - string: + name: DEB_SECURITY + default: "mos-security,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-security main restricted" + description: 'ubuntu-security repositories for master ui' + - string: + name: PLUGIN_VERSION + default: '1.0' + description: 'There is a version of plugin stored in common sotrage' + - string: + name: PLUGIN_PATH + default: '' + description: 'The path to the plugin package on storage' + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{gerrit-branch}' + description: 'Refspecs for commits in fuel-qa gerrit separated with spaces' + - string: + name: GERRIT_BRANCH + default: 'origin/{gerrit-branch}' + description: 'The branch for fuel-qa gerrit' + - string: + name: ENV_PREFIX + default: '{fuel-release}.{plugin-name}' + description: 'The name of devops env. Needed to properly work of existing mode of devops' + - bool: + name: BONDING + default: false + - bool: + name: USE_SNAPSHOTS + default: false + descrition: 'Will be used existing environment' + +## parameters for jobs created per test-group and custom test job +- test-parameters: &test-parameters + name: 'test-parameters' + parameters: + - bool: + name: DEBUG + default: true + description: "Set -x (xtrace) for jobs' bash scripts" + - string: + name: TEST_GROUP + default: '{testgroup}' + - string: + name: ISO_FILE + default: '{iso-file}' + description: 'ISO file name that is on the tpi-s1 in /storage/downloads' + - string: + name: ISO_STORAGE + default: '/storage/downloads' + description: 'Storage for iso files' + - string: + name: ISO_VERSION + description: 'Contrainer for storing an iso build number to output it as job name' + - string: + name: PLUGIN_DISTRIBUTION + default: '' + - string: + name: PLUGIN_VERSION + default: '{plugin-version}' + description: 'There is a version of plugin packages stored in common storage' + - string: + name: PLUGIN_PATH + default: '' + - bool: + name: UPDATE_MASTER + default: true + description: 'turns on update fuel master node to maintenance update' + - string: + name: MIRROR + default: 'http://mirror.seed-cz1.fuel-infra.org' + description: 'mirror for package repositories. this mirror is optimized for CZ' + - string: + name: UPDATE_FUEL_MIRROR + default: "${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/security/x86_64/ ${{MIRROR}}/mos-repos/centos/mos7.0-centos6-fuel/updates/x86_64/" + description: 'repositories to update fuel master node' + - string: + name: MIRROR_UBUNTU + default: 'deb ${{MIRROR}}/pkgs/ubuntu/ trusty main universe multiverse|deb ${{MIRROR}}/pkgs/ubuntu/ trusty-updates main universe multiverse' + description: 'proposed repositories to update ubuntu cluster' + - string: + name: DEB_UPDATES + default: "mos-updates,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-updates main restricted" + description: 'ubuntu-updates repositories for master ui' + - string: + name: DEB_SECURITY + default: "mos-security,deb ${{MIRROR}}/mos/ubuntu/dists/mos7.0-security main restricted" + description: 'ubuntu-security repositories for master ui' + - string: + name: OPENSTACK_RELEASE + default: 'Ubuntu' + description: 'Openstack release (CentOS, Ubuntu)' + - string: + name: GERRIT_REFSPEC + default: 'refs/heads/{gerrit-branch}' + description: 'Refspecs for commits in fuel-qa gerrit separated with spaces' + - string: + name: GERRIT_BRANCH + default: 'origin/{gerrit-branch}' + description: 'The branch for fuel-qa gerrit' + - string: + name: NODE_VOLUME_SIZE + default: '512' + - string: + name: NODES_COUNT + default: '10' + description: 'Amount of nodes in the test lab' + - string: + name: ADMIN_NODE_MEMORY + default: '4096' + description: 'Amount of vitrual RAM for admin node' + - string: + name: SLAVE_NODE_MEMORY + default: '4096' + description: 'Amount of vitrual RAM per slave node' + - string: + name: ADMIN_NODE_CPU + default: '4' + description: 'Amount of vitrual CPUs for admin node' + - string: + name: SLAVE_NODE_CPU + default: '4' + description: 'Amount of vitrual CPUs per slave node' + - string: + name: ENV_PREFIX + default: '{fuel-release}.{plugin-name}' + description: 'The name of devops env. Needed to properly work of existing mode of devops' + - bool: + name: BONDING + default: false + - bool: + name: USE_SNAPSHOTS + default: false + descrition: 'Will be used existing environment' + +- project: + name: 'predefined_parameters' + fuel-release: '7.0' + plugin-version: '1.0' + plugin-name: 'testplugin' + + build-name: '{fuel-release}.{plugin-name}.{plugin-version}.build' + smoke-name: '{fuel-release}.{plugin-name}.{plugin-version}.smoke' + bvt-name: '{fuel-release}.{plugin-name}.{plugin-version}.bvt' + regression-name: '{fuel-release}.{plugin-name}.{plugin-version}.regression' + nightly-name: '{fuel-release}.{plugin-name}.{plugin-version}.nightly' + custom-name: '{fuel-release}.{plugin-name}.{plugin-version}.custom' + + nightly-timer: 'H 21 * * *' + regression-timer: 'H 19 * * *' + + iso-file: 'fuel-7.0-301-2015-09-22_20-01-53.iso' + gerrit-repo: 'fuel-plugin-testplugin' + gerrit-branch: 'master' + email-to: 'devops@testcompany.com' + released-plugin-path: '/storage/testplugin/released/testplugin-1.0.noarch.rpm' + + testgroup: + - testplugin_bvt + - testplugin_smoke + - install_testplugin + + jobs: + - '{build-name}' + - '{custom-name}' + - '{smoke-name}' + - '{bvt-name}' + +- defaults: + name: global + disabled: false + node: 'testplugin' + logrotate: + daysToKeep: 7 + numToKeep: 10 + artifactDaysToKeep: 7 + artifactNumToKeep: 10 + <<: *test-properties + <<: *test-parameters + <<: *common-scm + builders: + - copyartifact: + project: '{build-name}' + which-build: last-successful + - shell: + !include-raw-escape builders/testplugin.prepare.sh + - shell: + !include-raw-escape builders/testplugin.test.sh + wrappers: + - ansicolor: + colormap: xterm + - timeout: + fail: true + timeout: 240 + publishers: + - postbuildscript: + builders: + - shell: env > properties + # need to delete packages, because we don't wipe workspace, + # but packages could be duplicated + - shell: rm -f .*rpm.* + script-only-if-succeeded: False + - archive: + artifacts: 'build.properties' + allow-empty: false + - archive: + artifacts: 'properties' + allow-empty: false + - archive: + artifacts: '**/nosetests.xml' + allow-empty: true + fingerprint: true + - archive: + artifacts: 'logs/*' + allow-empty: true + fingerprint: true + - xunit: + types: + - junit: + pattern: '**/nosetest.xml' + skip-if-no-test-files: true + - email: + recipients: '{email-to}' + +# job for building plugin package +- job-template: + name: '{build-name}' + node: 'runner' + concurrent: true + disabled: false + description: | + ' + Build {plugin-name} plugin from fuel-plugins project' + <<: *common-scm + triggers: + - gerrit: + <<: *build-gerrit-events + <<: *generic-gerrit-projects + parameters: + - string: + name: 'GERRIT_REFSPEC' + default: 'refs/heads/{gerrit-branch}' + properties: + - heavy-job: + weight: 1 + builders: + - shell: + !include-raw-escape './builders/build-plugin.sh' + - shell: + !include-raw-escape './builders/rpm-check.sh' + publishers: + - postbuildscript: + builders: + - shell: env > build.properties + script-only-if-succeeded: False + - archive: + artifacts: '*.rpm' + allow-empty: false + - archive: + artifacts: 'build.properties' + allow-empty: false + - email: + recipients: '{email-to}' + +# jobs for system tests +- job-template: + name: '{custom-name}' + description: 'The custom test for {fuel-release}.{plugin-name}.{plugin-version}' + concurrent: true + +- job-template: + name: '{smoke-name}' + disabled: false + description: 'The Smoke test for {fuel-release}.{plugin-name}.{plugin-version}' + concurrent: true + project-type: multijob + node: runner + <<: *runner-parameters + <<: *runner-properties + scm: [] + triggers: + - gerrit: + <<: *smoke-gerrit-events + <<: *generic-gerrit-projects + builders: + - copyartifact: + project: '{build-name}' + which-build: last-successful + stable: true + - multijob: + name: 'Smoke tests for open contrail' + condition: SUCCESSFUL + projects: + - name: '{fuel-release}.{plugin-name}.{plugin-version}.testplugin_smoke' + current-parameters: true + kill-phase-on: NEVER + publishers: + - postbuildscript: + script-only-if-succeeded: False + builders: + - shell: env > smoke.properties + - archive: + artifacts: 'build.properties' + allow-empty: false + - archive: + artifacts: 'smoke.properties' + allow-empty: false + - email: + recipients: '{email-to}' + +- job-template: + name: '{bvt-name}' + description: 'The Smoke test for {fuel-release}.{plugin-name}.{plugin-version}' + disabled: false + concurrent: true + project-type: multijob + node: runner + <<: *runner-parameters + <<: *runner-properties + scm: [] + triggers: + - gerrit: + <<: *bvt-gerrit-events + <<: *generic-gerrit-projects + builders: + - copyartifact: + project: '{build-name}' + which-build: last-successful + stable: true + - multijob: + name: 'Smoke tests for testplugin' + condition: SUCCESSFUL + projects: + - name: '{fuel-release}.{plugin-name}.{plugin-version}.testplugin' + current-parameters: true + kill-phase-on: NEVER + publishers: + - postbuildscript: + script-only-if-succeeded: False + builders: + - shell: env > bvt.properties + - archive: + artifacts: '*.rpm' + allow-empty: false + - archive: + artifacts: 'build.properties' + allow-empty: false + - archive: + artifacts: 'bvt.properties' + allow-empty: false + - email: + recipients: '{email-to}'