Add deployment scripts for App Catalog CI

- Added Puppet module and additional shell scripts
- Deployment documentation can be found at deployment/README.md file

Change-Id: Ic81bb539a23a155f4c48eb69c916d953bb12c918
This commit is contained in:
Sergey Kolekonov 2015-05-25 16:06:07 +03:00
parent 1c69e8f726
commit 9d37eba442
10 changed files with 520 additions and 0 deletions

44
deployment/README.md Normal file
View File

@ -0,0 +1,44 @@
apps-catalog-ci
========
Description
----------------------
This is a collection of deployment scripts for apps-catalog CI project.
It consists of Puppet module catalog_ci and an additional shell script.
The scripts allow to setup Jenkins with access to Gerrit to trigger
appropriate jobs on a commit to the apps-catalog project.
Deployment
----------------------
Execute deploy.sh script to begin deployment:
(some operations require superuser access rights)
./deploy.sh
The script will install necessary packages and deploy/configure Jenkins.
You will be able to access it at http://server_ip:8080
The following steps are required after deployment:
- Setup access rights. By default Jenkins uses Launchpad OpenID and all users
have full access to Jenkins.
Proceed to Manage Jenkins -> Configure Global Security and setup security.
- Finish Gerrit auth setup:
Add a private key file (id_rsa) to Jenkins ssh directory:
* sudo mkdir -p /var/lib/jenkins/.ssh
* sudo cp id_rsa /var/lib/jenkins/.ssh
* sudo chown -R jenkins:jenkins /var/lib/jenkins/.ssh
* sudo chmod 600 /var/lib/jenkins/.ssh/id_rsa
Then proceed to Manage Jenkins -> Gerrit Trigger and press the button
in 'Status' column. If button will change its color to green, your connection
to Gerrit works OK and Jenkins is receiving Gerrit events. Otherwise please
check Gerrit server parameters.
- rclone (http://rclone.org/) is used to upload images to CDN.
Please install and configure it manually, if it's required.
'jenkins' user should be able to access default rclone configuration file
in order to use it.
Jenkins Jobs
----------------------
Jenkins Job Builder is used to configure Jenkins jobs. It will be automatically
installed by deployment scripts. Jobs configuration files will be placed to
/etc/jenkins_jobs/jobs. You can use the following command to apply your changes
jenkins-jobs update /etc/jenkins_jobs/jobs

View File

@ -0,0 +1 @@
include catalog_ci

View File

@ -0,0 +1,45 @@
<?xml version='1.0' encoding='UTF-8'?>
<hudson>
<disabledAdministrativeMonitors/>
<version>1.596.2</version>
<numExecutors>3</numExecutors>
<mode>NORMAL</mode>
<useSecurity>true</useSecurity>
<authorizationStrategy class="hudson.security.GlobalMatrixAuthorizationStrategy">
<permission>hudson.model.Hudson.Administer:anonymous</permission>
<permission>hudson.model.Hudson.Read:anonymous</permission>
<permission>hudson.model.Item.Read:anonymous</permission>
</authorizationStrategy>
<securityRealm class="hudson.plugins.openid.OpenIdSsoSecurityRealm" plugin="openid@2.1.1">
<endpoint>https://login.launchpad.net/+openid</endpoint>
</securityRealm>
<disableRememberMe>false</disableRememberMe>
<projectNamingStrategy class="jenkins.model.ProjectNamingStrategy$DefaultProjectNamingStrategy"/>
<workspaceDir>${JENKINS_HOME}/workspace/${ITEM_FULLNAME}</workspaceDir>
<buildsDir>${ITEM_ROOTDIR}/builds</buildsDir>
<markupFormatter class="hudson.markup.RawHtmlMarkupFormatter" plugin="antisamy-markup-formatter@1.1">
<disableSyntaxHighlighting>false</disableSyntaxHighlighting>
</markupFormatter>
<jdks/>
<viewsTabBar class="hudson.views.DefaultViewsTabBar"/>
<myViewsTabBar class="hudson.views.DefaultMyViewsTabBar"/>
<clouds/>
<slaves/>
<quietPeriod>5</quietPeriod>
<scmCheckoutRetryCount>0</scmCheckoutRetryCount>
<views>
<hudson.model.AllView>
<owner class="hudson" reference="../../.."/>
<name>All</name>
<filterExecutors>false</filterExecutors>
<filterQueue>false</filterQueue>
<properties class="hudson.model.View$PropertyList"/>
</hudson.model.AllView>
</views>
<primaryView>All</primaryView>
<slaveAgentPort>0</slaveAgentPort>
<label></label>
<nodeProperties/>
<globalNodeProperties/>
<noUsageStatistics>true</noUsageStatistics>
</hudson>

View File

@ -0,0 +1,67 @@
<?xml version='1.0' encoding='UTF-8'?>
<com.sonyericsson.hudson.plugins.gerrit.trigger.PluginImpl plugin="gerrit-trigger@2.13.0">
<servers class="java.util.concurrent.CopyOnWriteArrayList">
<com.sonyericsson.hudson.plugins.gerrit.trigger.GerritServer>
<name>review.openstack.org</name>
<noConnectionOnStartup>false</noConnectionOnStartup>
<config class="com.sonyericsson.hudson.plugins.gerrit.trigger.config.Config">
<gerritHostName>review.openstack.org</gerritHostName>
<gerritSshPort>29418</gerritSshPort>
<gerritProxy></gerritProxy>
<gerritUserName>catalog-ci</gerritUserName>
<gerritEMail>catalog-ci@mirantis.com</gerritEMail>
<gerritAuthKeyFile>/var/lib/jenkins/.ssh/id_rsa</gerritAuthKeyFile>
<useRestApi>false</useRestApi>
<restCodeReview>false</restCodeReview>
<restVerified>false</restVerified>
<gerritBuildCurrentPatchesOnly>false</gerritBuildCurrentPatchesOnly>
<gerritVerifiedCmdBuildSuccessful>gerrit review &lt;CHANGE&gt;,&lt;PATCHSET&gt; --message &apos;Build Successful &lt;BUILDS_STATS&gt;&apos; --verified &lt;VERIFIED&gt; --code-review &lt;CODE_REVIEW&gt;</gerritVerifiedCmdBuildSuccessful>
<gerritVerifiedCmdBuildUnstable>gerrit review &lt;CHANGE&gt;,&lt;PATCHSET&gt; --message &apos;Build Unstable &lt;BUILDS_STATS&gt;&apos; --verified &lt;VERIFIED&gt; --code-review &lt;CODE_REVIEW&gt;</gerritVerifiedCmdBuildUnstable>
<gerritVerifiedCmdBuildFailed>gerrit review &lt;CHANGE&gt;,&lt;PATCHSET&gt; --message &apos;Build Failed &lt;BUILDS_STATS&gt;&apos; --verified &lt;VERIFIED&gt; --code-review &lt;CODE_REVIEW&gt;</gerritVerifiedCmdBuildFailed>
<gerritVerifiedCmdBuildStarted></gerritVerifiedCmdBuildStarted>
<gerritVerifiedCmdBuildNotBuilt>gerrit review &lt;CHANGE&gt;,&lt;PATCHSET&gt; --message &apos;No Builds Executed &lt;BUILDS_STATS&gt;&apos; --verified &lt;VERIFIED&gt; --code-review &lt;CODE_REVIEW&gt;</gerritVerifiedCmdBuildNotBuilt>
<gerritFrontEndUrl>https://review.openstack.org/</gerritFrontEndUrl>
<gerritBuildStartedVerifiedValue>0</gerritBuildStartedVerifiedValue>
<gerritBuildStartedCodeReviewValue>0</gerritBuildStartedCodeReviewValue>
<gerritBuildSuccessfulVerifiedValue>1</gerritBuildSuccessfulVerifiedValue>
<gerritBuildSuccessfulCodeReviewValue>0</gerritBuildSuccessfulCodeReviewValue>
<gerritBuildFailedVerifiedValue>-1</gerritBuildFailedVerifiedValue>
<gerritBuildFailedCodeReviewValue>0</gerritBuildFailedCodeReviewValue>
<gerritBuildUnstableVerifiedValue>0</gerritBuildUnstableVerifiedValue>
<gerritBuildUnstableCodeReviewValue>0</gerritBuildUnstableCodeReviewValue>
<gerritBuildNotBuiltVerifiedValue>0</gerritBuildNotBuiltVerifiedValue>
<gerritBuildNotBuiltCodeReviewValue>0</gerritBuildNotBuiltCodeReviewValue>
<enableManualTrigger>true</enableManualTrigger>
<enablePluginMessages>true</enablePluginMessages>
<buildScheduleDelay>3</buildScheduleDelay>
<dynamicConfigRefreshInterval>30</dynamicConfigRefreshInterval>
<categories class="linked-list">
<com.sonyericsson.hudson.plugins.gerrit.trigger.VerdictCategory>
<verdictValue>CRVW</verdictValue>
<verdictDescription>Code Review</verdictDescription>
</com.sonyericsson.hudson.plugins.gerrit.trigger.VerdictCategory>
<com.sonyericsson.hudson.plugins.gerrit.trigger.VerdictCategory>
<verdictValue>VRIF</verdictValue>
<verdictDescription>Verified</verdictDescription>
</com.sonyericsson.hudson.plugins.gerrit.trigger.VerdictCategory>
</categories>
<replicationConfig>
<enableReplication>false</enableReplication>
<slaves class="linked-list"/>
<enableSlaveSelectionInJobs>false</enableSlaveSelectionInJobs>
</replicationConfig>
<watchdogTimeoutMinutes>0</watchdogTimeoutMinutes>
<watchTimeExceptionData>
<daysOfWeek/>
<timesOfDay class="linked-list"/>
</watchTimeExceptionData>
<notificationLevel>ALL</notificationLevel>
</config>
</com.sonyericsson.hudson.plugins.gerrit.trigger.GerritServer>
</servers>
<pluginConfig>
<numberOfReceivingWorkerThreads>3</numberOfReceivingWorkerThreads>
<numberOfSendingWorkerThreads>1</numberOfSendingWorkerThreads>
<replicationCacheExpirationInMinutes>360</replicationCacheExpirationInMinutes>
</pluginConfig>
</com.sonyericsson.hudson.plugins.gerrit.trigger.PluginImpl>

View File

@ -0,0 +1,67 @@
- job:
name: check-image
node: master
project-type: freestyle
description: "This job checks glance images"
defaults: global
disabled: false
concurrent: true
scm:
- git:
url: https://git.openstack.org/stackforge/apps-catalog
refspec: $GERRIT_REFSPEC
name:
choosing-strategy: gerrit
skip-tag: false
wipe-workspace: true
branches:
- "$GERRIT_BRANCH"
triggers:
- gerrit:
server-name: "review.openstack.org"
trigger-on-patchset-uploaded-event: true
projects:
- project-compare-type: 'PLAIN'
project-pattern: 'stackforge/apps-catalog'
branch-compare-type: 'PLAIN'
branch-pattern: master
builders:
- shell: "git checkout $FETCH_HEAD\
\n$JENKINS_HOME/scripts/app-catalog.sh"
- job:
name: merge-image
node: master
project-type: freestyle
description: "This job checks and uploads glance images to CDN"
defaults: global
disabled: false
concurrent: true
scm:
- git:
url: https://git.openstack.org/stackforge/apps-catalog
refspec: $GERRIT_REFSPEC
name:
choosing-strategy: gerrit
skip-tag: false
wipe-workspace: true
branches:
- "$GERRIT_BRANCH"
triggers:
- gerrit:
server-name: "review.openstack.org"
trigger-on-change-merged-event: true
projects:
- project-compare-type: 'PLAIN'
project-pattern: 'stackforge/apps-catalog'
branch-compare-type: 'PLAIN'
branch-pattern: master
builders:
- shell: "git checkout $FETCH_HEAD\
\n$JENKINS_HOME/scripts/app-catalog.sh"

View File

@ -0,0 +1,96 @@
#!/bin/bash -x
WDIR=$JENKINS_HOME/scripts
PATCH=$GERRIT_CHANGE_NUMBER
CODES=(100 200 302)
TMP_DIR=$(mktemp -d)
TMP_FILE=$(mktemp)
EVENT=$GERRIT_EVENT_TYPE
FILE_LIST=$(git diff HEAD~1 --name-only)
IMAGE_CONFIG="openstack_catalog/web/static/glance_images.yaml"
IMAGE_CDN_PATH="catalog_ci:catalog/images"
clean() {
rm -rf $TMP_DIR
rm -f $TMP_FILE
}
upload_image () {
local CONFIG
local OLD_CONFIG
local IMAGE_NAME
local IMAGE_PATH
IMAGE_PATH=$1
CONFIG=$TMP_DIR/$(basename $IMAGE_CONFIG)
OLD_CONFIG=$TMP_DIR/$(basename $IMAGE_CONFIG).old
IMAGE_NAME=$(python $WDIR/generate_names.py glance $OLD_CONFIG $CONFIG)
if [ -z "$IMAGE_NAME" ]; then
echo "Image file can't be generated"
exit 1
fi
echo "Uploading image $IMAGE_NAME from $IMAGE_PATH"
mv $IMAGE_PATH $(dirname $IMAGE_PATH)/$IMAGE_NAME
rclone copy $(dirname $IMAGE_PATH)/$IMAGE_NAME $IMAGE_CDN_PATH
clean
}
main() {
local URL
local HASH
local REAL_HASH
local HTTP_CODE
local IMAGE
ssh -p 29418 catalog-ci@review.openstack.org gerrit query $PATCH > $TMP_FILE
URL=$(cat $TMP_FILE | egrep "^\s*Image-URL:\s(https?|ftp)://.*" | egrep -o "(https?|ftp)://.*$")
HASH=$(cat $TMP_FILE | egrep "^\s*Image-hash:\s[A-Za-z0-9]*$" | egrep -o "[A-Za-z0-9]*$")
cat $TMP_FILE | grep Image-URL | grep -q Unknown && exit 0
# if [ $(cat $TMP_FILE | grep Image-URL | grep Unknown) ]; then
# echo "Image URL is unknown, no checks and changes will be performed"
# exit 0
# fi
if [ -z "$URL" -o -z "$HASH" ]; then
echo "Image URL or hash wasn't found"
clean
exit 0
else
HTTP_CODE=$(curl -o /dev/null --silent --head --write-out '%{http_code}\n' $URL)
if ! [[ " ${CODES[*]} " == *" $HTTP_CODE "* ]]; then
echo "File wasn't found"
clean
exit 1
fi
fi
wget $URL -P $TMP_DIR
if [ "$HASH" == "Unknown" ]; then
echo "Image hash is unknown, skipping checks..."
else
REAL_HASH=$(md5sum $TMP_DIR/* | awk '{print $1}')
if [ "$REAL_HASH" != "$HASH" ]; then
echo "Hash mismatch"
clean
exit 1
else
echo "Image hash is correct"
fi
fi
if [ "$EVENT" == "change-merged" ]; then
IMAGE=$(ls $TMP_DIR/*)
cp $IMAGE_CONFIG $TMP_DIR
git checkout HEAD~1
cp $IMAGE_CONFIG $TMP_DIR/$(basename $IMAGE_CONFIG).old
upload_image $IMAGE
else
clean
fi
}
if [[ ${FILE_LIST[*]} =~ "$IMAGE_CONFIG" ]]; then
main "$@"
fi

View File

@ -0,0 +1,50 @@
#!/usr/bin/python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import itertools
import re
from sys import argv
import yaml
def yaml_to_dict(infile, k):
stream = open(infile, 'r')
rdict = yaml.load(stream)[k]
return rdict
def diff_images_config(images1, images2):
if images1 == images2:
return ''
intersec = [item for item in images1 if item in images2]
sym_diff = [item for item in itertools.chain(
images1, images2) if item not in intersec]
name = ''
d_size = len(sym_diff)
if d_size <= 2:
i = d_size - 1
else:
return ''
if 'name' in sym_diff[i].keys() and 'format' in sym_diff[i].keys():
i_name = re.sub('[(){}<>]', '', sym_diff[i]['name'])
i_type = sym_diff[i]['format']
name = i_name + '.' + i_type
name = name.lower().replace(" ", "_")
return name
if __name__ == '__main__':
if argv[1] == 'glance':
images1 = yaml_to_dict(argv[2], 'images')
images2 = yaml_to_dict(argv[3], 'images')
print(diff_images_config(images1, images2))

View File

@ -0,0 +1,123 @@
class catalog_ci {
$user = 'admin'
$password = 'adminpassword'
$jenkins_config = '/var/lib/jenkins/config.xml'
$gerrit_config = '/var/lib/jenkins/gerrit-trigger.xml'
class{ 'jenkins':
lts => true,
install_java => true,
plugin_hash => {
'git' => {},
'parameterized-trigger' => {},
'token-macro' => {},
'mailer' => {},
'scm-api' => {},
'promoted-builds' => {},
'matrix-project' => {},
'git-client' => {},
'ssh-credentials' => {},
'credentials' => {},
'gerrit-trigger' => {},
'rebuild' => {},
'git-client' => {},
'rabbitmq-consumer' => {},
'openid' => {},
'openid4java' => {},
}
}
jenkins::user { "$user":
email => 'admin@example.com',
password => "$password",
}
class{ 'jenkins::security':
security_model => 'full_control',
}
Class['jenkins'] -> Jenkins::User["$user"] -> Class['jenkins::security']
-> Exec['create_jobs'] -> File["$jenkins_config"]
package {'python-pip':
ensure => present,
}
package {'deepdiff':
ensure => present,
provider => 'pip',
}
package {'pyyaml':
ensure => present,
provider => 'pip',
}
package {'jenkins-job-builder':
ensure => present,
provider => 'pip',
}
file {'/var/lib/jenkins/scripts':
owner => 'root',
group => 'root',
ensure => directory,
source => 'puppet:///modules/catalog_ci/scripts',
recurse => true,
require => File['/etc/jenkins_jobs'],
}
file {'/etc/jenkins_jobs':
owner => 'root',
group => 'root',
ensure => directory,
require => Package['jenkins-job-builder'],
}
file {'/etc/jenkins_jobs/jenkins_jobs.ini':
owner => 'root',
group => 'root',
content => template('catalog_ci/jenkins_jobs.ini.erb'),
require => File['/etc/jenkins_jobs'],
}
file {'/etc/jenkins_jobs/jobs':
owner => 'root',
group => 'root',
ensure => directory,
source => 'puppet:///modules/catalog_ci/jobs',
recurse => true,
require => File['/etc/jenkins_jobs'],
}
file {"$jenkins_config":
owner => 'jenkins',
group => 'jenkins',
ensure => present,
source => 'puppet:///modules/catalog_ci/config.xml',
}
file {"$gerrit_config":
owner => 'jenkins',
group => 'jenkins',
ensure => present,
source => 'puppet:///modules/catalog_ci/gerrit-trigger.xml',
require => File["$jenkins_config"],
}
exec {'create_jobs':
command => 'jenkins-jobs update /etc/jenkins_jobs/jobs',
path => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin',
require => File['/etc/jenkins_jobs/jobs'],
}
exec{'restart_jenkins':
command => 'service jenkins restart',
path => '/bin:/sbin:/usr/bin:/usr/sbin',
subscribe => [ File["$jenkins_config"], File["$gerrit_config"] ],
}
Package['python-pip'] -> Package['deepdiff'] ->
Package['pyyaml'] -> Package['jenkins-job-builder']
}

View File

@ -0,0 +1,8 @@
[job_builder]
ignore_cache=True
[jenkins]
user=<%= @user %>
password=<%= @password %>
url=http://127.0.0.1:8080
query_plugins_info=False

19
deployment/deploy.sh Executable file
View File

@ -0,0 +1,19 @@
#!/bin/bash -e
# Generate a password for service account
ADMIN_PASS=$(< /dev/urandom tr -dc _A-Z-a-z-0-9 | head -c10)
sed -i "s/adminpassword/$ADMIN_PASS/g" catalog-ci-jenkins/modules/catalog_ci/manifests/init.pp
sudo apt-get update
sudo apt-get install git puppet -y
# Using custom (forked) puppet-jenkins module
# due to critical problem in the upstream one
git clone https://github.com/skolekonov/puppet-jenkins.git
tar czf rtyler-jenkins-1.3.0.tar.gz puppet-jenkins/*
sudo puppet module install rtyler-jenkins-1.3.0.tar.gz
sudo puppet apply -vd --modulepath catalog-ci-jenkins/modules:/etc/puppet/modules catalog-ci-jenkins/manifests/site.pp
echo "Deployment completed"
echo "WARNING. Please open Jenkins WebUI and setup user access matrix"