opendev
/
system-config
Code Issues Proposed changes

Merge in docs from openstack-ci. 

Change-Id: I49f71f8118e45f28d4b777ddc9588b8e30151d79
This commit is contained in:
Monty Taylor 2012-05-15 17:21:49 -04:00
parent 215f3afbe2
commit 90531483b0
13 changed files with 2658 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

216
doc/conf.py Normal file
View File

@ -0,0 +1,216 @@
# -*- coding: utf-8 -*-
#
# OpenStack CI documentation build configuration file, created by
# sphinx-quickstart on Mon Jul 18 13:42:23 2011.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
import sys, os, datetime
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))
# -- General configuration -----------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = []
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix of source filenames.
source_suffix = '.rst'
# The encoding of source files.
#source_encoding = 'utf-8-sig'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = u'OpenStack CI'
copyright = u'2011, Monty Taylor, James Blair and Andrew Hutchings'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = "%d.%02d" % (datetime.datetime.now().year, datetime.datetime.now().month)
# The full version, including alpha/beta/rc tags.
release = "%d.%02d.%02d" % (datetime.datetime.now().year, datetime.datetime.now().month, datetime.datetime.now().day)
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None
# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []
# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None
# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True
# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True
# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []
# -- Options for HTML output ---------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
html_theme = 'default'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}
# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []
# The name for this set of Sphinx documents. If None, it defaults to
# "<project> v<release> documentation".
#html_title = None
# A shorter title for the navigation bar. Default is the same as html_title.
#html_short_title = None
# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None
# The name of an image file (within the static path) to use as favicon of the
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'
# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True
# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}
# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}
# If false, no module index is generated.
#html_domain_indices = True
# If false, no index is generated.
#html_use_index = True
# If true, the index is split into individual pages for each letter.
#html_split_index = False
# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True
# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True
# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True
# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it. The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''
# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None
# Output file base name for HTML help builder.
htmlhelp_basename = 'OpenStackCIdoc'
# -- Options for LaTeX output --------------------------------------------------
# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'
# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
('index', 'OpenStackCI.tex', u'OpenStack CI Documentation',
u'Monty Taylor and James Blair', 'manual'),
]
# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None
# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False
# If true, show page references after internal links.
#latex_show_pagerefs = False
# If true, show URL addresses after external links.
#latex_show_urls = False
# Additional stuff for the LaTeX preamble.
#latex_preamble = ''
# Documents to append as an appendix to all manuals.
#latex_appendices = []
# If false, no module index is generated.
#latex_domain_indices = True
# -- Options for manual page output --------------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
('index', 'openstackci', u'OpenStack CI Documentation',
[u'Monty Taylor, James Blair and Andrew Hutchings'], 1)
]

979
doc/gerrit.rst Normal file
View File

@ -0,0 +1,979 @@
:title: Gerrit Installation
Gerrit
######
Objective
*********
A workflow where developers submit changes to gerrit, changes are
peer-reviewed and automatically tested by Jenkins before being
committed to the main repo. The public repo is on github.
References
**********
* http://gerrit.googlecode.com/svn/documentation/2.2.1/install.html
* http://feeding.cloud.geek.nz/2011/04/code-reviews-with-gerrit-and-gitorious.html
* http://feeding.cloud.geek.nz/2011/05/integrating-launchpad-and-gerrit-code.html
* http://www.infoq.com/articles/Gerrit-jenkins-hudson
* https://wiki.jenkins-ci.org/display/JENKINS/Gerrit+Trigger
* https://wiki.mahara.org/index.php/Developer_Area/Developer_Tools
Known Issues
************
* Don't use innodb until at least gerrit 2.2.2 because of:
http://code.google.com/p/gerrit/issues/detail?id=518
Installation
************
Host Installation
=================
Prepare Host
------------
This sets the host up with the standard OpenStack system
administration configuration. Skip this if you're not setting up a
host for use by the OpenStack project.
.. code-block:: bash
sudo apt-get install puppet git openjdk-6-jre-headless mysql-server
git clone git://github.com/openstack/openstack-ci-puppet.git
cd openstack-ci-puppet/
sudo puppet apply --modulepath=modules manifests/site.pp
Install MySQL
-------------
You should setup MySQL as follows, changing 'secret' to a suitable password:
.. code-block:: bash
mysql -u root -p
.. code-block:: mysql
CREATE USER 'gerrit2'@'localhost' IDENTIFIED BY 'secret';
CREATE DATABASE reviewdb;
ALTER DATABASE reviewdb charset=latin1;
GRANT ALL ON reviewdb.* TO 'gerrit2'@'localhost';
FLUSH PRIVILEGES;
Then create the gerrit2 system user as follows:
.. code-block:: bash
sudo useradd -mr gerrit2
sudo chsh gerrit2 -s /bin/bash
sudo su - gerrit2
With Gerrit 2.2.2 onwards edit /etc/mysql/my.cnf with the following:
.. code-block:: ini
[mysqld]
default-storage-engine=INNODB
Install Gerrit
--------------
Note that Openstack's gerrit installation currently uses a custom .war of gerrit
2.2.2. The following instruction is for the generic gerrit binaries:
.. code-block:: bash
wget http://gerrit.googlecode.com/files/gerrit-2.2.1.war
mv gerrit-2.2.1.war gerrit.war
java -jar gerrit.war init -d review_site
The .war file will bring up an interactive tool to change the settings, these
should be set as follows. Note that the password configured earlier for MySQL
should be provided when prompted::
*** Gerrit Code Review 2.2.1
***
Create '/home/gerrit2/review_site' [Y/n]?
*** Git Repositories
***
Location of Git repositories [git]:
*** SQL Database
***
Database server type [H2/?]: ?
Supported options are:
h2
postgresql
mysql
jdbc
Database server type [H2/?]: mysql
Gerrit Code Review is not shipped with MySQL Connector/J 5.1.10
** This library is required for your configuration. **
Download and install it now [Y/n]?
Downloading http://repo2.maven.org/maven2/mysql/mysql-connector-java/5.1.10/mysql-connector-java-5.1.10.jar ... OK
Checksum mysql-connector-java-5.1.10.jar OK
Server hostname [localhost]:
Server port [(MYSQL default)]:
Database name [reviewdb]:
Database username [gerrit2]:
gerrit2's password :
confirm password :
*** User Authentication
***
Authentication method [OPENID/?]:
*** Email Delivery
***
SMTP server hostname [localhost]:
SMTP server port [(default)]:
SMTP encryption [NONE/?]:
SMTP username :
*** Container Process
***
Run as [gerrit2]:
Java runtime [/usr/lib/jvm/java-6-openjdk/jre]:
Copy gerrit.war to /home/gerrit2/review_site/bin/gerrit.war [Y/n]?
Copying gerrit.war to /home/gerrit2/review_site/bin/gerrit.war
*** SSH Daemon
***
Listen on address [*]:
Listen on port [29418]:
Gerrit Code Review is not shipped with Bouncy Castle Crypto v144
If available, Gerrit can take advantage of features
in the library, but will also function without it.
Download and install it now [Y/n]?
Downloading http://www.bouncycastle.org/download/bcprov-jdk16-144.jar ... OK
Checksum bcprov-jdk16-144.jar OK
Generating SSH host key ... rsa... dsa... done
*** HTTP Daemon
***
Behind reverse proxy [y/N]? y
Proxy uses SSL (https://) [y/N]? y
Subdirectory on proxy server [/]:
Listen on address [*]:
Listen on port [8081]:
Canonical URL [https://review.openstack.org/]:
Initialized /home/gerrit2/review_site
Executing /home/gerrit2/review_site/bin/gerrit.sh start
Starting Gerrit Code Review: OK
Waiting for server to start ... OK
Opening browser ...
Please open a browser and go to https://review.openstack.org/#admin,projects
Configure Gerrit
----------------
The file /home/gerrit2/review_site/etc/gerrit.config will be setup automatically
by puppet.
Set Gerrit to start on boot:
.. code-block:: bash
ln -snf /home/gerrit2/review_site/bin/gerrit.sh /etc/init.d/gerrit
update-rc.d gerrit defaults 90 10
Then create the file ``/etc/default/gerritcodereview`` with the following
contents:
.. code-block:: ini
GERRIT_SITE=/home/gerrit2/review_site
Add "Approved" review type to gerrit:
.. code-block:: mysql
mysql -u root -p
use reviewdb;
insert into approval_categories values ('Approved', 'A', 2, 'MaxNoBlock', 'N', 'APRV');
insert into approval_category_values values ('No score', 'APRV', 0);
insert into approval_category_values values ('Approved', 'APRV', 1);
update approval_category_values set name = "Looks good to me (core reviewer)" where name="Looks good to me, approved";
Expand "Verified" review type to -2/+2:
.. code-block:: mysql
mysql -u root -p
use reviewdb;
update approval_category_values set value=2
where value=1 and category_id='VRIF';
update approval_category_values set value=-2
where value=-1 and category_id='VRIF';
insert into approval_category_values values
("Doesn't seem to work","VRIF",-1),
("Works for me","VRIF","1");
Reword the default messages that use the word Submit, as they imply that
we're not happy with people for submitting the patch in the first place:
.. code-block:: mysql
mysql -u root -p
use reviewdb;
update approval_category_values set name="Do not merge"
where category_id='CRVW' and value=-2;
update approval_category_values
set name="I would prefer that you didn't merge this"
where category_id='CRVW' and value=-1;
OpenStack currently uses a hybrid approach for CLA enforcement. We
use Gerrit's built in CLA system to ensure that contributors have
signed the CLA, but contributors don't actually use Gerrit to sign it.
Instead, developers use an external service (Echosign) to agree to the
CLA, and then request membership in a Launchpad group called
"openstack-cla". The moderators of that group (core members of any
OpenStack project) approve membership requests after verifying that
new contributors have signed the CLA at Echosign. The openstack-cla
group is kept synchronized with Gerrit. Gerrit is then configured
with a "dummy" CLA (which users are not expected to see), and the
administrator indicates to Gerrit that the entire openstack-cla group
has agreed to the CLA. This lets Gerrit enforce that the CLA has been
signed while the actual facility to sign it in Gerrit is disabled via
a source patch.
This configuration is not recommended for new projects and is merely
an artifact of legal requirements placed on the OpenStack project.
Here are the SQL commands to set it up:
.. code-block:: mysql
insert into contributor_agreement_id values (NULL);
insert into contributor_agreements values ('Y', 'N', 'N', 'CLA (Echosign)',
'OpenStack CLA via Echosign', 'static/echosign-cla.html', 1);
insert into account_group_agreements values (
now(), 'V', 1, now(), NULL,
(select group_id from account_group_names where name='openstack-cla'),
1);
Install Apache
--------------
::
apt-get install apache2
Create: /etc/apache2/sites-available/gerrit:
.. code-block:: apacheconf
<VirtualHost *:80>
ServerAdmin webmaster@localhost
ErrorLog ${APACHE_LOG_DIR}/gerrit-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/gerrit-access.log combined
Redirect / https://review-dev.openstack.org/
</VirtualHost>
<IfModule mod_ssl.c>
<VirtualHost _default_:443>
ServerAdmin webmaster@localhost
ErrorLog ${APACHE_LOG_DIR}/gerrit-ssl-error.log
LogLevel warn
CustomLog ${APACHE_LOG_DIR}/gerrit-ssl-access.log combined
SSLEngine on
SSLCertificateFile /etc/ssl/certs/ssl-cert-snakeoil.pem
SSLCertificateKeyFile /etc/ssl/private/ssl-cert-snakeoil.key
#SSLCertificateChainFile /etc/apache2/ssl.crt/server-ca.crt
<FilesMatch "\.(cgi|shtml|phtml|php)$">
SSLOptions +StdEnvVars
</FilesMatch>
<Directory /usr/lib/cgi-bin>
SSLOptions +StdEnvVars
</Directory>
BrowserMatch "MSIE [2-6]" \
nokeepalive ssl-unclean-shutdown \
downgrade-1.0 force-response-1.0
# MSIE 7 and newer should be able to use keepalive
BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown
RewriteEngine on
RewriteCond %{HTTP_HOST} !review-dev.openstack.org
RewriteRule ^.*$ https://review-dev.openstack.org/
ProxyPassReverse / http://localhost:8081/
<Location />
Order allow,deny
Allow from all
ProxyPass http://localhost:8081/ retry=0
</Location>
</VirtualHost>
</IfModule>
Run the following commands:
.. code-block:: bash
a2enmod ssl proxy proxy_http rewrite
a2ensite gerrit
a2dissite default
Install Exim
------------
::
apt-get install exim4
dpkg-reconfigure exim4-config
Choose "internet site", otherwise select defaults
edit: /etc/default/exim4 ::
QUEUEINTERVAL='5m'
GitHub Setup
============
Generate an SSH key for Gerrit for use on GitHub
------------------------------------------------
::
sudo su - gerrit2
gerrit2@gerrit:~$ ssh-keygen
Generating public/private rsa key pair.
Enter file in which to save the key (/home/gerrit2/.ssh/id_rsa):
Created directory '/home/gerrit2/.ssh'.
Enter passphrase (empty for no passphrase):
Enter same passphrase again:
GitHub Configuration
--------------------
#. create openstack-gerrit user on github
#. add gerrit2 ssh public key to openstack-gerrit user
#. create gerrit team in openstack org on github with push/pull access
#. add openstack-gerrit to gerrit team in openstack org
#. add public master repo to gerrit team in openstack org
#. save github host key in known_hosts
::
gerrit2@gerrit:~$ ssh git@github.com
The authenticity of host 'github.com (207.97.227.239)' can't be established.
RSA key fingerprint is 16:27:ac:a5:76:28:2d:36:63:1b:56:4d:eb:df:a6:48.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added 'github.com,207.97.227.239' (RSA) to the list of known hosts.
PTY allocation request failed on channel 0
You will also need to create the file ``github.secure.config`` in the gerrit2 user's home directory. The contents of this are as follows:
.. code-block:: ini
[github]
username = guthub-user
api_token = hexstring
The username should be the github username for gerrit to use when communicating
with github. The api_token can be found in github's account setting for the
account.
Gerrit Replication to GitHub
----------------------------
The file ``review_site/etc/replication.config`` is needed with the following
contents:
.. code-block:: ini
[remote "github"]
url = git@github.com:${name}.git
Jenkins / Gerrit Integration
============================
Create a Jenkins User in Gerrit
-------------------------------
With the jenkins public key, as a gerrit admin user::
cat jenkins.pub | ssh -p29418 review.openstack.org gerrit create-account --ssh-key - --full-name Jenkins jenkins
Create "CI Systems" group in gerrit, make jenkins a member
Create a Gerrit Git Prep Job in Jenkins
---------------------------------------
When gating trunk with Jenkins, we want to test changes as they will
appear once merged by Gerrit, but the gerrit trigger plugin will, by
default, test them as submitted. If HEAD moves on while the change is
under review, it may end up getting merged with HEAD, and we want to
test the result.
To do that, make sure the "Hudson Template Project plugin" is
installed, then set up a new job called "Gerrit Git Prep", and add a
shell command build step (no other configuration)::
#!/bin/sh -x
git checkout $GERRIT_BRANCH
git reset --hard remotes/origin/$GERRIT_BRANCH
git merge FETCH_HEAD
CODE=$?
if [ ${CODE} -ne 0 ]; then
git reset --hard remotes/origin/$GERRIT_BRANCH
exit ${CODE}
fi
Later, we will configure Jenkins jobs that we want to behave this way
to use this build step.
Auto Review Expiry
==================
Puppet automatically installs a daily cron job called ``expire_old_reviews.py``
onto the gerrit servers. This script follows two rules:
#. If the review hasn't been touched in 2 weeks, mark as abandoned.
#. If there is a negative review and it hasn't been touched in 1 week, mark as
abandoned.
If your review gets touched by either of these rules it is possible to
unabandon a review on the gerrit web interface.
Launchpad Sync
==============
The launchpad user sync process consists of two scripts which are in
openstack/openstack-ci on github: sync_launchpad_gerrit.py and
insert_gerrit.py.
Both scripts should be run as gerrit2 on review.openstack.org
sync_launchpad_users.py runs and creates a python pickle file, users.pickle,
with all of the user and group information. This is a long process. (12
minutes)
insert_gerrit.py reads the pickle file and applies it to the MySQL database.
The gerrit caches must then be flushed.
Depends
-------
::
apt-get install python-mysqldb python-openid python-launchpadlib
Keys
----
The key for the launchpad sync user is in ~/.ssh/launchpad_rsa. Connecting
to Launchpad requires oauth authentication - so the first time
sync_launchpad_gerrit.py is run, it will display a URL. Open this URL in a
browser and log in to launchpad as the hudson-openstack user. Subsequent
runs will run with cached credentials.
Running
-------
::
cd openstack-ci
git pull
python sync_launchpad_gerrit.py
python insert_gerrit.py
ssh -i /home/gerrit2/.ssh/launchpadsync_rsa -p29418 review.openstack.org gerrit flush-caches
Gerrit IRC Bot
==============
Installation
------------
Ensure there is an up-to-date checkout of openstack-ci in ~gerrit2.
::
apt-get install python-irclib python-daemon
cp ~gerrit2/openstack-ci/gerritbot.init /etc/init.d
chmod a+x /etc/init.d/gerritbot
update-rc.d gerritbot defaults
su - gerrit2
ssh-keygen -f /home/gerrit2/.ssh/gerritbot_rsa
As a Gerrit admin, create a user for gerritbot::
cat ~gerrit2/.ssh/gerritbot_rsa | ssh -p29418 gerrit.openstack.org gerrit create-account --ssh-key - --full-name GerritBot gerritbot
Configure gerritbot, including which events should be announced in the
gerritbot.config file:
.. code-block:: ini
[ircbot]
nick=NICNAME
pass=PASSWORD
server=irc.freenode.net
channel=openstack-dev
port=6667
[gerrit]
user=gerritbot
key=/home/gerrit2/.ssh/gerritbot_rsa
host=review.openstack.org
port=29418
events=patchset-created, change-merged, x-vrif-minus-1, x-crvw-minus-2
Register an account with NickServ on FreeNode, and put the account and
password in the config file.
::
sudo /etc/init.d/gerritbot start
Launchpad Bug Integration
=========================
In addition to the hyperlinks provided by the regex in gerrit.config,
we use a Gerrit hook to update Launchpad bugs when changes referencing
them are applied.
Installation
------------
Ensure an up-to-date checkout of openstack-ci is in ~gerrit2.
::
apt-get install python-pyme
cp ~gerrit2/gerrit-hooks/change-merged ~gerrit2/review_site/hooks/
Create a GPG and register it with Launchpad::
gerrit2@gerrit:~$ gpg --gen-key
gpg (GnuPG) 1.4.11; Copyright (C) 2010 Free Software Foundation, Inc.
This is free software: you are free to change and redistribute it.
There is NO WARRANTY, to the extent permitted by law.
Please select what kind of key you want:
(1) RSA and RSA (default)
(2) DSA and Elgamal
(3) DSA (sign only)
(4) RSA (sign only)
Your selection?
RSA keys may be between 1024 and 4096 bits long.
What keysize do you want? (2048)
Requested keysize is 2048 bits
Please specify how long the key should be valid.
0 = key does not expire
<n> = key expires in n days
<n>w = key expires in n weeks
<n>m = key expires in n months
<n>y = key expires in n years
Key is valid for? (0)
Key does not expire at all
Is this correct? (y/N) y
You need a user ID to identify your key; the software constructs the user ID
from the Real Name, Comment and Email Address in this form:
"Heinrich Heine (Der Dichter) <heinrichh@duesseldorf.de>"
Real name: Openstack Gerrit
Email address: review@openstack.org
Comment:
You selected this USER-ID:
"Openstack Gerrit <review@openstack.org>"
Change (N)ame, (C)omment, (E)mail or (O)kay/(Q)uit? o
You need a Passphrase to protect your secret key.
gpg: gpg-agent is not available in this session
You don't want a passphrase - this is probably a *bad* idea!
I will do it anyway. You can change your passphrase at any time,
using this program with the option "--edit-key".
We need to generate a lot of random bytes. It is a good idea to perform
some other action (type on the keyboard, move the mouse, utilize the
disks) during the prime generation; this gives the random number
generator a better chance to gain enough entropy.
gpg: /home/gerrit2/.gnupg/trustdb.gpg: trustdb created
gpg: key 382ACA7F marked as ultimately trusted
public and secret key created and signed.
gpg: checking the trustdb
gpg: 3 marginal(s) needed, 1 complete(s) needed, PGP trust model
gpg: depth: 0 valid: 1 signed: 0 trust: 0-, 0q, 0n, 0m, 0f, 1u
pub 2048R/382ACA7F 2011-07-26
Key fingerprint = 21EF 7F30 C281 F61F 44CD EC48 7424 9762 382A CA7F
uid Openstack Gerrit <review@openstack.org>
sub 2048R/95F6FA4A 2011-07-26
gerrit2@gerrit:~$ gpg --send-keys --keyserver keyserver.ubuntu.com 382ACA7F
gpg: sending key 382ACA7F to hkp server keyserver.ubuntu.com
Log into the Launchpad account and add the GPG key to the account.
Adding New Projects
*******************
Creating a Project in Gerrit
============================
Using ssh key of a gerrit admin (you)::
ssh -p 29418 review.openstack.org gerrit create-project --name openstack/PROJECT
If the project is an API project (eg, image-api), we want it to share
some extra permissions that are common to all API projects (eg, the
OpenStack documentation coordinators can approve changes, see
:ref:`acl`). Run the following command to reparent the project if it
is an API project::
ssh -p 29418 gerrit.openstack.org gerrit set-project-parent --parent API-Projects openstack/PROJECT
Add yourself to the "Project Bootstrappers" group in Gerrit which will
give you permissions to push to the repo bypassing code review.
Do the initial push of the project with::
git push ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git HEAD:refs/heads/master
git push --tags ssh://USERNAME@review.openstack.org:29418/openstack/PROJECT.git
Remove yourself from the "Project Bootstrappers" group, and then set
the access controls as specified in :ref:`acl`.
Have Jenkins Monitor a Gerrit Project
=====================================
In jenkins, under source code management:
* select git
* url: ssh://jenkins@review.openstack.org:29418/openstack/project.git
* click "advanced"
* refspec: $GERRIT_REFSPEC
* branches: origin/$GERRIT_BRANCH
* click "advanced"
* choosing stragety: gerrit trigger
* select gerrit event under build triggers:
* Trigger on Comment Added
* Approval Category: APRV
* Approval Value: 1
* plain openstack/project
* path **
* Select "Add build step" under "Build"
* select "Use builders from another project"
* Template Project: "Gerrit Git Prep"
* make sure this build step is the first in the sequence
Create a Project in GitHub
==========================
As a github openstack admin:
* Visit https://github.com/organizations/openstack
* Click New Repository
* Visit the gerrit team admin page
* Add the new repository to the gerrit team
Pull requests can not be disabled for a project in Github, so instead
we have a script that runs from cron to close any open pull requests
with instructions to use Gerrit.
* Edit openstack/openstack-ci-puppet:manifests/site.pp
and add the project to the list of github projects in the gerrit class
for the gerrit.openstack.org node.
Migrating a Project from bzr
============================
Add the bzr PPA and install bzr-fastimport:
add-apt-repository ppa:bzr/ppa
apt-get update
apt-get install bzr-fastimport
Doing this from the bzr PPA is important to ensure at least version 0.10 of
bzr-fastimport.
Clone the git-bzr-ng from termie:
git clone https://github.com/termie/git-bzr-ng.git
In git-bzr-ng, you'll find a script, git-bzr. Put it somewhere in your path.
Then, to get a git repo which contains the migrated bzr branch, run:
git bzr clone lp:${BRANCHNAME} ${LOCATION}
So, for instance, to do glance, you would do:
git bzr clone lp:glance glance
And you will then have a git repo of glance in the glance dir. This git repo
is now suitable for uploading in to gerrit to become the new master repo.
Project Config
==============
There are a few options which need to be enabled on the project in the Admin
interface.
* Merge Strategy should be set to "Merge If Necessary"
* "Automatically resolve conflicts" should be enabled
* "Require Change-Id in commit message" should be enabled
* "Require a valid contributor agreement to upload" should be enabled
Optionally, if the PTL agrees to it:
* "Require the first line of the commit to be 50 characters or less" should
be enabled.
.. _acl:
Access Controls
***************
High level goals:
#. Anonymous users can read all projects.
#. All registered users can perform informational code review (+/-1)
on any project.
#. Jenkins can perform verification (blocking or approving: +/-1).
#. All registered users can create changes.
#. The OpenStack Release Manager and Jenkins can tag releases (push
annotated tags).
#. Members of $PROJECT-core group can perform full code review
(blocking or approving: +/- 2), and submit changes to be merged.
#. Members of openstack-release (Release Manager and PTLs), and
$PROJECT-drivers (PTL and release minded people) exclusively can
perform full code review (blocking or approving: +/- 2), and submit
changes to be merged on milestone-proposed branches.
#. Full code review (+/- 2) of API projects should be available to the
-core group of the corresponding implementation project as well as to
the OpenStack Documentation Coordinators.
#. Full code review of stable branches should be available to the
-core group of the project as well as the openstack-stable-maint
group.
To manage API project permissions collectively across projects, API
projects are reparented to the "API-Projects" meta-project instead of
"All-Projects". This causes them to inherit permissions from the
API-Projects project (which, in turn, inherits from All-Projects).
These permissions try to achieve the high level goals::
All Projects (metaproject):
refs/*
read: anonymous
push annotated tag: release managers, ci tools, project bootstrappers
forge author identity: registered users
forge committer identity: project bootstrappers
push (w/ force push): project bootstrappers
create reference: project bootstrappers, release managers
push merge commit: project bootstrappers
refs/for/refs/*
push: registered users
refs/heads/*
label code review:
-1/+1: registered users
-2/+2: project bootstrappers
label verified:
-2/+2: ci tools
-2/+2: project bootstrappers
-1/+1: external tools
label approved 0/+1: project bootstrappers
submit: ci tools
submit: project bootstrappers
refs/heads/milestone-proposed
label code review (exclusive):
-2/+2 openstack-release
-1/+1 registered users
label approved (exclusive): 0/+1: openstack-release
owner: openstack-release
refs/heads/stable/*
label code review (exclusive):
-2/+2 opestack-stable-maint
-1/+1 registered users
label approved (exclusive): 0/+1: opestack-stable-maint
refs/meta/config
read: project owners
API Projects (metaproject):
refs/*
owner: Administrators
refs/heads/*
label code review -2/+2: openstack-doc-core
label approved 0/+1: openstack-doc-core
project foo:
refs/*
owner: Administrators
refs/heads/*
label code review -2/+2: foo-core
label approved 0/+1: foo-core
refs/heads/milestone-proposed
label code review -2/+2: foo-drivers
label approved 0/+1: foo-drivers
Renaming a Project
******************
Renaming a project is not automated and is disruptive to developers,
so it should be avoided. Allow for an hour of downtime for the
project in question, and about 10 minutes of downtime for all of
Gerrit. All Gerrit changes, merged and open, will carry over, so
in-progress changes do not need to be merged before the move.
To rename a project:
#. Make it inacessible by editing the Access pane. Add a "read" ACL
for "Administrators", and mark it "exclusive". Be sure to save
changes.
#. Update the database::
update account_project_watches
set project_name = "openstack/OLD"
where project_name = "openstack/NEW";
update changes
set dest_project_name = "openstack/OLD"
where dest_project_name = "openstack/NEW";
#. Wait for Jenkins to be idle (or take it offline)
#. Stop Gerrit and move the Git repository::
/etc/init.d/gerrit stop
cd /home/gerrit2/review_site/git/openstack/
mv OLD.git/ NEW.git
/etc/init.d/gerrit start
#. (Bring Jenkins online if need be)
#. Rename the project in GitHub
#. Update Jenkins jobs te reference the new name. Rename the jobs
themselves as appropriate
#. Remove the read access ACL you set in the first step from project
#. Submit a change that updates .gitreview with the new location of the
project
Developers will either need to re-clone a new copy of the repository,
or manually update their remotes.
Deleting a User from Gerrit
***************************
This isn't normally necessary, but if you find that you need to
completely delete an account from Gerrit, here's how:
.. code-block:: mysql
delete from account_agreements where account_id=NNNN;
delete from account_diff_preferences where id=NNNN;
delete from account_external_ids where account_id=NNNN;
delete from account_group_members where account_id=NNNN;
delete from account_group_members_audit where account_id=NNNN;
delete from account_patch_reviews where account_id=NNNN;
delete from account_project_watches where account_id=NNNN;
delete from account_ssh_keys where account_id=NNNN;
delete from accounts where account_id=NNNN;
.. code-block:: bash
ssh review.openstack.org -p29418 gerrit flush-caches --all
Adding A New Project On The Command Line
****************************************
All of the steps involved in adding a new project to Gerrit can be
accomplished via the commandline, with the exception of creating a new repo
on github and adding the jenkins jobs.
First of all, add the .gitreview file to the repo that will be added. Then,
assuming an ssh config alias of `review` for the gerrit instance, as a person
in the Project Bootstrappers group::
ssh review gerrit create-project --name openstack/$PROJECT
git review -s
git push gerrit HEAD:refs/heads/master
git push --tags gerrit
At this point, the branch contents will be in gerrit, and the project config
settings and ACLs need to be set. These are maintained in a special branch
inside of git in gerrit. Check out the branch from git::
git fetch gerrit +refs/meta/*:refs/remotes/gerrit-meta/*
git checkout -b config remotes/gerrit-meta/config
There will be two interesting files, `groups` and `project.config`. `groups`
contains UUIDs and names of groups that will be referenced in
`project.config`. There is a helper script in the openstack-ci repo called
`get_group_uuid.py` which will fetch the UUID for a given group. For
$PROJECT-core and $PROJECT-drivers::
openstack-ci/gerrit/get_group_uuid.py $GROUP_NAME
And make entries in `groups` for each one of them. Next, edit
`project.config` to look like::
[access "refs/*"]
owner = group Administrators
[receive]
requireChangeId = true
requireContributorAgreement = true
[submit]
mergeContent = true
[access "refs/heads/*"]
label-Code-Review = -2..+2 group $PROJECT-core
label-Approved = +0..+1 group $PROJECT-core
[access "refs/heads/milestone-proposed"]
label-Code-Review = -2..+2 group $PROJECT-drivers
label-Approved = +0..+1 group $PROJECT-drivers
Replace $PROJECT with the name of the project.
Finally, commit the changes and push the config back up to Gerrit::
git commit -m "Initial project config"
git push gerrit HEAD:refs/meta/config

47
doc/index.rst Normal file
View File

@ -0,0 +1,47 @@
.. OpenStack CI documentation master file, created by
sphinx-quickstart on Mon Jul 18 13:42:23 2011.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
OpenStack Continuous Integration
================================
This documentation covers the installation and maintenance of the
Continuous Integration (CI) infrastructure used by OpenStack. It
may be of interest to people who may want to help develop this
infrastructure or integrate their tools into it. Some instructions
may be useful to other projects that want to set up similar CI
systems.
OpenStack developers or users do not need to read this documentation.
Instead, see http://wiki.openstack.org/ to learn how contribute to or
use OpenStack.
Howtos:
.. toctree::
:maxdepth: 2
third_party
stackforge
Contents:
.. toctree::
:maxdepth: 2
systems
jenkins
gerrit
puppet
puppet_modules
jenkins_jobs
meetbot
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`

340
doc/jenkins.rst Normal file
View File

@ -0,0 +1,340 @@
:title: Jenkins Configuration
Jenkins
#######
Overview
********
Jenkins is a Continuous Integration system and the central control
system for the orchestration of both pre-merge testing and post-merge
actions such as packaging and publishing of documentation.
The overall design that Jenkins is a key part of implementing is that
all code should be reviewed and tested before being merged in to trunk,
and that as many tasks around review, testing, merging and release that
can be automated should be.
Jenkis is essentially a job queing system, and everything that is done
through Jenkins can be thought of as having a few discreet components:
* Triggers - What causes a job to be run
* Location - Where do we run a job
* Steps - What actions are taken when the job runs
* Results - What is the outcome of the job
The OpenStack Jenkins can be found at http://jenkins.openstack.org
OpenStack uses :doc:`gerrit` to manage code reviews, which in turns calls
Jenkins to test those reviews.
Authorization
*************
Jenkins is set up to use OpenID in a Single Sign On mode with Launchpad.
This means that all of the user and group information is managed via
Launchpad users and teams. In the Jenkins Security Matrix, a Launchpad team
name can be specified and any members of that team will be granted those
permissions. However, because of the way the information is processed, a
user will need to re-log in upon changing either team membership on
Launchpad, or changing that team's authorization in Jenkins for the new
privileges to take effect.
Integration Testing
*******************
TODO: How others can get involved in testing and integrating with
OpenStack Jenkins.
Rackspace Bare-Metal Testing Cluster
====================================
The CI team mantains a cluster of machines supplied by Rackspace to
perform bare-metal deployment and testing of OpenStack as a whole.
This installation is intended as a reference implementation of just
one of many possible testing platforms, all of which can be integrated
with the OpenStack Jenkins system. This is a cluster of several
physical machines meaning the test environment has access to all of
the native processor features, and real-world networking, including
tagged VLANs.
Each time the trunk repo is updated, a Jenkins job will deploy an
OpenStack cluster using devstack and then run the openstack-test-rax
test suite against the cluster.
Deployment and Testing Process
------------------------------
The cluster deployment is divided into two phases: base operating
system installation, and OpenStack installation. Because the
operating system install takes considerable time (15 to 30 minutes),
has external network resource dependencies (the distribution mirror),
and has no bearing on the outcome of the OpenStack tests themselves,
the process used here effectively snapshots the machines immediately
after the base OS install and before OpenStack is installed. LVM
snapshots and kexec are used to immediately return the cluster to a
newly installed state without incurring the additional time it would
take to install from scratch. The Jenkins testing job invokes the
process starting at :ref:`rax_openstack_install`.
Installation Server Configuration
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
The CI team runs the Ubuntu Orchestra server (based on cobbler) on our
Jenkins slave node to manage the OS installation on the test machines.
The configuration for the Orchestra server is kept in the CI team's
puppet modules. If you want to set up your own system, Orchestra is
not required, any system capable of performing the following steps is
suitable. However, if you want to stand up a test system as quickly
and simply as possible, you may find it easiest to base your system on
the one the CI team uses. You may use the puppet modules yourself, or
follow the instructions below.
The CI team's Orchestra configuration module is at:
https://github.com/openstack/openstack-ci-puppet/tree/master/modules/orchestra
Install Orchestra
"""""""""""""""""
Install Ubuntu 11.10 (Oneiric) and Orchestra::
sudo apt-get install ubuntu-orchestra-server ipmitool
The install process will prompt you to enter a password for Cobbler.
Have one ready and keep it in a safe place. The procedure here will
not use it, but if you later want to use the Cobbler web interface,
you will need it.
Configure Orchestra
"""""""""""""""""""
Install the following files on the Orchestra server so that it deploys
machines with our LVM/kexec test framework.
We update the dnsmasq.conf cobbler template to add
"dhcp-ignore=tag:!known", and some site-specific network
configuration::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/dnsmasq.template \
-O /etc/cobbler/dnsmasq.template
Our servers need a kernel module blacklisted in order to boot
correctly. If you don't need to blacklist any modules, you should
either create an empty file here, or remove the reference to this file
from the preseed file later::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/openstack_module_blacklist \
-O /var/lib/cobbler/snippets/openstack_module_blacklist
This cobbler snippet uses cloud-init to set up the LVM/kexec
environment and configures TCP syslogging to the installation
server/Jenkins slave::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/openstack_cloud_init \
-O /var/lib/cobbler/snippets/openstack_cloud_init
This snippet holds the mysql root password that will be configured at
install time. It's currently a static string, but you could
dynamically write this file, or simply replace it with something more
secure::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/openstack_mysql_password \
-O /var/lib/cobbler/snippets/openstack_mysql_password
This preseed file manages the OS install on the test nodes. It
includes the snippets installed above::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/openstack-test.preseed \
-O /var/lib/cobbler/kickstarts/openstack-test.preseed
The following sudoers configuration is needed to allow Jenkins to
control cobbler, remove syslog files from the test hosts before
starting new tests, and restart rsyslog::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/orchestra-jenkins-sudoers -O /etc/sudoers.d/orchestra-jenkins
Replace the Orchestra rsyslog config file with a simpler one that logs
all information from remote hosts in one file per host::
wget https://raw.github.com/openstack/openstack-ci-puppet/master/modules/orchestra/files/99-orchestra.conf -O /etc/rsyslog.d/99-orchestra.conf
Make sure the syslog directories exist and restart rsyslog::
mkdir -p /var/log/orchestra/rsyslog/
chown -R syslog.syslog /var/log/orchestra/
restart rsyslog
Add an "OpenStack Test" system profile to cobbler that uses the
preseed file above::
cobbler profile add \
--name=natty-x86_64-ostest \
--parent=natty-x86_64 \
--kickstart=/var/lib/cobbler/kickstarts/openstack-test.preseed \
--kopts="priority=critical locale=en_US"
Add each of your systems to cobbler with a command similar to this
(you may need different kernel options)::
cobbler system add \
--name=baremetal1 \
--hostname=baremetal1 \
--profile=natty-x86_64-ostest \
--mac=00:11:22:33:44:55 \
--power-type=ipmitool \
--power-user=IPMI_USERNAME \
--power-pass=IPMI_PASS \
--power-address=IPMI_IP_ADDR \
--ip-address=SYSTEM_IP_ADDRESS \
--subnet=SYSTEM_SUBNET \
--kopts="netcfg/choose_interface=auto netcfg/dhcp_timeout=60 auto=true priority=critical"
When complete, have cobbler write out its configuration files::
cobbler sync
Set Up Jenkins Jobs
"""""""""""""""""""
We have Jenkins jobs to handle all of the tasks after the initial
Orchestra configuration so that we can easily run them at any time.
This includes the OS installation on the test nodes, even though we
don't run that often because the state is preserved in an LVM
snapshot, we may want to change the configuration used and make a new
snapshot. In that case we just need to trigger the Jenkins job again.
The Jenkins job that kicks off the operating system installation calls
the "baremetal-os-install.sh" script from the openstack-ci repo:
https://github.com/openstack/openstack-ci/blob/master/slave_scripts/baremetal-os-install.sh
That script instructs cobbler to install the OS on each of the test
nodes.
To speed up the devstack installation and avoid excessive traffic to
the pypi server, we build a PIP package cache on the installation
server. That is also an infrequent task that we configure as a
jenkins job. That calls:
https://github.com/openstack/openstack-ci/blob/master/slave_scripts/update-pip-cache.sh
That builds a PIP package cache that the test script later copies to
the test servers for use by devstack.
Run those two jobs, and once complete, the test nodes are ready to go.
This is the end of the operating system installation, and the system
is currently in the pristine state that will be used by the test
procedure (which is stored in the LVM volume "orig_root").
.. _rax_openstack_install:
OpenStack Installation
~~~~~~~~~~~~~~~~~~~~~~
When the deployment and integration test job runs, it does the
following, each time starting from the pristine state arrived at the
end of the previous section.
Reset the Test Nodes
""""""""""""""""""""
The Jenkins deployment and test job first runs the deployment script:
https://github.com/openstack/openstack-ci/blob/master/slave_scripts/baremetal-deploy.sh
Which invokes the following script on each host to reset it to the
pristine state:
https://github.com/openstack/openstack-ci/blob/master/slave_scripts/lvm-kexec-reset.sh
Because kexec is in use, resetting the environment and rebooting into
the pristine state takes only about 3 seconds.
The deployment script then removes the syslog files from the previous
run and restarts rsyslog to re-open them. Once the first test host
finishes booting and brings up its network, OpenStack installation
starts.
Run devstack on the Test Nodes
""""""""""""""""""""""""""""""
Devstack's build_bm_multi script is run, which invokes devstack on
each of the test nodes. First on the "head" node which runs all of
the OpenStack services for the remaining "compute" nodes.
Run Test Suite
""""""""""""""
Once devstack is complete, the test suite is run. All logs from the
test nodes should be sent via syslog to the Jenkins slave, and at the
end of the test, the logs are archived with the Job for developers to
inspect in case of problems.
Cluster Configuration
---------------------
Here are the configuration parameters of the CI team's test cluster.
The cluster is currently divided into three mini-clusters so that
independent Jenkins jobs can run in parallel on the different
clusters.
VLANs
~~~~~
+----+--------------------------------+
|VLAN| Description |
+====+================================+
|90 | Native VLAN |
+----+--------------------------------+
|91 | Internal cluster communication |
| | network: 192.168.91.0/24 |
+----+--------------------------------+
|92 | Public Internet (fake) |
| | network: 192.168.92.0/24 |
+----+--------------------------------+
Servers
~~~~~~~
The servers are located on the Rackspace network, only accessible via
VPN.
+-----------+--------------+---------------+
| Server | Primary IP | Management IP |
+===========+==============+===============+
|deploy-rax | 10.14.247.36 | 10.14.247.46 |
+-----------+--------------+---------------+
|baremetal1 | 10.14.247.37 | 10.14.247.47 |
+-----------+--------------+---------------+
|baremetal2 | 10.14.247.38 | 10.14.247.48 |
+-----------+--------------+---------------+
|baremetal3 | 10.14.247.39 | 10.14.247.49 |
+-----------+--------------+---------------+
|baremetal4 | 10.14.247.40 | 10.14.247.50 |
+-----------+--------------+---------------+
|baremetal5 | 10.14.247.41 | 10.14.247.51 |
+-----------+--------------+---------------+
|baremetal6 | 10.14.247.42 | 10.14.247.52 |
+-----------+--------------+---------------+
|baremetal7 | 10.14.247.43 | 10.14.247.53 |
+-----------+--------------+---------------+
|baremetal8 | 10.14.247.44 | 10.14.247.54 |
+-----------+--------------+---------------+
|baremetal9 | 10.14.247.45 | 10.14.247.55 |
+-----------+--------------+---------------+
deploy-rax
The deployment server and Jenkins slave. It deploys the servers
using Orchestra and Devstack, and runs the test framework. It
should not run any OpenStack components, but we can install
libraries or anything else needed to run tests.
baremetal1, baremetal4, baremetal7
Configured as "head" nodes to run nova, mysql, and glance. Each one
is the head node of a three node cluster including the two compute
nodes following it
baremetal2-3, baremtal5-6, baremetal8-9
Configured as compute nodes for each of the three mini-clusters.

133
doc/jenkins_jobs.rst Normal file
View File

@ -0,0 +1,133 @@
Jenkins Job Builder
===================
Overview
--------
In order to make the process of managing hundreds of Jenkins Jobs easier a
Python based utility was designed to take YAML based configurations and convert
those into jobs that are injected into Jenkins.
Adding a project
----------------
The YAML scripts to make this work are stored in the ``openstack-ci-puppet``
repository in the ``modules/jenkins_jobs/files/projects/site/project.yaml``
directory. Where ``site`` is either `openstack` or `stackforge` and ``project``
is the name of the project the YAML file is for.
Once the YAML file is added the puppet module needs to be told that the project
is there. For example:
.. code-block:: ruby
:linenos:
class { "jenkins_jobs":
site => "stackforge",
projects => ['reddwarf', 'ceilometer']
}
In this example the YAML files for `reddwarf` and `ceilometer` in the
`stackforge` projects directory will be executed.
YAML Format
-----------
The bare minimum YAML needs to look like this:
.. code-block:: yaml
:linenos:
---
modules:
- properties
- scm
- assignednode
- trigger_none
- builders
- publisher_none
main:
name: 'job-name'
site: 'stackforge'
project: 'project'
authenticatedBuild: 'false'
disabled: 'false'
This example starts with ``---``, this signifies the start of a job, there can
be multiple jobs per project file.
The ``modules`` entry is an array of modules that should be loaded for this job.
Modules are located in the ``modules/jenkins_jobs/files/modules/`` directory
and are python scripts to generate the required XML. Each module has a comment
near the top showing the required YAML to support that module. The follow
modules are required to generate a correct XML that Jenkins will support:
* properties (supplies the <properties> XML data)
* scm (supplies the <scm> XML data, required even is scm is not used
* trigger_* (a trigger module is required)
* builders
* publisher_* (a publisher module is required)
Each module also requires a ``main`` section which has the main data for the
modules, inside this there is:
* name - the name of the job
* site - openstack or stackforge
* project - the name of the project
* authenticatedBuild - whether or not you need to be authenticated to hit the
build button
* disabled - whether or not this job should be disabled
Testing for Job Changes
-----------------------
The Jenkins Jobs builder maintains a special YAML file in
``~/.jenkins_jobs_cache.yml``. This contains an MD5 of every generated XML that
it builds. If it finds the XML is different then it will proceed to send this
to Jenkins, otherwise it is skipped. If a job is accidentally deleted then this
file should be modified or removed.
Sending a Job to Jenkins
------------------------
The Jenkins Jobs builder talks to Jenkins using the Jenkins API. This means
that it can create and modify jobs directly without the need to restart or
reload the Jenkins server. It also means that Jenkins will verify the XML and
cause the Jenkins Jobs builder to fail if there is a problem.
For this to work a configuration file is needed. This needs to be stored in
``/root/secret-files/jenkins_jobs.ini`` and puppet will automatically put it in
the right place. The format for this file is as follows:
.. code-block:: ini
[jenkins]
user=username
password=password
url=jenkins_url
The password can be obtained by logging into the Jenkins user, clicking on your
username in the top-right, clicking on `Configure` and then `Show API Token`.
This API Token is your password for the API.
Adding a Module
---------------
Modules need to contain a class with the same name as the filename. The basic
layout is:
.. code-block:: python
import xml.etree.ElementTree as XML
class my_module(object):
def __init__(self, data):
self.data = data
def gen_xml(self, xml_parent):
The ``__init__`` function will be provided with ``data`` which is a Python
dictionary representing the YAML data for the job.
The ``gen_xml`` function will be provided with ``xml_parent`` which is an
XML ElementTree object to be modified.

89
doc/meetbot.rst Normal file
View File

@ -0,0 +1,89 @@
Meetbot
==============
Overview
--------
The OpenStack CI team run a slightly modified
`Meetbot <http://wiki.debian.org/MeetBot>`_ to log IRC channel activity and
meeting minutes. Meetbot is a plugin for
`Supybot <http://sourceforge.net/projects/supybot/>`_ which adds meeting
support features to the Supybot IRC bot.
Supybot
-------
In order to run Meetbot you will need to get Supybot. You can find the latest
release `here <http://sourceforge.net/projects/supybot/files/>`_. Once you have
extracted the release you will want to read the ``INSTALL`` and
``doc/GETTING_STARTED`` files. Those two files should have enough information to
get you going, but there are other goodies in ``doc/``.
Once you have Supybot installed you will need to configure a bot. The
``supybot-wizard`` command can get you started with a basic config, or you can
have Puppet do the heavy lifting. The OpenStack CI Meetbot Puppet module creates
a configuration and documentation for that module is at
:ref:`Meetbot_Puppet_Module`.
One important config setting is ``supybot.reply.whenAddressedBy.chars``, which
sets the prefix character for this bot. This should be set to something other
than ``#`` as ``#`` will conflict with Meetbot (you can leave the setting blank
if you don't want a prefix character).
Meetbot
-------
The OpenStack CI Meetbot fork can be found at
https://github.com/openstack-ci/meetbot. Manual installation of the Meetbot
plugin is straightforward and documented in that repository's README.
OpenStack CI installs and configures Meetbot through Puppet. Documentation for
the Puppet module that does that can be found at :ref:`Meetbot_Puppet_Module`.
Voting
^^^^^^
The OpenStack CI Meetbot fork adds simple voting features. After a meeting has
been started a meeting chair can begin a voting block with the ``#startvote``
command. The command takes two arguments, a question posed to voters (ending
with a ``?``), and the valid voting options. If the second argument is missing
the default options are "Yes" and "No". For example:
``#startvote Should we vote now? Yes, No, Maybe``
Meeting participants vote using the ``#vote`` command. This command takes a
single argument, which should be one of the options listed for voting by the
``#startvote`` command. For example:
``#vote Yes``
Note that you can vote multiple times, but only your last vote will count.
One can check the current vote tallies useing the ``#showvote`` command, which
takes no arguments. This will list the number of votes and voters for each item
that has votes.
When the meeting chair(s) are ready to stop the voting process they can issue
the ``#endvote`` command, which takes no arguments. Doing so will report the
voting results and log these results in the meeting minutes.
A somewhat contrived voting example:
::
foo | #startvote Should we vote now? Yes, No, Maybe
meetbot | Begin voting on: Should we vote now? Valid vote options are Yes, No, Maybe.
meetbot | Vote using '#vote OPTION'. Only your last vote counts.
foo | #vote Yes
bar | #vote Absolutely
meetbot | bar: Absolutely is not a valid option. Valid options are Yes, No, Maybe.
bar | #vote Yes
bar | #showvote
meetbot | Yes (2): foo, bar
foo | #vote No
foo | #showvote
meetbot | Yes (1): bar
meetbot | No (1): foo
foo | #endvote
meetbot | Voted on "Should we vote now?" Results are
meetbot | Yes (1): bar
meetbot | No (1): foo

97
doc/puppet.rst Normal file
View File

@ -0,0 +1,97 @@
Puppet Master
=============
Overview
--------
Instead of using a cron job, StackForge uses a puppet master to host the puppet
manifests and modules. The other nodes then connect to this as puppet agents
to get their configuration.
Puppet Master
-------------
The puppet master is setup using a combination of Apache and mod passenger to
ship the data to the clients. To install this:
.. code-block:: bash
sudo apt-get install puppet puppetmaster puppetmaster-passenger
Note that this may break the first time round due to not-so-perfect packaging
involved. You will also need to stop the puppetmaster service and edit the
``/etc/defaults/puppetmaster`` file to change ``START=no``. Puppetmaster needs
to run first because it creates the SSL CA used to sign puppet agents (the
passenger service does not do this).
This should then allow you to start ``apache2`` which in turn will automatically
manage the puppet master.
Files for puppet master are stored in ``/etc/puppet`` with the subdirectories
``manifests`` and ``modules`` being the important ones. In StackForge we have
a ``root`` cron job that automatically populates these from our puppet git
repository as follows:
.. code-block:: bash
*/15 * * * * sleep $((RANDOM\%600)) && cd /srv/openstack-ci-puppet && /usr/bin/git pull -q && cp /srv/openstack-ci-puppet/manifests/users.pp /etc/puppet/manifests/ && cp /srv/openstack-ci-puppet/manifests/stackforge.pp /etc/puppet/manifests/site.pp && cp -a /srv/openstack-ci-puppet/modules/ /etc/puppet/
Adding a node
-------------
On the new server connecting to the puppet master:
.. code-block:: bash
sudo apt-get install puppet
Then edit the ``/etc/default/puppet`` file to look like this:
.. code-block:: ini
# Defaults for puppet - sourced by /etc/init.d/puppet
# Start puppet on boot?
START=yes
# Startup options
DAEMON_OPTS="--server puppet.stackforge.org"
You can then start the puppet agent with:
.. code-block:: bash
sudo service puppet start
Once the node has started it will make a request to the puppet master to have
its SSL cert signed. On the puppet master:
.. code-block:: bash
sudo puppet cert list
You should get a list of entries similar to the one below::
review.novalocal (44:18:BB:DF:08:50:62:70:17:07:82:1F:D5:70:0E:BF)
If you see the new node there you can sign its cert on the puppet master with:
.. code-block:: bash
sudo puppet cert sign review.novalocal
Now that it is signed the puppet agent will execute any instructions for its
node on the next run (default is every 30 minutes). You can trigger this
earlier by restarting the puppet service on the new node.
Important Notes
---------------
#. The hostname of the nodes **must** match the the forward looking for the DNS.
For example the server pointed to with the DNS entry
``jenkins.stackforge.org`` must have the hostname ``jenkins.stackforge.org``
otherwise the SSL signing or standard run will fail.
#. Make sure the site manifest **does not** include the puppet cron job, this
conflicts with puppet master and can cause issues. The initial puppet run
that create users should be done using the puppet agent configuration above.

276
doc/puppet_modules.rst Normal file
View File

@ -0,0 +1,276 @@
Puppet Modules
==============
Overview
--------
Much of the OpenStack project infrastructure is deployed and managed using
puppet.
The OpenStack CI team manage a number of custom puppet modules outlined in this
document.
Doc Server
----------
The doc_server module configures nginx [3]_ to serve the documentation for
several specified OpenStack projects. At the moment to add a site to this
you need to edit ``modules/doc_server/manifests/init.pp`` and add a line as
follows:
.. code-block:: ruby
:linenos:
doc_server::site { "swift": }
In this example nginx will be configured to serve ``swift.openstack.org``
from ``/srv/docs/swift`` and ``swift.openstack.org/tarballs/`` from
``/srv/tarballs/swift``
Lodgeit
-------
The lodgeit module installs and configures lodgeit [1]_ on required servers to
be used as paste installations. For OpenStack we use
`a fork <https://github.com/openstack-ci/lodgeit>`_ of this which is based on
one with bugfixes maintained by
`dcolish <https://bitbucket.org/dcolish/lodgeit-main>`_ but adds back missing
anti-spam features required by Openstack.
Puppet will configure lodgeit to use drizzle [2]_ as a database backend,
nginx [3]_ as a front-end proxy and upstart scripts to run the lodgeit
instances. It will store and maintain local branch of the the mercurial
repository for lodgeit in ``/tmp/lodgeit-main``.
To use this module you need to add something similar to the following in the
main ``site.pp`` manifest:
.. code-block:: ruby
:linenos:
node "paste.openstack.org" {
include openstack_server
include lodgeit
lodgeit::site { "openstack":
port => "5000",
image => "header-bg2.png"
}
lodgeit::site { "drizzle":
port => "5001"
}
}
In this example we include the lodgeit module which will install all the
pre-requisites for Lodgeit as well as creating a checkout ready.
The ``lodgeit::site`` calls create the individual paste sites.
The name in the ``lodgeit::site`` call will be used to determine the URL, path
and name of the site. So "openstack" will create ``paste.openstack.org``,
place it in ``/srv/lodgeit/openstack`` and give it an upstart script called
``openstack-paste``. It will also change the h1 tag to say "Openstack".
The port number given needs to be a unique port which the lodgeit service will
run on. The puppet script will then configure nginx to proxy to that port.
Finally if an image is given that will be used instead of text inside the h1
tag of the site. The images need to be stored in the ``modules/lodgeit/files``
directory.
Lodgeit Backups
^^^^^^^^^^^^^^^
The lodgeit module will automatically create a git repository in ``/var/backups/lodgeit_db``. Inside this every site will have its own SQL file, for example "openstack" will have a file called ``openstack.sql``. Every day a cron job will update the SQL file (one job per file) and commit it to the git repository.
.. note::
Ideally the SQL files would have a row on every line to keep the diffs stored
in git small, but ``drizzledump`` does not yet support this.
Planet
------
The planet module installs Planet Venus [4]_ along with required dependancies
on a server. It also configures specified planets based on options given.
Planet Venus works by having a cron job which creates static files. In this
module the static files are served using nginx [3]_.
To use this module you need to add something similar to the following into the
main ``site.pp`` manifest:
.. code-block:: ruby
:linenos:
node "planet.openstack.org" {
include planet
planet::site { "openstack":
git_url => "https://github.com/openstack/openstack-planet.git"
}
}
In this example the name "openstack" is used to create the site
``paste.openstack.org``. The site will be served from
``/srv/planet/openstack/`` and the checkout of the ``git_url`` supplied will
be maintained in ``/var/lib/planet/openstack/``.
This module will also create a cron job to pull new feed data 3 minutes past each hour.
The ``git_url`` parameter needs to point to a git repository which stores the
planet.ini configuration for the planet (which stores a list of feeds) and any required theme data. This will be pulled every time puppet is run.
.. _Meetbot_Puppet_Module:
Meetbot
-------
The meetbot module installs and configures meetbot [5]_ on a server. The
meetbot version installed by this module is pulled from the
`Openstack CI fork <https://github.com/openstack-ci/meetbot/>`_ of the project.
It also configures nginix [3]_ to be used for accessing the public IRC logs of
the meetings.
To use this module simply add a section to the site manifest as follows:
.. code-block:: ruby
:linenos:
node "eavesdrop.openstack.org" {
include openstack_cron
class { 'openstack_server':
iptables_public_tcp_ports => [80]
}
include meetbot
meetbot::site { "openstack":
nick => "openstack",
network => "FreeNode",
server => "chat.us.freenode.net:7000",
url => "eavesdrop.openstack.org",
channels => "#openstack #openstack-dev #openstack-meeting",
use_ssl => "True"
}
}
You will also need a file ``/root/secret-files/name-nickserv.pass`` where `name`
is the name specified in the call to the module (`openstack` in this case).
Each call to meetbot::site will create setup a meebot in ``/var/lib/meetbot``
under a subdirectory of the name of the call to the module. It will also
configure nginix to go to that site when the ``/meetings`` directory is
specified on the URL.
The puppet module also creates startup scripts for meetbot and will ensure that
it is running on each puppet run.
Gerrit
------
The Gerrit puppet module configures the basic needs of a Gerrit server. It does
not (yet) install Gerrit itself and mostly deals with the configuration files
and skinning of Gerrit.
Using Gerrit
^^^^^^^^^^^^
Gerrit is set up when the following class call is added to a node in the site
manifest:
.. code-block:: ruby
class { 'gerrit':
canonicalweburl => "https://review.stackforge.org/",
email => "review@stackforge.org",
github_projects => [ {
name => 'stackforge/MRaaS',
close_pull => 'true'
} ],
logo => 'stackforge.png'
}
Most of these options are self-explanitory. The github_projects is a list of
all projects in GitHub which are managed by the gerrit server.
Skinning
^^^^^^^^
Gerrit is skinned using files supplied by the puppet module. The skin is
automatically applied as soon as the module is executed. In the site manifest
setting the logo is important:
.. code-block:: ruby
class { 'gerrit':
...
logo => 'openstack.png'
}
This specifies a PNG file which must be stored in the ``modules/gerrit/files/``
directory.
Jenkins Master
--------------
The Jenkins Master puppet module installs and supplies a basic Jenkins
configuration. It also supplies a skin to Jenkins to make it look more like an
OpenStack site. It does not (yet) install the additional Jenkins plugins used
by the OpenStack project.
Using Jenkins Master
^^^^^^^^^^^^^^^^^^^^
In the site manifest a node can be configured to be a Jenkins master simply by
adding the class call below:
.. code-block:: ruby
class { 'jenkins_master':
site => 'jenkins.openstack.org',
serveradmin => 'webmaster@openstack.org',
logo => 'openstack.png'
}
The ``site`` and ``serveradmin`` parameters are used to configure Apache. You
will also need in this instance the following files for Apache to start::
/etc/ssl/certs/jenkins.openstack.org.pem
/etc/ssl/private/jenkins.openstack.org.key
/etc/ssl/certs/intermediate.pem
The ``jenkins.openstack.org`` is replace by the setting in the ``site``
parameter.
Skinning
^^^^^^^^
The Jenkins skin uses the `Simple Theme Plugin
<http://wiki.jenkins-ci.org/display/JENKINS/Simple+Theme+Plugin>`_ for Jenkins.
The puppet module will install and configure most aspects of the skin
automatically, with a few adjustments needed.
In the site.pp file the ``logo`` parameter is important:
.. code-block:: ruby
class { 'jenkins_master':
...
logo => 'openstack.png'
}
This relates to a PNG file that must be in the ``modules/jenkins_master/files/``
directory.
Once puppet installs this and the plugin is installed you need to go into
``Manage Jenkins -> Configure System`` and look for the ``Theme`` heading.
Assuming we are skinning the main OpenStack Jenkins site, in the ``CSS`` box
enter
``https://jenkins.openstack.org/plugin/simple-theme-plugin/openstack.css`` and
in the ``JS`` box enter
``https://jenkins.openstack.org/plugin/simple-theme-plugin/openstack.js``.
.. rubric:: Footnotes
.. [1] `Lodgeit homepage <http://www.pocoo.org/projects/lodgeit/>`_
.. [2] `Drizzle homepage <http://www.drizzle.org/>`_
.. [3] `nginx homepage <http://nginx.org/en/>`_
.. [4] `Planet Venus homepage <http://intertwingly.net/code/venus/docs/index.html>`_
.. [5] `Meetbot homepage <http://wiki.debian.org/MeetBot>`_

41
doc/stackforge.rst Normal file
View File

@ -0,0 +1,41 @@
HOWTO: Add a Project to StackForge
==================================
Overview
--------
StackForge is a Gerrit review and Jenkins CI setup similar to that of the main
OpenStack project but for use with projects that are not under the main
OpenStack umbrella.
Any project can be added to StackForge as long as it is related to OpenStack in
some way.
Launchpad
---------
All the developers of the project need to sign up to Launchpad and a team is
needed for the core project reviewers to join. This team also needs to be
a sub-team of the `OpenStack team <https://launchpad.net/~openstack>`_ so that
Gerrit will be able to see it.
GitHub
------
If you already have a branch on GitHub for the project this will need moving to
the StackForge GitHub organization. Otherwise a new branch will need creating
for you. The OpenStack Core Infrastructure team can assist in this.
Jenkins and Gerrit
------------------
Until the setup is more automated the OpenStack Core Infrastructure team will
need to do the Jenkins and Gerrit portion of the setup too. If you project is
Python based we have a `Project Testing Interface <http://wiki.openstack.org/ProjectTestingInterface>`_ that we prefer you use. Otherwise please let the CI
team know the testing requirements for Jenkins.
Contacting the CI Team
----------------------
The best way to get the CI team to help with the above steps is to `file a CI bug <https://bugs.launchpad.net/openstack-ci>`_. We are also available on the
#openstack-infra IRC channel or to the `CI Admins email address <mailto:openstack-ci-admins@lists.launchpad.net>`_.

77
doc/systems.rst Normal file
View File

@ -0,0 +1,77 @@
:title: Infrastructure Systems
Infrastructure Systems
######################
The OpenStack CI team maintains a number of systems that are critical
to the operation of the OpenStack project. At the time of writing,
these include:
* Gerrit (review.openstack.org)
* Jenkins (jenkins.openstack.org)
* community.openstack.org
Additionally the team maintains the project sites on Launchpad and
GitHub. The following policies have been adopted to ensure the
continued and secure operation of the project.
SSH Access
**********
For any of the systems managed by the CI team, the following practices
must be observed for SSH access:
* SSH access is only permitted with SSH public/private key
authentication.
* Users must use a strong passphrase to protect their private key. A
passphrase of several words, at least one of which is not in a
dictionary is advised, or a random string of at least 16
characters.
* To mitigate the inconvenience of using a long passphrase, users may
want to use an SSH agent so that the passphrase is only requested
once per desktop session.
* Users private keys must never be stored anywhere except their own
workstation(s). In particular, they must never be stored on any
remote server.
* If users need to 'hop' from a server or bastion host to another
machine, they must not copy a private key to the intermediate
machine (see above). Instead SSH agent forwarding may be used.
However due to the potential for a compromised intermediate machine
to ask the agent to sign requests without the users knowledge, in
this case only an SSH agent that interactively prompts the user
each time a signing request (ie, ssh-agent, but not gnome-keyring)
is received should be used, and the SSH keys should be added with
the confirmation constraint ('ssh-add -c').
* The number of SSH keys that are configured to permit access to
OpenStack machines should be kept to a minimum.
* OpenStack CI machines must use puppet to centrally manage and
configure user accounts, and the SSH authorized_keys files from the
openstack-ci-puppet repository.
* SSH keys should be periodically rotated (at least once per year).
During rotation, a new key can be added to puppet for a time, and
then the old one removed.
GitHub Access
*************
To ensure that code review and testing are not bypassed in the public
Git repositories, only Gerrit will be permitted to commit code to
OpenStack repositories. Because GitHub always allows project
administrators to commit code, accounts that have access to manage the
GitHub projects necessarily will have commit access to the
repositories. Therefore, to avoid inadvertent commits to the public
repositories, unique administrative-only accounts must be used to
manage the OpenStack GitHub organization and projects. These accounts
will not be used to check out or commit code for any project.
Launchpad Teams
***************
Each OpenStack project should have the following teams on Launchpad:
* foo -- contributors to project 'foo'
* foo-core -- core developers
* foo-bugs -- people interested in receieving bug reports
* foo-drivers -- people who may approve and target blueprints
The openstack-admins team should be a member of each of those teams.

153
doc/third_party.rst Normal file
View File

@ -0,0 +1,153 @@
HOWTO: Third Party Testing
==========================
Overview
--------
Gerrit has an event stream which can be subscribed to, using this it is possible
to test commits against testing systems beyond those supplied by OpenStack's
Jenkins setup. It is also possible for these systems to feed information back
into Gerrit and they can also leave non-gating votes on Gerrit review requests.
An example of one such system is `Smokestack <http://smokestack.openstack.org/>`_.
Smokestack reads the Gerrit event stream and runs it's own tests on the commits.
If one of the tests fails it will publish information and links to the failure
on the review in Gerrit.
Reading the Event Stream
------------------------
It is possible to use ssh to connect to ``review.openstack.org`` on port 29418
with your ssh key if you are signed up as an OpenStack developer on Launchpad.
This will give you a real-time JSON stream of events happening inside Gerrit.
For example:
.. code-block:: bash
$ ssh -p 29418 review.example.com gerrit stream-events
Will give a stream with an output like this (line breaks and indentation added
in this document for readability, the read JSON will be all one line per event):
.. code-block:: javascript
{"type":"comment-added","change":
{"project":"openstack/keystone","branch":"stable/essex","topic":"bug/969088","id":"I18ae38af62b4c2b2423e20e436611fc30f844ae1","number":"7385","subject":"Make import_nova_auth only create roles which don\u0027t already exist","owner":
{"name":"Chuck Short","email":"chuck.short@canonical.com","username":"zulcss"},"url":"https://review.openstack.org/7385"},
"patchSet":
{"number":"1","revision":"aff45d69a73033241531f5e3542a8d1782ddd859","ref":"refs/changes/85/7385/1","uploader":
{"name":"Chuck Short","email":"chuck.short@canonical.com","username":"zulcss"},
"createdOn":1337002189},
"author":
{"name":"Mark McLoughlin","email":"markmc@redhat.com","username":"markmc"},
"approvals":
[{"type":"CRVW","description":"Code Review","value":"2"},{"type":"APRV","description":"Approved","value":"0"}],
"comment":"Hmm, I actually thought this was in Essex already.\n\nIt\u0027s a pretty annoying little issue for folks migrating for nova auth. Fix is small and pretty safe. Good choice for backporting"}
For most purposes you will want to trigger on ``patchset-created`` for when a
new patchset has been uploaded.
Further documentation on how to use the events stream can be found in `Gerrit's stream event documentation page <http://gerrit-documentation.googlecode.com/svn/Documentation/2.3/cmd-stream-events.html>`_.
Posting Result To Gerrit
------------------------
External testing systems can give non-gating votes to Gerrit by means of a -1/+1
verify vote. OpenStack Jenkins has extra permissions to give a +2/-2 verify
vote which is gating. Comments should also be provided to explain what kind of
test failed.. We do also ask that the comments contain public links to the
failure so that the developer can see what caused the failure.
An example of how to post this is as follows:
.. code-block:: bash
$ ssh -p 29418 review.example.com gerrit review -m '"Test failed on MegaTestSystem <http://megatestsystem.org/tests/1234>"' --verified=-1 c0ff33
In this example ``c0ff33`` is the commit ID for the review. You can set the
verified to either `-1` or `+1` depending on whether or not it passed the tests.
Further documentation on the `review` command in Gerrit can be found in the `Gerrit review documentation page <http://gerrit-documentation.googlecode.com/svn/Documentation/2.3/cmd-review.html>`_.
We do suggest cautious testing of these systems and have a development Gerrit
setup to test on if required. In SmokeStack's case all failures are manually
reviewed before getting pushed to OpenStack, whilst this may no scale it is
advisable during initial testing of the setup.
.. _request-account-label:
Requesting a Service Account
----------------------------
To request a sevice acconut for your system you first need to create a new
account in LaunchPad. This account needs to be joined to the
`OpenStack Team <https://launchpad.net/~openstack>`_ or one of the related teams
so that Gerrit can pick it up. You can then contact the
OpenStack CI Admins via `email <mailto:openstack-ci-admins@lists.launchpad.net>`_
or the #openstack-infra IRC channel. We will set things up on Gerrit to
receive your system's votes.
Feel free to contact the CI team to arrange setting up a dedicated user so your
system can post reviews up using a system name rather than your user name.
The Jenkins Gerrit Trigger Plugin Way
-------------------------------------
There is a Gerrit Trigger plugin for Jenkins which automates all of the
processes described in this document. So if your testing system is Jenkins
based you can use it to simplify things. You will still need an account to do
this as described in the :ref:`request-account-label` section above.
The OpenStack version of the Gerrit Trigger plugin for Jenkins can be found on
`the Jenkins packaging job <https://jenkins.openstack.org/view/All/job/gerrit-trigger-plugin-package/lastSuccessfulBuild/artifact/gerrithudsontrigger/target/gerrit-trigger.hpi>`_ for it. You can install it using the Advanced tab in the
Jenkins Plugin Manager.
Once installed Jenkins will have a new `Gerrit Trigger` option in the `Manage
Jenkins` menu. This should be given the following options::
Hostname: review.openstack.org
Frontend URL: https://review.openstack.org/
SSH Port: 29418
Username: (the Launchpad user)
SSH Key File: (path to the user SSH key)
Verify
------
Started: 0
Successful: 1
Failed: -1
Unstable: 0
Code Review
-----------
Started: 0
Successful: 0
Failed: 0
Unstable: 0
(under Advanced Button):
Stated: (blank)
Successful: gerrit approve <CHANGE>,<PATCHSET> --message 'Build Successful <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW> --submit
Failed: gerrit approve <CHANGE>,<PATCHSET> --message 'Build Failed <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW>
Unstable: gerrit approve <CHANGE>,<PATCHSET> --message 'Build Unstable <BUILDS_STATS>' --verified <VERIFIED> --code-review <CODE_REVIEW>
Note that it is useful to include something in the messages about what testing
system is supplying these messages.
When creating jobs in Jenkins you will have the option to add triggers. You
should configure as follows::
Trigger on Patchset Uploaded: ticked
(the rest unticked)
Type: Plain
Pattern: openstack/project-name (where project-name is the name of the project)
Branches:
Type: Path
Pattern: **
This job will now automatically trigger when a new patchset is uploaded and will
report the results to Gerrit automatically.

189
modules/jenkins_slave/files/slave_scripts/tardiff.py Executable file
View File

@ -0,0 +1,189 @@
#!/usr/bin/python
# tardiff.py -- compare the tar package with git archive. Error out if
# it's different. The files to exclude are stored in a file, one per line,
# and it's passed as argument to this script.
#
# You should run this script from the project directory. For example, if
# you are verifying the package for glance project, you should run this
# script from that directory.
import getopt
import sys
import os
import commands
class OpenStackTarDiff:
""" main class to verify tar generated in each openstack projects """
def __init__(self):
self.init_vars()
self.validate_args()
self.check_env()
def check_env(self):
""" exit if dist/ directory already exists """
if not self.package and os.path.exists(self.dist_dir):
self.error("dist directory '%s' exist. Please remove it before " \
"running this script" % self.dist_dir)
def validate_args(self):
try:
opts = getopt.getopt(sys.argv[1:], 'hvp:e:',
['help', 'verbose', 'package=',
'exclude='])[0]
except getopt.GetoptError:
self.usage('invalid option selected')
for opt, value in opts:
if (opt in ('-h', '--help')):
self.usage()
elif (opt in ('-e', '--exclude')):
self.e_file = value
elif (opt in ('-p', '--package')):
self.package = value
elif (opt in ('-v', '--verbose')):
self.verbose = True
else:
self.usage('unknown option : ' + opt)
if not self.e_file:
self.usage('specify file name containing list of files to '
'exclude in tar diff')
if not os.path.exists(self.e_file):
self.usage("file '%s' does not exist" % self.e_file)
if self.package and not os.path.exists(self.package):
self.usage("package '%s' specified, but does not "
"exist" % self.package)
def init_vars(self):
self.dist_dir = 'dist/'
self.verbose = False
self.e_file = None
self.project_name = None
self.prefix = None
self.package = None
self.sdist_files = []
self.exclude_files = []
self.git_files = []
self.missing_files = []
def verify(self):
self.get_exclude_files()
self.get_project_name()
self.get_sdist_files()
self.prefix = self.sdist_files[0]
self.get_git_files()
for file in self.git_files:
if os.path.basename(file) in self.exclude_files:
self.debug("excluding file '%s'" % file)
continue
if file not in self.sdist_files:
self.missing_files.append(file)
else:
#self.debug("file %s matches" % file)
pass
if len(self.missing_files) > 0:
self.error("files missing in package: %s" % self.missing_files)
print "SUCCESS: Generated package '%s' is valid" % self.package
def get_project_name(self):
""" get git project name """
self.project_name = os.path.basename(os.path.abspath(os.curdir))
def get_exclude_files(self):
""" read the file and get file list """
fh = open(self.e_file, 'r')
content = fh.readlines()
fh.close()
self.debug("files to exclude: %s" % content)
# remove trailing new lines.
self.exclude_files = [x.strip() for x in content]
def get_git_files(self):
""" read file list from git archive """
git_tar = os.path.join(os.getcwd(), '%s.tar' % self.project_name)
try:
a_cmd = "git archive -o %s HEAD --prefix=%s" % \
(git_tar, self.prefix)
self.debug("executing command '%s'" % a_cmd)
(status, out) = commands.getstatusoutput(a_cmd)
if status != 0:
self.debug("command '%s' returned status '%s'" %
(a_cmd, status))
if os.path.exists(git_tar):
os.unlink(git_tar)
self.error('git archive failed: %s' % out)
except Exception, err:
if os.path.exists(git_tar):
os.unlink(git_tar)
self.error('git archive failed: %s' % err)
try:
tar_cmd = "tar tf %s" % git_tar
self.debug("executing command '%s'" % tar_cmd)
(status, out) = commands.getstatusoutput(tar_cmd)
if status != 0:
self.error('invalid tar file: %s' % git_tar)
self.git_files = out.split('\n')
self.debug("Removing git archive ... %s ..." % git_tar)
os.remove(git_tar)
except Exception, err:
self.error('unable to read tar: %s' % err)
def get_sdist_files(self):
""" create package for project and get file list in it"""
if not self.package:
try:
sdist_cmd = "python setup.py sdist"
self.debug("executing command '%s'" % sdist_cmd)
(status, out) = commands.getstatusoutput(sdist_cmd)
if status != 0:
self.error("command '%s' failed" % sdist_cmd)
except Exception, err:
self.error("command '%s' failed" % (sdist_cmd, err))
self.package = os.listdir(self.dist_dir)[0]
self.package = os.path.join(self.dist_dir, self.package)
tar_cmd = "tar tzf %s" % self.package
try:
self.debug("executing command '%s'" % tar_cmd)
(status, out) = commands.getstatusoutput(tar_cmd)
if status != 0:
self.error("command '%s' failed" % tar_cmd)
#self.debug(out)
self.sdist_files = out.split('\n')
except Exception, err:
self.error("command '%s' failed: %s" % (tar_cmd, err))
def debug(self, msg):
if self.verbose:
sys.stdout.write('DEBUG: %s\n' % msg)
def error(self, msg):
sys.stderr.write('ERROR: %s\n' % msg)
sys.exit(1)
def usage(self, msg=None):
if msg:
stream = sys.stderr
else:
stream = sys.stdout
stream.write("usage: %s [--help|h] [-v] "
"[-p|--package=sdist_package.tar.gz] "
"-e|--exclude=filename\n" \
% os.path.basename(sys.argv[0]))
if msg:
stream.write("\nERROR: " + msg + "\n")
exitCode = 1
else:
exitCode = 0
sys.exit(exitCode)
if __name__ == '__main__':
tardiff = OpenStackTarDiff()
tardiff.verify()

21
setup.py Normal file
View File

@ -0,0 +1,21 @@
import datetime
from setuptools import setup
from sphinx.setup_command import BuildDoc
ci_cmdclass={}
class local_BuildDoc(BuildDoc):
def run(self):
for builder in ['html', 'man']:
self.builder = builder
self.finalize_options()
BuildDoc.run(self)
ci_cmdclass['build_sphinx'] = local_BuildDoc
setup(name='nova',
version="%d.%02d" % (datetime.datetime.now().year, datetime.datetime.now().month),
description="OpenStack Continuous Integration Scripts",
author="OpenStack CI Team",
author_email="openstack-ci@lists.launchpad.net",
url="http://launchpad.net/openstack-ci",
cmdclass=ci_cmdclass)