From b2efaa39faed5a7eff20353e081a55c6bf3f3c6d Mon Sep 17 00:00:00 2001
From: Derek Higgins <derekh@redhat.com>
Date: Mon, 22 Jul 2013 17:42:13 +0100
Subject: [PATCH] Sync toci with current devtest

Toci hasn't worked in nearly two weeks, ever since some stuff was refactored,
after this commit toci
1. is now back in sync with devtest in incubator.
2. brings up an overcloud by default (set TOCI_DO_OVERCLOUD=0 to stop a
   undercloud).
3. clones all git repositories to /opt/toci (these arn't refreshed
   between toci runs unless removed).
4. changes some env variables so that the default behaviour is now
   closer to a what a user setting up a dev env would want.
5. removes unused patches and adds some new ones.
6. Adds a bit more documentation.

Change-Id: I19a34b1a400ec646f37310f265d1345bb6882dc3
---
 FAQ.md                                        | 46 +++++++++
 README.md                                     | 28 ++----
 ...er-Allow-heat-admin-sudo-without-tty.patch | 24 +++++
 .../diskimage-builder-Downgrade-dnsmasq.patch | 24 +++++
 ...o-image-elements-0001-heat-log-chown.patch | 31 ------
 ...t-rabbitmq-server-in-case-of-failure.patch | 29 ------
 patches/tripleo-image-elements-metadata.patch | 56 +++++++++++
 ...tor-0001-no-virsh-prealloc-metadata.patch} |  0
 ...a-toci-cache-file-or-use-if-present.patch} | 33 ++++---
 toci.sh                                       | 65 +++++++++----
 toci_cleanup.sh                               |  4 +-
 toci_functions.sh                             | 22 ++++-
 toci_git.sh                                   | 45 ++++++---
 toci_setup.sh                                 | 23 ++++-
 toci_test.sh                                  | 95 ++++++++++++++++---
 15 files changed, 381 insertions(+), 144 deletions(-)
 create mode 100644 FAQ.md
 create mode 100644 patches/diskimage-builder-Allow-heat-admin-sudo-without-tty.patch
 create mode 100644 patches/diskimage-builder-Downgrade-dnsmasq.patch
 delete mode 100644 patches/tripleo-image-elements-0001-heat-log-chown.patch
 delete mode 100644 patches/tripleo-image-elements-0002-Restart-rabbitmq-server-in-case-of-failure.patch
 create mode 100644 patches/tripleo-image-elements-metadata.patch
 rename patches/{incubator-0001-no-virsh-prealloc-metadata.patch => tripleo-incubator-0001-no-virsh-prealloc-metadata.patch} (100%)
 rename patches_dev/{diskimage-builder-0001-Save-images-in-a-toci-cache-file-or-use-if-present.patch => diskimage-builder-Save-images-in-a-toci-cache-file-or-use-if-present.patch} (62%)

diff --git a/FAQ.md b/FAQ.md
new file mode 100644
index 000000000..f2b7e0822
--- /dev/null
+++ b/FAQ.md
@@ -0,0 +1,46 @@
+Before Running any commands below ensure you have sourced the toci_env file
+
+    $ cd /opt/toci
+    $ . toci_env
+
+**Q. How can I ssh to the seed vm**
+
+A. $ ssh root@$($TOCI_WORKING_DIR/incubator/scripts/get-vm-ip seed)
+
+**Q. How can I ssh to the undercloud controller**
+
+A. $ ssh heat-admin@192.0.2.2
+
+**Q. How do I list the nodes making up the overcloud**
+
+    $ . undercloudrc
+    $ nova list
+
+**Q. How can I create more bm_poseur nodes and add them to my undercloud**
+
+    $ . undercloudrc
+    #              <cpus> <memory> <disk> <quantity>
+    $ create-nodes  1      1024     10     5
+    $ export MACS=$($TOCI_WORKING_DIR/bm_poseur/bm_poseur get-macs)
+    $ setup-baremetal 1 768 10 all
+
+**Q. How can I create a new image, using the elements fedora, selinux-permissive and stackuser**
+
+A. $ ./diskimage-builder/bin/disk-image-create -a i386 -o myimage fedora selinux-permissive stackuser
+
+**Q. I have a heat template, how can I start a stack on the undercloud using it**
+
+    $ . undercloudrc
+    $ heat stack-create -f /path/to/myheattemplate.yaml mystack
+
+**Q. I want to rerun toci without having to build a new set of images each time.**
+
+A. Yes, toci can apply a patch to disk-imagebuilder that will cause it place build images in a /opt/toci/image_cache, all other runs of toci will use these files instead of building a new one. To use the patch move it into the patches directory where you clone toci to.
+
+    # Toci need to reclone disk-imagebuilder so that the patch is applied
+    $ rm -rf /opt/toci/diskimage-builder
+    $ cp patches_dev/diskimage-builder-0001-Save-images-in-a-toci-cache-file-or-use-if-present.patch  patches
+    $ ./toci.sh
+
+Do not forget to remove images from /opt/toci/image_cache if you change anything that would require a new image
+
diff --git a/README.md b/README.md
index bf775d89e..1e892d9f7 100644
--- a/README.md
+++ b/README.md
@@ -41,30 +41,22 @@ directly to start the setup and tests.
 Using Toci to setup a dev environment
 -------------------------------------
 
-I usualy do this a root, in theory it will also work as a non privilaged user.
+I usually do this as root, in theory it will also work as a non privilaged user.
 
-    $ git clone https://github.com/toci-dev/toci.git
+    $ git clone https://github.com/openstack-infra/tripleo-ci.git
     $ cd toci
     $ vi ~/.toci # Will work without a proxy but a lot slower
     export http_proxy=http://192.168.1.104:8080
-    export https_proxy=http://192.168.1.104:8080
 
-To run toci here is your command, were setting
-TOCI_REMOVE=0 TOCI_CLEANUP=0 so that it doesn't clean up after itself, so befor each run the virsh commands do the cleanup if there are any VM's defined
+To run toci here is your command
 
-    $ for NAME in $(virsh list --name --all ); do virsh destroy $NAME ; virsh undefine --remove-all-storage $NAME ; done
-    $ for NAME in $(virsh vol-list default | grep /var/ | awk '{print $1}' ); do virsh vol-delete --pool default $NAME ; done
-    $ TOCI_REMOVE=0 TOCI_CLEANUP=0 ./toci.sh
+    $ ./toci.sh
 
 Toci will start with a line outputing the working and log directories e.g.
-Starting run Wed  3 Jul 11:46:39 IST 2013 ( /tmp/toci_working_g1Eb2NO /tmp/toci_logs_nGnrhLN )
+Starting run Wed  3 Jul 11:46:39 IST 2013 ( /opt/toci /tmp/toci_logs_nGnrhLN )
 
-On success it echo's 0 to the terminal or 1 on error
-
-Once it ran successfully you should have a running seed node that can be used to start images. also /tmp/toci_working_* can be used as a work directory from which to build/start images e.g.
-
-    $ . toci_env ; export ELEMENTS_PATH=$TOCI_WORKING_DIR/tripleo-image-elements/elements
-    $ TOCI_WORKING_DIR/diskimage-builder/bin/disk-image-create -u -a i386 -o stackuserimage stackuser
-    $ unset http_proxy ; unset https_proxy ; . ~/seedrc
-    $ $TOCI_WORKING_DIR/incubator/scripts/load-image stackuserimage.qcow2
-    $ nova boot --flavor 256 --key_name default stackuserimage --image  stackuserimage
+Once it ran successfully (ERROR wasn't echo'd to the terminal) you should have
+1. seed vm
+2. undercloud vm
+3. overcloud controller vm
+4. overcloud compute vm
diff --git a/patches/diskimage-builder-Allow-heat-admin-sudo-without-tty.patch b/patches/diskimage-builder-Allow-heat-admin-sudo-without-tty.patch
new file mode 100644
index 000000000..4a6ddb158
--- /dev/null
+++ b/patches/diskimage-builder-Allow-heat-admin-sudo-without-tty.patch
@@ -0,0 +1,24 @@
+From 76fb773073a02d9d6c39d0b0150947573c6de4e9 Mon Sep 17 00:00:00 2001
+From: Derek Higgins <derekh@redhat.com>
+Date: Fri, 19 Jul 2013 10:21:27 +0100
+Subject: [PATCH] Allow heat-admin sudo without tty
+
+---
+ elements/fedora/pre-install.d/00-allow-root-sudo | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/elements/fedora/pre-install.d/00-allow-root-sudo b/elements/fedora/pre-install.d/00-allow-root-sudo
+index 4736ac8..ff59cfb 100755
+--- a/elements/fedora/pre-install.d/00-allow-root-sudo
++++ b/elements/fedora/pre-install.d/00-allow-root-sudo
+@@ -5,3 +5,8 @@ set -e
+ echo "Defaults:root !requiretty" >> /etc/sudoers.d/root-notty
+ chmod 0440 /etc/sudoers.d/root-notty
+ visudo -c
++
++# For toci we need heat-admin to sudo without tty
++echo "Defaults:heat-admin !requiretty" >> /etc/sudoers.d/heat-admin-notty
++chmod 0440 /etc/sudoers.d/heat-admin-notty
++visudo -c
+-- 
+1.8.1.4
diff --git a/patches/diskimage-builder-Downgrade-dnsmasq.patch b/patches/diskimage-builder-Downgrade-dnsmasq.patch
new file mode 100644
index 000000000..421c904b5
--- /dev/null
+++ b/patches/diskimage-builder-Downgrade-dnsmasq.patch
@@ -0,0 +1,24 @@
+From 4945650d351e92b248c7104e541189372bd724cd Mon Sep 17 00:00:00 2001
+From: Derek Higgins <derekh@redhat.com>
+Date: Fri, 19 Jul 2013 10:46:43 +0100
+Subject: [PATCH] Downgrade dnsmasq
+
+Temp workaround for
+https://bugzilla.redhat.com/show_bug.cgi?id=977555
+---
+ elements/base/install.d/99-up-to-date | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/elements/base/install.d/99-up-to-date b/elements/base/install.d/99-up-to-date
+index 53f327c..f97124f 100755
+--- a/elements/base/install.d/99-up-to-date
++++ b/elements/base/install.d/99-up-to-date
+@@ -5,3 +5,5 @@
+ set -e
+
+ install-packages -u
++
++yum downgrade -y http://kojipkgs.fedoraproject.org//packages/dnsmasq/2.65/5.fc18/i686/dnsmasq-2.65-5.fc18.i686.rpm http://kojipkgs.fedoraproject.org//packages/dnsmasq/2.65/5.fc18/i686/dnsmasq-utils-2.65-5.fc18.i686.rpm || true
+-- 
+1.8.1.4
+
diff --git a/patches/tripleo-image-elements-0001-heat-log-chown.patch b/patches/tripleo-image-elements-0001-heat-log-chown.patch
deleted file mode 100644
index 689089104..000000000
--- a/patches/tripleo-image-elements-0001-heat-log-chown.patch
+++ /dev/null
@@ -1,31 +0,0 @@
-From b301f0a21e231a725dcc66d46e935f71dd2c4f6f Mon Sep 17 00:00:00 2001
-From: Derek Higgins <derekh@redhat.com>
-Date: Wed, 10 Jul 2013 14:28:06 +0100
-Subject: [PATCH] Ensure /var/log/heat/engine.log is owned by heat
-
-If heat-manage db_sync creates the logfile before the
-heat engine service then it will be owned by root causing
-heat-engine to error.
-
-Fixes bug 1199789
-
-Change-Id: Ib23a0d46ebf354a3293a64215715038497faf5b0
----
- elements/boot-stack/bin/reset-db |    1 +
- 1 file changed, 1 insertion(+)
-
-diff --git a/elements/boot-stack/bin/reset-db b/elements/boot-stack/bin/reset-db
-index d903614..52186fc 100755
---- a/elements/boot-stack/bin/reset-db
-+++ b/elements/boot-stack/bin/reset-db
-@@ -27,6 +27,7 @@ $venvs/glance/bin/glance-manage db_sync
- 
- os-db-create heat heat $db_pass
- $venvs/heat/bin/heat-manage db_sync
-+chown heat:heat /var/log/heat/engine.log
- 
- os-db-create ovs_neutron neutron $db_pass
- 
--- 
-1.7.9.5
-
diff --git a/patches/tripleo-image-elements-0002-Restart-rabbitmq-server-in-case-of-failure.patch b/patches/tripleo-image-elements-0002-Restart-rabbitmq-server-in-case-of-failure.patch
deleted file mode 100644
index 3583f5a66..000000000
--- a/patches/tripleo-image-elements-0002-Restart-rabbitmq-server-in-case-of-failure.patch
+++ /dev/null
@@ -1,29 +0,0 @@
-From b61e548125cae41a3d4203b4d2928856f425e002 Mon Sep 17 00:00:00 2001
-From: Lucas Alvares Gomes <lucasagomes@gmail.com>
-Date: Thu, 4 Jul 2013 11:17:29 +0100
-Subject: [PATCH 1/1] Restart rabbitmq-server in case of failure
-
-Tells systemd to restart the rabbitmq-server in case of failure
-
-Change-Id: If5416ecc636e199d639be2c6bb8777326e7ea659
----
- elements/rabbitmq-server/install.d/20-rabbitmq-server | 3 +++
- 1 file changed, 3 insertions(+)
-
-diff --git a/elements/rabbitmq-server/install.d/20-rabbitmq-server b/elements/rabbitmq-server/install.d/20-rabbitmq-server
-index 2001a08..ab4e56a 100755
---- a/elements/rabbitmq-server/install.d/20-rabbitmq-server
-+++ b/elements/rabbitmq-server/install.d/20-rabbitmq-server
-@@ -23,6 +23,9 @@ if [ $DISTRO = "Fedora" ]; then
-     # Delay the rc-local.service start-up until rabbitmq-server.service is started up
-     sed -i 's/\[Unit\]/\[Unit\]\nBefore=rc-local.service/g' /lib/systemd/system/rabbitmq-server.service
- 
-+    # Respawn rabbitmq-server in case the process exits with an nonzero exit code
-+    sed -i 's/\[Service\]/\[Service\]\nRestart=on-failure/g' /lib/systemd/system/rabbitmq-server.service
-+
-     # Make systemd take notice of it
-     systemctl daemon-reload
- 
--- 
-1.8.1.4
-
diff --git a/patches/tripleo-image-elements-metadata.patch b/patches/tripleo-image-elements-metadata.patch
new file mode 100644
index 000000000..3b0ed24b4
--- /dev/null
+++ b/patches/tripleo-image-elements-metadata.patch
@@ -0,0 +1,56 @@
+From 66ac4f5c5d1511e16180c592dde1b8e4bf385c6f Mon Sep 17 00:00:00 2001
+From: Derek Higgins <derekh@redhat.com>
+Date: Tue, 23 Jul 2013 10:34:45 +0100
+Subject: [PATCH] Ensure metadata is available for os-refresh-config
+
+On Feodra 18 during boot when cron runs os-refresh-config, a race
+condition was present, cron was getting run before the metadata was
+retrieved. As a result in 49-heat-localip the loop that edited the
+metadata was editing nothing.
+
+Change-Id: I1b8048daf42786b9a9008db2062bd62b6a7ee7b9
+---
+ .../os-refresh-config/configure.d/49-heat-localip  |    7 +++++++
+ .../os-refresh-config/install.d/75-cfn-hup-cronjob |   11 +++++++++++
+ 2 files changed, 18 insertions(+)
+
+diff --git a/elements/heat-localip/os-refresh-config/configure.d/49-heat-localip b/elements/heat-localip/os-refresh-config/configure.d/49-heat-localip
+index d5ccb8d..ee5dd39 100755
+--- a/elements/heat-localip/os-refresh-config/configure.d/49-heat-localip
++++ b/elements/heat-localip/os-refresh-config/configure.d/49-heat-localip
+@@ -8,6 +8,13 @@ if [ -z "$local_ip" ]; then
+   # routes.
+   exit 0
+ fi
++
++# We may have to wait for metadata to appear if this script is being started on the first boot
++for i in {0..10} ; do
++    [ -e /var/lib/heat-cfntools/cfn-init-data -o -e /var/cache/heat-cfntools/last_metadata ] && break
++    sleep 2
++done
++
+ for f in /var/lib/heat-cfntools/cfn-init-data /var/cache/heat-cfntools/last_metadata ; do
+     if [ -e $f ] ; then
+         sed -i "s/\<0\.0\.0\.0\>/$local_ip/g" $f
+diff --git a/elements/os-refresh-config/install.d/75-cfn-hup-cronjob b/elements/os-refresh-config/install.d/75-cfn-hup-cronjob
+index c5548fd..69beb08 100755
+--- a/elements/os-refresh-config/install.d/75-cfn-hup-cronjob
++++ b/elements/os-refresh-config/install.d/75-cfn-hup-cronjob
+@@ -10,3 +10,14 @@ PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+ # This is a workaround for cfn-hup not actually being a daemon
+ */5 * * * * root cfn-hup --no-daemon
+ EOF
++
++# If using systemd in order of guarantee metadata is available,
++# ensure cron doesn't come up until after cloud-init has run
++if [ -e /usr/lib/systemd/system/cloud-final.service ] ; then
++    # We are editing a file packaged in cloud-init, so we update 
++    # cloud-init now, if its updated later it will undo our changes
++    yum update -y cloud-init || true
++    # TODO : this should be possible by placing a link in crond.service.wants
++    # but I couldn't get this to work
++    sed -i -e 's/Requires=cloud-config.target/Requires=cloud-config.target\nBefore=crond.service/g' /usr/lib/systemd/system/cloud-final.service
++fi
+-- 
+1.7.9.5
+
diff --git a/patches/incubator-0001-no-virsh-prealloc-metadata.patch b/patches/tripleo-incubator-0001-no-virsh-prealloc-metadata.patch
similarity index 100%
rename from patches/incubator-0001-no-virsh-prealloc-metadata.patch
rename to patches/tripleo-incubator-0001-no-virsh-prealloc-metadata.patch
diff --git a/patches_dev/diskimage-builder-0001-Save-images-in-a-toci-cache-file-or-use-if-present.patch b/patches_dev/diskimage-builder-Save-images-in-a-toci-cache-file-or-use-if-present.patch
similarity index 62%
rename from patches_dev/diskimage-builder-0001-Save-images-in-a-toci-cache-file-or-use-if-present.patch
rename to patches_dev/diskimage-builder-Save-images-in-a-toci-cache-file-or-use-if-present.patch
index f7d2376ff..9451ac215 100644
--- a/patches_dev/diskimage-builder-0001-Save-images-in-a-toci-cache-file-or-use-if-present.patch
+++ b/patches_dev/diskimage-builder-Save-images-in-a-toci-cache-file-or-use-if-present.patch
@@ -1,6 +1,6 @@
-From 5304e2de786bcea1e9074c03421b26d03751e1d6 Mon Sep 17 00:00:00 2001
+From 448fbaa131521e6060f9d87c83098c8cb90696ea Mon Sep 17 00:00:00 2001
 From: Derek Higgins <derekh@redhat.com>
-Date: Fri, 17 May 2013 12:16:09 +0100
+Date: Fri, 19 Jul 2013 13:55:41 +0100
 Subject: [PATCH] Save images in a toci cache file or use if present
 
 FOR DEV ONLY
@@ -9,22 +9,20 @@ o skip building images in place of a cached version,
 o mv first-boot scripts instead of rm
 
 place this file in patches.
-
-Change-Id: Ieb3dbc176de7d7ae8bfefa66e82d2a53c73d2a38
 ---
- bin/disk-image-create        | 10 ++++++++++
- elements/base/dib-first-boot |  2 +-
- 2 files changed, 11 insertions(+), 1 deletion(-)
+ bin/disk-image-create        | 8 ++++++++
+ elements/base/dib-first-boot | 2 +-
+ 2 files changed, 9 insertions(+), 1 deletion(-)
 
 diff --git a/bin/disk-image-create b/bin/disk-image-create
-index d71fa26..4413faa 100755
+index efbcf85..e7fc46f 100755
 --- a/bin/disk-image-create
 +++ b/bin/disk-image-create
-@@ -83,6 +83,13 @@ arg_to_elements "$@"
+@@ -97,6 +97,13 @@ arg_to_elements "$@"
  
  IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPE}}
  
-+CACHEFILE=$TOCI_CACHE_DIR/$(basename $IMAGE_NAME.$IMAGE_TYPE)
++CACHEFILE=$TOCI_WORKING_DIR/image_cache/$(basename $IMAGE_NAME.$IMAGE_TYPE)
 +if [ -e $CACHEFILE ] ; then
 +    echo Using $CACHEFILE
 +    cp $CACHEFILE $IMAGE_NAME.$IMAGE_TYPE
@@ -34,13 +32,14 @@ index d71fa26..4413faa 100755
  mk_build_dir
  create_base
  run_d extra-data
-@@ -127,3 +134,6 @@ finalise_base
- unmount_image
- compress_image
- save_image $IMAGE_NAME.$IMAGE_TYPE
-+
-+cp $IMAGE_NAME.$IMAGE_TYPE $TOCI_CACHE_DIR/$(basename $IMAGE_NAME.$IMAGE_TYPE)
-+
+@@ -144,6 +151,7 @@ unmount_image
+ if [ "$IS_RAMDISK" == "0" ]; then
+   compress_image
+   save_image $IMAGE_NAME.$IMAGE_TYPE
++  cp $IMAGE_NAME.$IMAGE_TYPE $TOCI_WORKING_DIR/image_cache/$(basename $IMAGE_NAME.$IMAGE_TYPE)
+ else
+   remove_image
+ fi
 diff --git a/elements/base/dib-first-boot b/elements/base/dib-first-boot
 index 1e39860..c13a930 100755
 --- a/elements/base/dib-first-boot
diff --git a/toci.sh b/toci.sh
index 7cd37a169..ceaaa429b 100755
--- a/toci.sh
+++ b/toci.sh
@@ -2,37 +2,58 @@
 
 . toci_functions.sh
 
+# Check for some dependencies
+commands=("patch" "make" "tar" "ssh" "arp" "busybox")
+for cmd in "${commands[@]}"; do
+  which "${cmd}" > /dev/null 2>&1 || ERROR "$cmd: command not found"
+done
+
+python -c 'import yaml' > /dev/null 2>&1 || ERROR "Please install PyYAML"
+
+# TODO : why do I need to do this, heat client complains without it
+python -c 'import keystoneclient' || ERROR "Please install python-keystoneclient"
+export PYTHONPATH=$(python -c 'import keystoneclient; print keystoneclient.__file__.rsplit("/", 1)[0]'):$PYTHONPATH
+
 export STARTTIME=$(date)
 export TOCI_SOURCE_DIR=$PWD
 
+# Toci defaults
+[ -e tocirc ] && source tocirc
 # env specific to this run, can contain
 # TOCI_RESULTS_SERVER, http_proxy, TOCI_UPLOAD, TOCI_REMOVE,
-source ~/.toci
+[ -e ~/.toci ] && source ~/.toci
 
 export TOCI_GIT_CHECKOUT
 
-# All temp files should go here
-export TOCI_WORKING_DIR=${TOCI_WORKING_DIR:-$(mktemp -d --tmpdir toci_working_XXXXXXX)}
+# All toci working files should go here
+export TOCI_WORKING_DIR=${TOCI_WORKING_DIR:-/opt/toci}
 mkdir -p $TOCI_WORKING_DIR
-# Any files to be uploaded to results server goes here
+# Any files to be uploaded to results server go here
 export TOCI_LOG_DIR=${TOCI_LOG_DIR:-$(mktemp -d --tmpdir toci_logs_XXXXXXX)}
 mkdir -p $TOCI_LOG_DIR
 # Files that should be cached between runs should go in here
 # e.g. downloaded images, git repo's etc...
 export TOCI_CACHE_DIR=/var/tmp/toci_cache
 
-export TOCI_ARCH=${TOCI_ARCH:-'i686'}
+export TOCI_ARCH=${TOCI_ARCH:-'i386'}
 export TOCI_DIB_ARCH='i386'
 if [ "$TOCI_ARCH" == 'x86_64' ]; then
   export TOCI_DIB_ARCH='amd64'
 fi
+export TRIPLEO_ROOT=$TOCI_WORKING_DIR
 
 export TOCI_DISTROELEMENT=${TOCI_DISTROELEMENT:-'fedora selinux-permissive'}
 
+# Are we going to set up an overcloud
+export TOCI_DO_OVERCLOUD=${TOCI_DO_OVERCLOUD:-1}
+
 RESULT_CACHE=$TOCI_CACHE_DIR/results_cache.html
 
 echo "Starting run $STARTTIME ( $TOCI_WORKING_DIR $TOCI_LOG_DIR )"
 
+# On Exit write relevant toci env to a rc file
+trap get_tocienv EXIT
+
 # If running in cron $USER isn't setup
 export USER=${USER:-$(whoami)}
 
@@ -41,22 +62,35 @@ mkdir -p $TOCI_CACHE_DIR
 STATUS=0
 
 mark_time Starting git
-timeout --foreground 60m ./toci_git.sh > $TOCI_LOG_DIR/git.out 2>&1 || STATUS=1
+./toci_git.sh > $TOCI_LOG_DIR/git.out 2>&1 || STATUS=1
+
+
+# set d-i-b env variables to fetch git repositories from local caches
+for repo in $TOCI_WORKING_DIR/*/.git ; do
+    repo_dir=$(dirname $repo)
+    repo_name=$(basename $repo_dir)
+    if [[ "^(tripleo-incubator|bm_poseur|diskimage-builder|tripleo-image-elements|tripleo-heat-templates)$" =~ "$repo_name" ]] ; then
+        continue
+    fi
+    export DIB_REPOLOCATION_$repo_name=$repo_dir
+done
 
 # Add incubator scripts to path
-export PATH=$PATH:$TOCI_WORKING_DIR/incubator/scripts
+export PATH=$PATH:$TOCI_WORKING_DIR/tripleo-incubator/scripts
 
+mark_time Starting pre-cleanup
+./toci_cleanup.sh > $TOCI_LOG_DIR/cleanup.out 2>&1
 if [ $STATUS == 0 ] ; then
   mark_time Starting setup
-  timeout --foreground 60m ./toci_setup.sh > $TOCI_LOG_DIR/setup.out 2>&1 || STATUS=1
+  ./toci_setup.sh > $TOCI_LOG_DIR/setup.out 2>&1 || STATUS=1
 fi
 if [ $STATUS == 0 ] ; then
     mark_time Starting test
-    timeout --foreground 60m ./toci_test.sh > $TOCI_LOG_DIR/test.out 2>&1 || STATUS=1
+    ./toci_test.sh > $TOCI_LOG_DIR/test.out 2>&1 || STATUS=1
 fi
-if [ ${TOCI_CLEANUP:-1} == 1 ] ; then
+if [ ${TOCI_CLEANUP:-0} == 1 ] ; then
     mark_time Starting cleanup
-    timeout --foreground 60m ./toci_cleanup.sh > $TOCI_LOG_DIR/cleanup.out 2>&1 || STATUS=1
+    ./toci_cleanup.sh >> $TOCI_LOG_DIR/cleanup.out 2>&1 || STATUS=1
 fi
 mark_time Finished
 
@@ -87,12 +121,11 @@ fi
         send_irc $TOCI_IRC ERROR during toci run, see http://$TOCI_RESULTS_SERVER/toci/$(basename $TOCI_LOG_DIR)/
     fi
 
-if [ ${TOCI_REMOVE:-1} == 1 ] ; then
+if [ ${TOCI_REMOVE:-0} == 1 ] ; then
     rm -rf $TOCI_WORKING_DIR $TOCI_LOG_DIR
 fi
 
-declare | grep -e "^PATH=" -e "^http.*proxy" -e "^TOCI_" -e '^DIB_' | sed  -e 's/^/export /g' > $TOCI_WORKING_DIR/toci_env
-echo 'export no_proxy=$($TOCI_WORKING_DIR/incubator/scripts/get-vm-ip seed)' >> $TOCI_WORKING_DIR/toci_env
-
-echo $STATUS
+if [ $STATUS != 0 ] ; then
+    echo ERROR
+fi
 exit $STATUS
diff --git a/toci_cleanup.sh b/toci_cleanup.sh
index 0bb7c054c..dd2efcdbb 100755
--- a/toci_cleanup.sh
+++ b/toci_cleanup.sh
@@ -2,11 +2,11 @@
 
 set -x
 
-for NAME in $(sudo virsh list --name --all | grep "^\(seed\|bootstrap\|baremetal_[0-9]\)$"); do
+for NAME in $(sudo virsh list --name --all | grep "^\(seed\|bootstrap\|baremetal_.*\)$"); do
   sudo virsh destroy $NAME
   sudo virsh undefine --remove-all-storage $NAME
 done
 
-for NAME in $(virsh vol-list default | grep /var/ | awk '{print $1}' | grep "^\(seed\|bootstrap\|baremetal-[0-9]\)" ); do
+for NAME in $(virsh vol-list default | grep /var/ | awk '{print $1}' | grep "^\(seed\|bootstrap\|baremetal-\)" ); do
   sudo virsh vol-delete --pool default $NAME
 done
diff --git a/toci_functions.sh b/toci_functions.sh
index e4897a01d..4221504a1 100644
--- a/toci_functions.sh
+++ b/toci_functions.sh
@@ -1,24 +1,24 @@
 #!/usr/bin/env bash
 
 get_get_repo(){
-    CACHEDIR=$TOCI_CACHE_DIR/${1/[^\/]*\//}
+    CACHEDIR=$TOCI_WORKING_DIR/${1/[^\/]*\//}
     if [ ! -e $CACHEDIR ] ; then
         git clone https://github.com/$1.git $CACHEDIR
+        repo_basename=${1#*/}
+        apply_patches ${repo_basename} ${repo_basename}*
     else
         pushd $CACHEDIR
         git fetch
-        git reset --hard origin/master
         popd
     fi
-    cp -r $CACHEDIR $TOCI_WORKING_DIR/${1/[^\/]*\//}
 }
 
 ssh_noprompt(){
-    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=QUIET $@
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=QUIET -o PasswordAuthentication=no $@
 }
 
 scp_noprompt(){
-    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=QUIET $@
+    scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=QUIET -o PasswordAuthentication=no $@
 }
 
 wait_for(){
@@ -55,6 +55,13 @@ get_state_from_host(){
                                      tar -czf - --exclude=udev/hwdb.bin --exclude=selinux/targeted /var/log /etc || true" > $TOCI_LOG_DIR/bootstraplogs.tgz
 }
 
+# On Exit write relevant toci env to a rc file
+get_tocienv(){
+    declare | grep -e "^PATH=" -e "^http.*proxy" -e "^TOCI_" -e '^DIB_' | sed  -e 's/^/export /g' > $TOCI_WORKING_DIR/toci_env
+    # Some IP we sont want to proxy
+    echo 'export no_proxy=$($TOCI_WORKING_DIR/tripleo-incubator/scripts/get-vm-ip seed),192.0.2.2,192.0.2.5,192.0.2.6,192.0.2.7,192.0.2.8' >> $TOCI_WORKING_DIR/toci_env
+}
+
 # Sends a message to a freenode irc channel
 send_irc(){
     exec 3<>/dev/tcp/irc.freenode.net/6667
@@ -72,3 +79,8 @@ send_irc(){
 
     cat <&3 > /dev/null
 }
+
+ERROR(){
+    echo $@
+    exit 1
+}
diff --git a/toci_git.sh b/toci_git.sh
index 005b688b2..39e23e2cb 100755
--- a/toci_git.sh
+++ b/toci_git.sh
@@ -4,7 +4,7 @@ set -xe
 . toci_functions.sh
 
 # Get the tripleO repo's
-for repo in 'tripleo/incubator' 'tripleo/bm_poseur' 'stackforge/diskimage-builder' 'stackforge/tripleo-image-elements' 'stackforge/tripleo-heat-templates' ; do
+for repo in 'openstack/tripleo-incubator' 'tripleo/bm_poseur' 'stackforge/diskimage-builder' 'stackforge/tripleo-image-elements' 'stackforge/tripleo-heat-templates' ; do
     if [ ${TOCI_GIT_CHECKOUT:-1} == 1 ] ; then
       get_get_repo $repo
     else
@@ -14,13 +14,36 @@ for repo in 'tripleo/incubator' 'tripleo/bm_poseur' 'stackforge/diskimage-builde
     fi
 done
 
-#only patch if we do the git checkout
-if [ ${TOCI_GIT_CHECKOUT:-1} == 1 ] ; then
-  # patches can be added to git repo's like this, this just a temp measure we need to make faster progress
-  # until we get up and runing properly
-  apply_patches incubator incubator*
-  apply_patches bm_poseur bm_poseur*
-  apply_patches diskimage-builder diskimage-builder*
-  apply_patches tripleo-image-elements tripleo-image-elements*
-  apply_patches tripleo-heat-templates tripleo-heat-templates*
-fi
+# Get a local copy of each of the git repositories  referenced in
+REGEX="^([^ ]+) (git|tar) ([/~][^ ]+) ([^ ]+) ?([^ ]*)$"
+for sr in $TOCI_WORKING_DIR/*/elements/*/source-repository* ; do
+    while read line ; do
+        # ignore blank lines and lines begining in '#'
+        [[ "$line" == \#* ]] || [[ -z "$line" ]] && continue
+        if [[ "$line" =~ $REGEX ]]  ; then
+            REPONAME=${BASH_REMATCH[1]//-/_}
+            REPOTYPE=${BASH_REMATCH[2]}
+            REPOLOCATION=${BASH_REMATCH[4]}
+            REPOREF=${BASH_REMATCH[5]:-master}
+
+            REPOREF_OVERRIDE=TOCI_REPOREF_$REPONAME
+            REPOREF=${!REPOREF_OVERRIDE:-$REPOREF}
+
+            REPO_DIRECTORY=$TOCI_WORKING_DIR/$REPONAME
+
+            if [ $REPOTYPE = git ] ; then
+                if [ ! -e $REPO_DIRECTORY ] ; then
+                    git clone $REPOLOCATION $REPO_DIRECTORY
+                    pushd $REPO_DIRECTORY
+                    git reset --hard $REPOREF
+                    popd
+                fi
+            else
+                echo "Unsupported repository type"
+            fi
+        else
+            echo "Couldn't parse '$line' as a source repository"
+            return 1
+        fi
+    done < $sr
+done
diff --git a/toci_setup.sh b/toci_setup.sh
index e3bac7ef8..36dee523c 100755
--- a/toci_setup.sh
+++ b/toci_setup.sh
@@ -5,6 +5,9 @@ set -xe
 
 cd $TOCI_WORKING_DIR
 
+# Were going to cache images here
+mkdir -p $TOCI_WORKING_DIR/image_cache
+
 # install deps on host machine
 install-dependencies
 setup-network
@@ -31,7 +34,23 @@ if [ -n "$TOCI_PM_DRIVER" ]; then
 fi
 
 sed -i "s/\"user\": \"stack\",/\"user\": \"`whoami`\",/" $TOCI_WORKING_DIR/tripleo-image-elements/elements/seed-stack-config/config.json
-EXTRA_ELEMENTS=$TOCI_DISTROELEMENT $TOCI_WORKING_DIR/incubator/scripts/boot-seed-vm
 
+# Create a deployment ramdisk + kernel
+#$TOCI_WORKING_DIR/diskimage-builder/bin/ramdisk-image-create -x -a $TOCI_DIB_ARCH $TOCI_DISTROELEMENT deploy -o deploy-ramdisk
+$TOCI_WORKING_DIR/diskimage-builder/bin/ramdisk-image-create -x -a $TOCI_DIB_ARCH ubuntu deploy -o deploy-ramdisk
+
+
+# If using Fedora keep using F18 for now
+if [[ "$TOCI_DISTROELEMENT" =~ fedora ]] ; then
+    export DIB_CLOUD_IMAGES=http://mattdm.fedorapeople.org/cloud-images
+    export DIB_RELEASE=Fedora18
+    export BASE_IMAGE_FILE=Fedora18-Cloud-$TOCI_ARCH-latest.qcow2
+fi
+
+# Boot a seed vm
+EXTRA_ELEMENTS=$TOCI_DISTROELEMENT $TOCI_WORKING_DIR/tripleo-incubator/scripts/boot-seed-vm
+
+# Make the tripleo image elements accessible to diskimage-builder
 export ELEMENTS_PATH=$TOCI_WORKING_DIR/diskimage-builder/elements:$TOCI_WORKING_DIR/tripleo-image-elements/elements
-$TOCI_WORKING_DIR/diskimage-builder/bin/disk-image-create -u -a $TOCI_DIB_ARCH -o $TOCI_WORKING_DIR/notcompute $TOCI_DISTROELEMENT stackuser boot-stack heat-cfntools neutron-network-node
+
+$TOCI_WORKING_DIR/diskimage-builder/bin/disk-image-create -a $TOCI_DIB_ARCH -o $TOCI_WORKING_DIR/undercloud $TOCI_DISTROELEMENT boot-stack nova-baremetal heat-localip heat-cfntools stackuser
diff --git a/toci_test.sh b/toci_test.sh
index df631995e..28ccff7d6 100755
--- a/toci_test.sh
+++ b/toci_test.sh
@@ -4,7 +4,7 @@ set -xe
 . toci_functions.sh
 
 cd $TOCI_WORKING_DIR
-SEED_IP=`$TOCI_WORKING_DIR/incubator/scripts/get-vm-ip seed`
+SEED_IP=`$TOCI_WORKING_DIR/tripleo-incubator/scripts/get-vm-ip seed`
 
 # Get logs from the node on exit
 trap get_state_from_host EXIT
@@ -15,13 +15,12 @@ source $TOCI_WORKING_DIR/seedrc
 
 export no_proxy=$no_proxy,$SEED_IP
 
-# wait for a successfull os-refresh-config
-wait_for 60 10 ssh_noprompt root@$SEED_IP ls /opt/stack/boot-stack.ok
+# wait for a successful os-refresh-config
+wait_for 60 10 ssh_noprompt root@$SEED_IP ls /opt/stack/boot-stack/init-neutron.ok
 
-nova list
-
-#Adds nova keypair
-user-config
+# Make sure nova has had a chance to start responding to requests
+wait_for 10 5 nova list
+user-config #Adds nova keypair
 
 if [ -n "$TOCI_MACS" ]; then
   MACS=( $TOCI_MACS )
@@ -34,14 +33,14 @@ if [ -n "$TOCI_MACS" ]; then
     COUNT=$(( $COUNT + 1 ))
   done
 else
-  create-nodes 1 768 10 3
+  create-nodes 1 1024 10 5
   export MACS=$($TOCI_WORKING_DIR/bm_poseur/bm_poseur get-macs)
-  setup-baremetal 1 512 10 all
+  setup-baremetal 1 1024 10 seed
 fi
 
 # Load images into glance
 export DIB_PATH=$TOCI_WORKING_DIR/diskimage-builder
-$TOCI_WORKING_DIR/incubator/scripts/load-image notcompute.qcow2
+$TOCI_WORKING_DIR/tripleo-incubator/scripts/load-image undercloud.qcow2
 
 keystone role-create --name heat_stack_user
 
@@ -65,7 +64,9 @@ fi
 # but for now I'm tired so I'm going to
 sleep 67
 
-heat stack-create -f $TOCI_WORKING_DIR/tripleo-heat-templates/bootstack-vm.yaml overcloud -P 'Image=notcompute'
+# TODO : find a better solution then guessing what the controller-address will be
+sed -i $TOCI_WORKING_DIR/tripleo-heat-templates/undercloud-vm.yaml -e 's/\(.*controller-address:\).*/\1 192.0.2.2/'
+heat stack-create -f $TOCI_WORKING_DIR/tripleo-heat-templates/undercloud-vm.yaml -P "PowerUserName=$(whoami)" undercloud
 
 # Just sleeping here so that we don't fill the logs with so many loops
 sleep 180
@@ -75,7 +76,75 @@ heat list
 wait_for 40 20 heat list \| grep CREATE_COMPLETE
 
 # Delete the rule that prevent the Fedora bootstrap vm from forwarding
-# packages. If the rule doesn't exist just do nothing...
+# packets. If the rule doesn't exist just do nothing...
 ssh_noprompt root@$SEED_IP iptables -D FORWARD -j REJECT --reject-with icmp-host-prohibited || true
-wait_for 20 15 ping -c 1 $(nova list | grep overcloud | sed -e "s/.*=\(.*\) .*/\1/g")
+wait_for 20 15 ping -c 1 $(nova list | grep undercloud | sed -e "s/.*=\(.*\) .*/\1/g")
 
+export UNDERCLOUD_IP=$(nova list | grep ctlplane | sed -e "s/.*=\([0-9.]*\).*/\1/")
+cp $TOCI_WORKING_DIR/tripleo-incubator/undercloudrc $TOCI_WORKING_DIR/undercloudrc
+source $TOCI_WORKING_DIR/undercloudrc
+sed -i -e "s/\$UNDERCLOUD_IP/$UNDERCLOUD_IP/g" $TOCI_WORKING_DIR/undercloudrc
+export no_proxy=$no_proxy,$UNDERCLOUD_IP
+
+# If using Fedora keep using F18 for now
+if [[ "$TOCI_DISTROELEMENT" =~ fedora ]] ; then
+    export DIB_CLOUD_IMAGES=http://mattdm.fedorapeople.org/cloud-images
+    export DIB_RELEASE=Fedora18
+    export BASE_IMAGE_FILE=Fedora18-Cloud-$TOCI_ARCH-latest.qcow2
+fi
+
+# Make the tripleo image elements accessible to diskimage-builder
+export ELEMENTS_PATH=$TOCI_WORKING_DIR/diskimage-builder/elements:$TOCI_WORKING_DIR/tripleo-image-elements/elements
+
+if [ "$TOCI_DO_OVERCLOUD" = "1" ] ; then
+    $TOCI_WORKING_DIR/diskimage-builder/bin/disk-image-create -a $TOCI_DIB_ARCH -o overcloud-control $TOCI_DISTROELEMENT boot-stack heat-localip heat-cfntools neutron-network-node stackuser
+fi
+
+# wait for a successful os-refresh-config
+wait_for 60 10 ssh_noprompt heat-admin@$UNDERCLOUD_IP ls /opt/stack/boot-stack/init-neutron.ok
+
+# Make sure nova has had a chance to start responding to requests
+wait_for 10 5 nova list
+
+if [ "$TOCI_DO_OVERCLOUD" != "1" ] ; then
+    exit 0
+fi
+
+user-config
+setup-baremetal 1 1024 10 undercloud
+ssh_noprompt heat-admin@$UNDERCLOUD_IP "cat /opt/stack/boot-stack/virtual-power-key.pub" >> ~/.ssh/authorized_keys
+
+$TOCI_WORKING_DIR/diskimage-builder/bin/disk-image-create -a $TOCI_DIB_ARCH -o overcloud-compute $TOCI_DISTROELEMENT nova-compute nova-kvm neutron-openvswitch-agent heat-localip heat-cfntools stackuser
+
+if [ -d /var/log/upstart ]; then
+    wait_for 40 10 ssh_noprompt heat-admin@$UNDERCLOUD_IP grep 'record\\ updated\\ for' /var/log/upstart/nova-compute.log -A 100 \| grep \'Updating host status\'
+else
+    wait_for 40 10 ssh_noprompt heat-admin@$UNDERCLOUD_IP sudo journalctl _SYSTEMD_UNIT=nova-compute.service \| grep \'record updated for\' -A 100 \| grep \'Updating host status\'
+fi
+
+sleep 67
+
+load-image overcloud-control.qcow2
+load-image overcloud-compute.qcow2
+
+make -C $TOCI_WORKING_DIR/tripleo-heat-templates overcloud.yaml
+heat stack-create -f $TOCI_WORKING_DIR/tripleo-heat-templates/overcloud.yaml -P 'notcomputeImage=overcloud-control' overcloud
+
+sleep 161
+
+wait_for 40 20 heat list \| grep CREATE_COMPLETE
+
+export OVERCLOUD_IP=$(nova list | grep ctlplane | grep notcompute | sed -e "s/.*=\([0-9.]*\).*/\1/")
+sed -e "s/$UNDERCLOUD_IP/$OVERCLOUD_IP/g" undercloudrc > overcloudrc
+source $TOCI_WORKING_DIR/overcloudrc
+export no_proxy=$no_proxy,$OVERCLOUD_IP
+
+# wait for a successful os-refresh-config
+ssh_noprompt heat-admin@$UNDERCLOUD_IP sudo iptables -D FORWARD -j REJECT --reject-with icmp-host-prohibited || true
+wait_for 60 10 ssh_noprompt heat-admin@$OVERCLOUD_IP ls /opt/stack/boot-stack/init-neutron.ok
+
+# Make sure nova has had a chance to start responding to requests
+wait_for 10 5 nova list
+
+# Lets add a cirros image to the overcloud
+curl -L https://launchpad.net/cirros/trunk/0.3.0/+download/cirros-0.3.0-i386-disk.img | glance image-create --name cirros --disk-format qcow2 --container-format bare --is-public 1