From 0175da2ee6d164f602af4dfa367b3e58d33bfa9a Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Fri, 27 Feb 2015 12:11:34 +1300 Subject: [PATCH] Elements for building a heat test image The heat functional test job needs to boot full images containing the heat agent code (os-collect-config etc) so that it can test the interaction between heat and the agent. Images built with the elements in this change contain the necessary distro packages so that only pip packages need to be installed on server boot. This is very fast compared to doing a disk-image-create on every gate run or doing distro package installs during boot. Initially pip packages would be installed from upstream pypi during boot, but devstack already has the code required [1] to build pip repos from local git checkouts so it will be possible to check and gate on agent code changes, which is not currently possible. The intent would be for these images to replace the vanilla Fedora-x86_64-20-20140618-sda which devstack currently uses during gate jobs. The Fedora-x86_64-20-20140618-sda qcow is cached on the nodepool images but it might be better for these images to be hosted on a server like tarballs.o.o and downloaded on every gate run. Initially I'd like to request that an image be manually built and uploaded somewhere so we can start using it immediately and drop Fedora-x86_64-20-20140618-sda. Building these images does not need to be fully automated and likely only needs to be triggered when there are dependency changes in the heat agent tools. The script build-heat-test-image.sh will trigger a image build defaulting to fedora. Ubuntu and CentOS7 are also fully supported by these elements. [1] https://github.com/openstack-dev/devstack/blob/master/lib/dib#L52 Change-Id: I3f10f9137cc4391de884fddcab5b4fcb802a8dd8 --- hot/software-config/README.rst | 7 +++- hot/software-config/test-image/README.rst | 22 +++++++++++ .../test-image/build-heat-test-image.sh | 25 ++++++++++++ .../test-image/elements/centos7-rdo/README.md | 4 ++ .../elements/centos7-rdo/element-deps | 1 + .../centos7-rdo/package-installs.yaml | 3 ++ .../test-image/elements/centos7-rdo/pkg-map | 5 +++ .../heat-agent-pkg-requires/README.md | 10 +++++ .../heat-agent-pkg-requires/element-deps | 1 + .../package-installs.yaml | 15 +++++++ .../elements/heat-agent-pkg-requires/pkg-map | 39 +++++++++++++++++++ .../heat-config-hook-requires/README.md | 8 ++++ .../heat-config-hook-requires/element-deps | 1 + .../package-installs.yaml | 3 ++ .../heat-config-hook-requires/pkg-map | 7 ++++ .../test-image/elements/python-dev/README.md | 4 ++ .../elements/python-dev/element-deps | 1 + .../elements/python-dev/package-installs.yaml | 9 +++++ .../test-image/elements/python-dev/pkg-map | 27 +++++++++++++ tools/validate-templates | 2 +- 20 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 hot/software-config/test-image/README.rst create mode 100755 hot/software-config/test-image/build-heat-test-image.sh create mode 100644 hot/software-config/test-image/elements/centos7-rdo/README.md create mode 100644 hot/software-config/test-image/elements/centos7-rdo/element-deps create mode 100644 hot/software-config/test-image/elements/centos7-rdo/package-installs.yaml create mode 100644 hot/software-config/test-image/elements/centos7-rdo/pkg-map create mode 100644 hot/software-config/test-image/elements/heat-agent-pkg-requires/README.md create mode 100644 hot/software-config/test-image/elements/heat-agent-pkg-requires/element-deps create mode 100644 hot/software-config/test-image/elements/heat-agent-pkg-requires/package-installs.yaml create mode 100644 hot/software-config/test-image/elements/heat-agent-pkg-requires/pkg-map create mode 100644 hot/software-config/test-image/elements/heat-config-hook-requires/README.md create mode 100644 hot/software-config/test-image/elements/heat-config-hook-requires/element-deps create mode 100644 hot/software-config/test-image/elements/heat-config-hook-requires/package-installs.yaml create mode 100644 hot/software-config/test-image/elements/heat-config-hook-requires/pkg-map create mode 100644 hot/software-config/test-image/elements/python-dev/README.md create mode 100644 hot/software-config/test-image/elements/python-dev/element-deps create mode 100644 hot/software-config/test-image/elements/python-dev/package-installs.yaml create mode 100644 hot/software-config/test-image/elements/python-dev/pkg-map diff --git a/hot/software-config/README.rst b/hot/software-config/README.rst index 4603acb0..c9ebca68 100644 --- a/hot/software-config/README.rst +++ b/hot/software-config/README.rst @@ -25,4 +25,9 @@ a custom-built image. example-templates ----------------- This contains example heat templates which demonstrate how the software config -resources and the hooks work together to perform software configuration. \ No newline at end of file +resources and the hooks work together to perform software configuration. + +test-image +---------- +This contains `diskimage-builder `_ +elements which will install the distro packages required for a base test image. \ No newline at end of file diff --git a/hot/software-config/test-image/README.rst b/hot/software-config/test-image/README.rst new file mode 100644 index 00000000..1cf2b970 --- /dev/null +++ b/hot/software-config/test-image/README.rst @@ -0,0 +1,22 @@ +======================================= +Elements for building a heat test image +======================================= + +The heat functional test job needs to boot full images containing the +heat agent code (os-collect-config etc) so that it can test the +interaction between heat and the agent. + +Images built with these elements contain the necessary +distro packages so that only pip packages need to be installed on +server boot. + +The script build-heat-test-image.sh will trigger a image build +defaulting to fedora. Ubuntu and CentOS7 are also fully supported by +these elements. Run the following to build all supported images: + + +:: + + DISTRO=fedora ./build-heat-test-image.sh + DISTRO=ubuntu ./build-heat-test-image.sh + DISTRO=centos7-rdo ./build-heat-test-image.sh diff --git a/hot/software-config/test-image/build-heat-test-image.sh b/hot/software-config/test-image/build-heat-test-image.sh new file mode 100755 index 00000000..7c2a40b0 --- /dev/null +++ b/hot/software-config/test-image/build-heat-test-image.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +# known good values of DISTRO are fedora, ubuntu, centos7-rdo +export DISTRO=${DISTRO:-fedora} + +export ELEMENTS_PATH=${ELEMENTS_PATH:-`dirname "$0"`/elements} +export IMAGE_NAME=${IMAGE_NAME:-$DISTRO-heat-test-image} + +disk-image-create -x --no-tmpfs -o $IMAGE_NAME $DISTRO \ + vm python-dev heat-agent-pkg-requires heat-config-hook-requires diff --git a/hot/software-config/test-image/elements/centos7-rdo/README.md b/hot/software-config/test-image/elements/centos7-rdo/README.md new file mode 100644 index 00000000..25495473 --- /dev/null +++ b/hot/software-config/test-image/elements/centos7-rdo/README.md @@ -0,0 +1,4 @@ +Adds the centos7 element and includes the Juno RDO yum repository + +The Juno RDO repo is required for OpenStack packages which are not in +CentOS7 or EPEL repos (python-oslo-config, python-*client) \ No newline at end of file diff --git a/hot/software-config/test-image/elements/centos7-rdo/element-deps b/hot/software-config/test-image/elements/centos7-rdo/element-deps new file mode 100644 index 00000000..fce07e4f --- /dev/null +++ b/hot/software-config/test-image/elements/centos7-rdo/element-deps @@ -0,0 +1 @@ +centos7 diff --git a/hot/software-config/test-image/elements/centos7-rdo/package-installs.yaml b/hot/software-config/test-image/elements/centos7-rdo/package-installs.yaml new file mode 100644 index 00000000..266a3ef5 --- /dev/null +++ b/hot/software-config/test-image/elements/centos7-rdo/package-installs.yaml @@ -0,0 +1,3 @@ +# uninstall fedora's heat-cfntools so it can be installed from pip +rdo_release_juno_package: + phase: pre-install.d diff --git a/hot/software-config/test-image/elements/centos7-rdo/pkg-map b/hot/software-config/test-image/elements/centos7-rdo/pkg-map new file mode 100644 index 00000000..326afb26 --- /dev/null +++ b/hot/software-config/test-image/elements/centos7-rdo/pkg-map @@ -0,0 +1,5 @@ +{ + "default": { + "rdo_release_juno_package": "https://repos.fedorapeople.org/repos/openstack/openstack-juno/rdo-release-juno-1.noarch.rpm" + } +} diff --git a/hot/software-config/test-image/elements/heat-agent-pkg-requires/README.md b/hot/software-config/test-image/elements/heat-agent-pkg-requires/README.md new file mode 100644 index 00000000..d254d6d9 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-agent-pkg-requires/README.md @@ -0,0 +1,10 @@ +Installs packages required by heat agent tools. + +These are the dependencies for the following projects which make up the +heat agent tools + +- os-collect-config +- os-apply-config +- os-refresh-config +- dib-utils +- heat-cfntools \ No newline at end of file diff --git a/hot/software-config/test-image/elements/heat-agent-pkg-requires/element-deps b/hot/software-config/test-image/elements/heat-agent-pkg-requires/element-deps new file mode 100644 index 00000000..5de08f6a --- /dev/null +++ b/hot/software-config/test-image/elements/heat-agent-pkg-requires/element-deps @@ -0,0 +1 @@ +package-installs \ No newline at end of file diff --git a/hot/software-config/test-image/elements/heat-agent-pkg-requires/package-installs.yaml b/hot/software-config/test-image/elements/heat-agent-pkg-requires/package-installs.yaml new file mode 100644 index 00000000..88a57730 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-agent-pkg-requires/package-installs.yaml @@ -0,0 +1,15 @@ +python_setuptools_package: +python_argparse_package: +python_anyjson_package: +python_dogpile_cache_package: +python_eventlet_package: +python_heatclient_package: +python_keystoneclient_package: +python_requests_package: +python_iso8601_package: +python_lxml_package: +python_six_package: +python_oslo_config_package: +pystache_package: +python_pbr_package: +python_boto_package: diff --git a/hot/software-config/test-image/elements/heat-agent-pkg-requires/pkg-map b/hot/software-config/test-image/elements/heat-agent-pkg-requires/pkg-map new file mode 100644 index 00000000..5410b552 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-agent-pkg-requires/pkg-map @@ -0,0 +1,39 @@ + +{ + "family": { + "redhat": { + "python_setuptools_package": "python-setuptools", + "python_argparse_package": "python-argparse", + "python_anyjson_package": "python-anyjson", + "python_dogpile_cache_package": "python-dogpile-cache", + "python_eventlet_package": "python-eventlet", + "python_heatclient_package": "python-heatclient", + "python_keystoneclient_package": "python-keystoneclient", + "python_requests_package": "python-requests", + "python_iso8601_package": "python-iso8601", + "python_lxml_package": "python-lxml", + "python_six_package": "python-six", + "python_oslo_config_package": "python-oslo-config", + "pystache_package": "pystache", + "python_pbr_package": "python-pbr", + "python_boto_package": "python-boto" + }, + "debian": { + "python_setuptools_package": "python-setuptools", + "python_argparse_package": "python-argparse", + "python_anyjson_package": "python-anyjson", + "python_dogpile_cache_package": "python-dogpile.cache", + "python_eventlet_package": "python-eventlet", + "python_heatclient_package": "python-heatclient", + "python_keystoneclient_package": "python-keystoneclient", + "python_requests_package": "python-requests", + "python_iso8601_package": "python-iso8601", + "python_lxml_package": "python-lxml", + "python_six_package": "python-six", + "python_oslo_config_package": "python-oslo-config", + "pystache_package": "python-pystache", + "python_pbr_package": "python-pbr", + "python_boto_package": "python-boto" + } + } +} diff --git a/hot/software-config/test-image/elements/heat-config-hook-requires/README.md b/hot/software-config/test-image/elements/heat-config-hook-requires/README.md new file mode 100644 index 00000000..b64caaa9 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-config-hook-requires/README.md @@ -0,0 +1,8 @@ +Configuration tools required by tested heat-config hooks + +This element installs the configuration tools required by some heat-config +hooks so that they are available for testing. Current tools installed are: + +- ansible +- puppet +- salt-minion diff --git a/hot/software-config/test-image/elements/heat-config-hook-requires/element-deps b/hot/software-config/test-image/elements/heat-config-hook-requires/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-config-hook-requires/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/hot/software-config/test-image/elements/heat-config-hook-requires/package-installs.yaml b/hot/software-config/test-image/elements/heat-config-hook-requires/package-installs.yaml new file mode 100644 index 00000000..bbbe902f --- /dev/null +++ b/hot/software-config/test-image/elements/heat-config-hook-requires/package-installs.yaml @@ -0,0 +1,3 @@ +puppet_package: +ansible_package: +salt_minion_package: diff --git a/hot/software-config/test-image/elements/heat-config-hook-requires/pkg-map b/hot/software-config/test-image/elements/heat-config-hook-requires/pkg-map new file mode 100644 index 00000000..37090fc7 --- /dev/null +++ b/hot/software-config/test-image/elements/heat-config-hook-requires/pkg-map @@ -0,0 +1,7 @@ +{ + "default": { + "puppet_package": "puppet", + "ansible_package": "ansible", + "salt_minion_package": "salt-minion" + } +} diff --git a/hot/software-config/test-image/elements/python-dev/README.md b/hot/software-config/test-image/elements/python-dev/README.md new file mode 100644 index 00000000..86e4809c --- /dev/null +++ b/hot/software-config/test-image/elements/python-dev/README.md @@ -0,0 +1,4 @@ +Install packages for common pip install dependencies + +This element installs packages needed for common pip installs, including +devel packages need by pip build of packages like PyYAML, lxml, pyOpenSSL. \ No newline at end of file diff --git a/hot/software-config/test-image/elements/python-dev/element-deps b/hot/software-config/test-image/elements/python-dev/element-deps new file mode 100644 index 00000000..7076aba9 --- /dev/null +++ b/hot/software-config/test-image/elements/python-dev/element-deps @@ -0,0 +1 @@ +package-installs diff --git a/hot/software-config/test-image/elements/python-dev/package-installs.yaml b/hot/software-config/test-image/elements/python-dev/package-installs.yaml new file mode 100644 index 00000000..06b36564 --- /dev/null +++ b/hot/software-config/test-image/elements/python-dev/package-installs.yaml @@ -0,0 +1,9 @@ +python_pip_package: +git_package: +gcc_package: +python_devel_package: +libyaml_devel_package: +openssl_devel_package: +libffi_devel_package: +libxml2_devel_package: +libxslt_devel_package: diff --git a/hot/software-config/test-image/elements/python-dev/pkg-map b/hot/software-config/test-image/elements/python-dev/pkg-map new file mode 100644 index 00000000..c375f877 --- /dev/null +++ b/hot/software-config/test-image/elements/python-dev/pkg-map @@ -0,0 +1,27 @@ + +{ + "family": { + "redhat": { + "python_pip_package": "python-pip", + "gcc_package": "gcc", + "git_package": "git-core", + "python_devel_package": "python-devel", + "libyaml_devel_package": "libyaml-devel", + "openssl_devel_package": "openssl-devel", + "libffi_devel_package": "libffi-devel", + "libxml2_devel_package": "libxml2-devel", + "libxslt_devel_package": "libxslt-devel" + }, + "debian": { + "python_pip_package": "python-pip", + "git_package": "git", + "gcc_package": "gcc", + "python_devel_package": "python-dev", + "libyaml_devel_package": "libyaml-dev", + "openssl_devel_package": "libssl-dev", + "libffi_devel_package": "libffi-dev", + "libxml2_devel_package": "libxml2-dev", + "libxslt_devel_package": "libxslt1-dev" + } + } +} diff --git a/tools/validate-templates b/tools/validate-templates index 6c1e2835..4e6c4a61 100755 --- a/tools/validate-templates +++ b/tools/validate-templates @@ -5,7 +5,7 @@ import subprocess import sys -EXCLUDED_DIRS = ('contrib',) +EXCLUDED_DIRS = ('contrib', 'elements') def main(args):