From 1ec0125c68601e664dbe878bc41cfcf4fd03f847 Mon Sep 17 00:00:00 2001 From: Dmitry Teselkin Date: Wed, 2 Oct 2013 15:50:16 +0400 Subject: [PATCH] Image builder is able to download required files. Change-Id: If44101746da65fc6dc840b9c90eb533902baf523 --- image-builder/Makefile | 43 +++--- image-builder/dependency.list | 68 +++++++++ image-builder/depmgr.sh | 267 ++++++++++++++++++++++++++++++++++ image-builder/install.sh | 28 ++++ 4 files changed, 388 insertions(+), 18 deletions(-) create mode 100644 image-builder/dependency.list create mode 100755 image-builder/depmgr.sh create mode 100644 image-builder/install.sh diff --git a/image-builder/Makefile b/image-builder/Makefile index 9ed38a16..ee70dffe 100644 --- a/image-builder/Makefile +++ b/image-builder/Makefile @@ -5,6 +5,7 @@ SHELL := /bin/bash image_name := boot_iso := +virtio_iso := virtio_win-0.1_52.iso build_root := /opt/image-builder shared_files_dir := ${build_root}/share/files shared_scripts_dir := ${build_root}/share/scripts @@ -26,20 +27,20 @@ dirs += ${shared_images_dir} # Static files == files that are not changed from one build to another # Typically it's binary prerequisite files -static_files := ${libvirt_images_dir}/ws-2012-eval.iso -static_files += ${libvirt_images_dir}/ws-2008r2-eval.iso -static_files += ${libvirt_images_dir}/virtio-win-0.1-52.iso -static_files += ${shared_files_dir}/CloudbaseInitSetup_Beta.msi -static_files += ${shared_files_dir}/Far30b3367.x64.20130426.msi -static_files += ${shared_files_dir}/Git-1.8.1.2-preview20130201.exe -static_files += ${shared_files_dir}/SysinternalsSuite.zip -static_files += ${shared_files_dir}/unzip.exe -static_files += ${shared_files_dir}/userdata.py -static_files += ${shared_files_dir}/Windows6.1-KB2506143-x64.msu -static_files += ${shared_files_dir}/dotNetFx40_Full_x86_x64.exe -static_files += ${shared_files_dir}/dotNetFx45_Full_setup.exe +#static_files := ${libvirt_images_dir}/ws-2012-eval.iso +#static_files += ${libvirt_images_dir}/ws-2008r2-eval.iso +#static_files += ${libvirt_images_dir}/virtio-win-0.1-52.iso +#static_files += ${shared_files_dir}/CloudbaseInitSetup_Beta.msi +#static_files += ${shared_files_dir}/Far30b3367.x64.20130426.msi +#static_files += ${shared_files_dir}/Git-1.8.1.2-preview20130201.exe +#static_files += ${shared_files_dir}/SysinternalsSuite.zip +#static_files += ${shared_files_dir}/unzip.exe +#static_files += ${shared_files_dir}/userdata.py +#static_files += ${shared_files_dir}/Windows6.1-KB2506143-x64.msu +#static_files += ${shared_files_dir}/dotNetFx40_Full_x86_x64.exe +#static_files += ${shared_files_dir}/dotNetFx45_Full_setup.exe # TODO: move the following files to the 'dynamic files' as they are 'version dependent' -static_files += ${shared_files_dir}/MuranoAgent.zip +#static_files += ${shared_files_dir}/MuranoAgent.zip # Dynamic files == files that might be updated from repository during the build. @@ -95,21 +96,27 @@ clean-all : clean ${MAKE} -C ${ps_modules_dir} clean -build-root : ${dirs} +build-root : ${dirs} prerequisites cp -r * ${build_root} rm ${build_root}/Makefile +prerequisites : ${dirs} + ./depmgr.sh download dependency.list + + ${dirs} : mkdir -p $@ -test-build-files : ${build_files} ${static_files} ${dynamic_files} +#test-build-files : ${build_files} ${static_files} ${dynamic_files} +test-build-files : ${build_files} ${dynamic_files} + ./depmgr.sh test dependency.list # Rule to check if all 'static files' are in place -${static_files} : - @[ -f $@ ] || echo "STOP: File not found '$@'"; exit 1 +#${static_files} : +# @[ -f $@ ] || echo "STOP: File not found '$@'"; exit 1 # Rule to copy updated 'build files' @@ -152,7 +159,7 @@ ${image_name} : ${shared_images_dir}/${image_name}.qcow2 ${shared_images_dir}/${image_name}.qcow2 : ${libvirt_images_dir}/${image_name}.flp | test-build-files @echo '' @echo '>>> Starting install-vm ...' - env BUILD_ROOT=${build_root} VM_NAME=${image_name}-${commit_hash} IMAGE_NAME=${image_name} FLOPPY_IMG=${image_name}.flp BOOT_ISO=${boot_iso} ${build_root}/install-vm.sh + env BUILD_ROOT=${build_root} VM_NAME=${image_name}-${commit_hash} IMAGE_NAME=${image_name} FLOPPY_IMG=${image_name}.flp BOOT_ISO=${boot_iso} VIRTIO_ISO=${virtio_iso} ${build_root}/install-vm.sh @echo '<<< done.' diff --git a/image-builder/dependency.list b/image-builder/dependency.list new file mode 100644 index 00000000..9e03d09a --- /dev/null +++ b/image-builder/dependency.list @@ -0,0 +1,68 @@ +[Windows Server 2012 Evluation] +skip = true +mandatory = true +name = ws-2012-eval.iso +path = $BUILD_ROOT/libvirt/images +url = http://technet.microsoft.com/en-us/evalcenter/hh670538.aspx + +[Windows Server 2008 R2 Evaluation] +skip = true +mandatory = true +name = ws-2008r2-eval.iso +path = $BUILD_ROOT/libvirt/images +url = http://www.microsoft.com/en-us/download/details.aspx?id=11093 + +[VirtIO Drivers] +mandatory = true +name = virtio_win-0.1_52.iso +path = $BUILD_ROOT/libvirt/images +url = http://alt.fedoraproject.org/pub/alt/virtio-win/stable/virtio-win-0.1-52.iso + +[CloudBase-Init] +mandatory = true +name = cloudbase_init.msi +path = $BUILD_ROOT/share/files +url = http://www.cloudbase.it/downloads/CloudbaseInitSetup_Beta.msi + +[Far Manager] +skip = true +name = far_manager-3.0-x64.msi +path = $BUILD_ROOT/share/files +url = http://www.farmanager.com/files/Far30b3525.x64.20130717.msi + +[mSysGit] +name = msysgit-1.8.3.exe +path = $BUILD_ROOT/share/files +url = https://msysgit.googlecode.com/files/Git-1.8.3-preview20130601.exe + +[Sysinternals Suite] +skip = true +name = sysinternals_suite.zip +path = $BUILD_ROOT/share/files +url = http://download.sysinternals.com/files/SysinternalsSuite.zip + +[unzip.exe] +mandatory = true +name = unzip.exe +path = $BUILD_ROOT/share/files +url = https://www.dropbox.com/sh/zthldcxnp6r4flm/W-lgLFo7Fz/unzip.exe + +#[PowerShell V3] +#mandatory = true +#path = $BUILD_ROOT/share/files +#url = http://www.microsoft.com/en-us/download/confirmation.aspx?id=34595 + +#[.NET 4.0] +#mandatory = true +#path = $BUILD_ROOT/share/files +#url = http://www.microsoft.com/en-us/download/confirmation.aspx?id=17718 + +#[.NET 4.5] +#mandatory = true +#path = $BUILD_ROOT/share/files +#url = http://www.microsoft.com/en-us/download/confirmation.aspx?id=30653 + +[Murano Agent] +mandatory = true +path = $BUILD_ROOT/share/files +url = https://www.dropbox.com/sh/zthldcxnp6r4flm/COFiuwTi5d/MuranoAgent.zip diff --git a/image-builder/depmgr.sh b/image-builder/depmgr.sh new file mode 100755 index 00000000..a754beef --- /dev/null +++ b/image-builder/depmgr.sh @@ -0,0 +1,267 @@ +#!/bin/bash + +mode=${1:-'print'} +config_file=${2:-'./config.file'} + +status_file=/tmp/status.log + +BUILD_ROOT=/opt/image-builder + +#------------------------------------------------------------------------------- + +trim() { + echo "$1" +} + +msg() { + cat << EOF + +* $@ +EOF +} + +err() { + cat << EOF >&1 + +* $@ +EOF +} + +status() { + echo -e "$@" >> $status_file +} + +process_item() { + case $mode in + 'download') + download_item + ;; + 'test') + test_item + ;; + 'print') + print_item + ;; + esac +} + +print_item() { + validate_item_property + if [ $item_invalid = 'true' ] ; then + return + fi + + cat << EOF + +Section = $section +Name = $item_name +Path = $item_path +URL = $item_url +Skip = $item_skip +Mandatory = $item_mandatory +EOF +} + +validate_item_property() { +# if [[ -z "$item_name" ]] ; then +# err "Section '$section', line $line_number: Name is empty!" +# item_invalid='true' +# fi + + if [[ -z "$item_path" ]] ; then + err "Section '$section', line $line_number: Path is empty!" + item_invalid='true' + fi + + if [[ -z "$item_url" ]] ; then + err "Section '$section', line $line_number: URL is empty!" + item_invalid='true' + fi + + if [ $item_invalid = 'true' ] ; then + msg "Bad input in section '$section'" + fi +} + +download_item() { + local file_name + + validate_item_property + if [ $item_invalid = 'true' ] ; then + return + fi + + if [ "$item_skip" = 'true' ] ; then + msg "Skipping section '$section'" + return + fi + + mkdir -p $item_path + + if [[ -z "$item_name" ]] ; then + file_name=$(basename $item_url) + else + file_name=$item_name + fi + + if [[ -f "$item_path/$file_name" ]] ; then + if [[ "$item_force_update" = 'true' ]] ; then + rm -f "$item_path/$file_name" + else + msg "Item '$item_path/$file_name' exists, skipping" + status "SKIPPED\t$item_path/$file_name" + return + fi + fi + + msg "Downloading from '$item_url'" + if [[ -z "$item_name" ]] ; then + wget --content-disposition --directory-prefix="$item_path" --timestamping $item_url + wget_result=$? + else + wget --content-disposition --output-document="$item_path/$file_name" $item_url + wget_result=$? + fi + + if [[ $wget_result -eq 0 ]] ; then + status "OK\t$item_url" + else + status "FAILED\t$item_url" + fi +} + + +test_item() { + local file_name + + validate_item_property + if [ $item_invalid = 'true' ] ; then + return + fi + + if [[ -z "$item_name" ]] ; then + file_name=$(basename $item_url) + else + file_name=$item_name + fi + + if [[ ! -f "$item_path/$file_name" ]] ; then + if [[ "$item_mandatory" = 'true' ]] ; then + status "NOT_FOUND\t$item_path/$file_name" + else + status "WARNING\t$item_path/$file_name" + fi + fi + +} + +#------------------------------------------------------------------------------- + +rm -f $status_file + +line_number=0 +section='' +while IFS='=' read -r key value +do + line_number=$(($line_number + 1)) + + [ -z "$key" ] && continue + + [[ "$key" =~ ^# ]] && continue + + if [[ "$key" =~ \[.*\] ]] ; then + key=${key#*[} + key=${key%]*} + if [ -n "$section" ] ; then + process_item + fi + section=$key + item_name='' + item_path='' + item_url='' + item_mandatory='false' + item_skip='false' + item_invalid='false' + item_force_update='false' + item_image_type='' + fi + + key=$(trim $key) + value=$(trim $value) + + case $key in + 'name') + item_name=$value + ;; + 'path') + eval "item_path=${value}" + ;; + 'url') + item_url=$value + ;; + 'skip') + item_skip=$value + ;; + 'mandatory') + item_mandatory=$value + ;; + 'force_update') + item_force_update=$value + ;; + 'image_type') + eval "item_image_type=${value}" + ;; + esac +done < $config_file +process_item + + +if [[ "$mode" = 'download' ]] ; then + if grep -q 'FAILED' $status_file ; then + cat << EOF + +ERROR: +- One or more errors occured while downloading the files +******************************************************** + +$(grep 'FAILED' $status_file) + +******************************************************** +EOF + exit 1 + fi + + exit 0 +fi + + +if [[ $mode = 'test' ]] ; then + if grep -q 'WARNING' $status_file ; then + cat << EOF + +WARNING: +- One or more non-mandatory files not found +******************************************* + +$(grep 'WARNING' $status_file) + +******************************************* +EOF + fi + + if grep -q 'NOT_FOUND' $status_file ; then + cat << EOF + +ERROR: +- One or more mandatory files not found +*************************************** + +$(grep 'NOT_FOUND' $status_file) + +*************************************** +EOF + exit 1 + fi + + exit 0 +fi diff --git a/image-builder/install.sh b/image-builder/install.sh new file mode 100644 index 00000000..5c42f8ea --- /dev/null +++ b/image-builder/install.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +apt-get install zip kvm libvirt-bin virtinst samba + +mkdir -p /opt/image-builder/share/files +mkdir -p /opt/image-builder/share/images +mkdir -p /opt/image-builder/share/scripts +mkdir -p /opt/image-builder/libvirt/images + +if grep -q 'image-builder-share' /etc/samba/smb.conf ; then + echo "Samba configuration already applied" +else + echo "Updating samba configuration" + cat << EOF >> /etc/samba/smb.conf + +[image-builder-share] + comment = Image Builder Share + path = /opt/image-builder/share + browsable = yes + guest ok = yes + guest account = nobody + read only = no + create mask = 0755 +EOF + + restart smbd + restart nmbd +fi