Scripts for export and import systest environments

Initial version of scripts for export and import systest environments,
i.e. all domains, networks, volumes, etc. together with created
external snapshots.
These scripts export/import records from/to fuel-devops DB, libvirt
definitions and content files both for volumes and memory state files.

The scripts are divided into 2 parts:
- env_* - export/import scripts
- libvirt_functions - functions with virsh calls

All the files are configured to be placed to the bin directory during
the fuel-devops installation to be available in PATH. It makes usage of
these scripts from Jenkins more comfortable.

blueprint: environment-snapshot
Change-Id: I9fd03a383bf1b47b778e1de0f0aa8094a4896cdc
This commit is contained in:
Aliaksei Cherniakou 2016-02-16 00:00:26 +03:00
parent 0b93c9c8b1
commit b3ee7a619b
4 changed files with 1379 additions and 1 deletions

749
bin/env_export.sh Executable file
View File

@ -0,0 +1,749 @@
#!/bin/bash
set -o errexit
# Set xtrace option if run by Jenkins
if [ -n "${WORKSPACE}" ]; then
set -o xtrace
fi
baseDir=$(dirname "$0")
DEFS_ONLY=0
DEBUG=0
EXPORT_CHANGES="${EXPORT_CHANGES:-0}"
HOST_PASSTHROUGH_CPU_MODE=0
VERBOSITY=0
WORKAROUND="${WORKAROUND:-0}"
INVALIDOPTS_ERR=100
NOENVFOUND_ERR=101
NOSNAPSHOTFOUND_ERR=102
INVALIDSNAPSHOTORDER_ERR=103
MEMORYSTATEEXPORT_ERR=104
ShowHelp() {
cat << EOF
Usage: $0 [OPTION]... ENV SNAPSHOT[,SNAPSHOT[,...]]
ENV - Name of the environment to export snapshots from.
SNAPSHOT - Snapshot name to export. Script will export all snapshots starting
from the provided one down to root snapshot of the snapshot chain.
Provided snapshot will be set as a current.
Example:
Snapshot chain:
s1 -> s2 -> s3 -> s4
s1 - root snapshot (has no parent)
s4 - leaf snapshot (has no children)
-> - chronological order
If you pass s3 snapshot as an argument, script will export s3,
s2, s1 snapshots and will set s3 snapshot as current. In this
case s3 will be the leaf snapshot (tip of the chain).
If environment has a snapshot tree and it is necessary to export
several snapshot branches you have to pass snapshot names from
the branches (one per branch) and separate them by commas.
In this case all passed snapshots will be the tips of the exported
branches, and the first snapshot in the list will be set
as a current one.
To identify snapshots to export when working with snapshot trees
script treats branches as separate snapshot chains (see example
above).
NOTE: Extra privileges are necessary to copy memory state files as libvirt creates
the files with read-write access rights only for root.
NOTE: If the list of snapshot names contains descendants of the first snapshot
from the list, script execution will be interrupted with an error.
The goal of the script is to export environment without any changes of
snapshot and volume trees. But implementation of the case when the script
exports snapshot with its descendants and then sets the snapshot as a current
requires modifications of the trees.
The following options are available:
-d - Only psql files with data from fuel-devops DB and files with
libvirt XML definitions for domains, networks, volumes, network
filters, storage pool, snapshots will be exported. Volumes and
memory state files are skipped.
-e path - Path to the directory where exported environment files will
be stored.
-h - Show this help page.
-p - Preserve changes on disks since the snapshot.
Disk snapshot in external form consists of 2 volume files:
the snapshot is one file, and the changes since the snapshot are
in another file.
This option enables export of both volume files - the snapshot and
the changes since the snapshot.
By default script exports only snapshot.
For instance, this option can be useful if it's necessary to make
full export of the environment (i.e. leaf snapshots together with
the changes made since the snapshots).
-s path - Directory where libvirt stores files with XML definitions
for snapshots on the source host.
-v - Set verbose log level.
-vv - Set debug log level.
-w - Volume export is implemented via 'virsh vol-download' command.
The older versions of libvirt has a bug with reduced performance
of 'vol-download'. This option enables workaround: direct access to
the volume files instead of using 'virsh vol-download'.
Note: it requires 'read' permissions for the volume files
-ww - The same as '-w' option but uses 'sudo cp'. This option should be
used if the volume files have no read permission.
You can override the following variables:
DEVOPS_DB_NAME - fuel-devops DB name
DEVOPS_DB_HOST - fuel-devops DB hostname
DEVOPS_DB_USER - fuel-devops DB username
DEVOPS_DB_PASSWORD - fuel-devops DB password
EXPORT_CHANGES - (0-default,1). 1 enables -p option (0 by default)
CONNECTION_STRING - hypervisor connection URI
STORAGE_POOL_NAME - name of the storage pool to use
SNAPSHOTS_DIR - directory where libvirt stores files with XML definitions
for snapshots on the source host
WORKAROUND - (0-default,1,2). 1 (-w option) / 2 (-ww option)
EOF
}
exec 3>&1
GetoptsVariables() {
while getopts ":de:hps:vw" opt; do
case $opt in
d)
DEFS_ONLY=1
;;
e)
EXPORT_DIR="${OPTARG}"
;;
h)
ShowHelp
exit 0
;;
p)
EXPORT_CHANGES=1
;;
s)
SNAPSHOTS_DIR="${OPTARG}"
;;
v)
VERBOSITY=$((VERBOSITY+1))
;;
w)
WORKAROUND=$((WORKAROUND+1))
;;
\?)
ShowHelp
exit ${INVALIDOPTS_ERR}
;;
:)
echo "Option -${OPTARG} requires an argument." >&2
ShowHelp
exit ${INVALIDOPTS_ERR}
;;
esac
done
shift $((OPTIND-1))
if [ ${#} -ne 2 ]; then
ShowHelp
exit ${INVALIDOPTS_ERR}
fi
case ${VERBOSITY} in
0)
exec 4> /dev/null
exec 5> /dev/null
;;
1)
exec 4>&1
exec 5> /dev/null
;;
*)
DEBUG=1
exec 4>&1
exec 5>&1
;;
esac
SYSTEST_ENV="${1}"
SNAPSHOT_NAMES="${2}"
[ "${DEBUG}" -eq 0 ] || echo "Debug mode on."
[ "${DEFS_ONLY}" -eq 0 ] ||
echo "Volume and memory state files will not be exported."
}
GlobalVariables() {
echo "Using DB Name: ${DEVOPS_DB_NAME:=fuel_devops}" >&4
echo "Using DB Host: ${DEVOPS_DB_HOST:=localhost}" >&4
echo "Using DB User: ${DEVOPS_DB_USER:=fuel_devops}" >&4
export PGPASSWORD="${DEVOPS_DB_PASSWORD:-fuel_devops}"
EXPORT_DIR="${EXPORT_DIR:-"${HOME}/.devops/export"}"
FUELDEVOPS_EXPORT_DIR="${EXPORT_DIR}/fuel-devops"
LIBVIRT_EXPORT_DIR="${EXPORT_DIR}/libvirt"
LIBVIRT_EXPORT_DEFS_DIR="${LIBVIRT_EXPORT_DIR}/definitions"
LIBVIRT_EXPORT_VOLUMES_DIR="${LIBVIRT_EXPORT_DIR}/volumes"
LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR="${LIBVIRT_EXPORT_DIR}/snapshot"
LIBVIRT_EXPORT_MEMSTATE_DIR="${LIBVIRT_EXPORT_DIR}/memstate"
CONNECTION_STRING="${CONNECTION_STRING:-"qemu:///system"}"
STORAGE_POOL="${STORAGE_POOL_NAME:-default}"
SNAPSHOTS_DIR="${SNAPSHOTS_DIR:-"/var/lib/libvirt/qemu/snapshot"}"
# Tables in fuel-devops DB
DEVOPS_ENVIRONMENT_TABLE="devops_environment"
DEVOPS_NETWORK_TABLE="devops_network"
DEVOPS_VOLUME_TABLE="devops_volume"
DEVOPS_NODE_TABLE="devops_node"
DEVOPS_INTERFACE_TABLE="devops_interface"
DEVOPS_ADDRESS_TABLE="devops_address"
DEVOPS_DISKDEVICE_TABLE="devops_diskdevice"
}
ExportItemsFromDB() {
# Args: $@ - SQL statement
printf " Run: psql -h %s -U %s -c \"%s\"\n" "${DEVOPS_DB_HOST}" \
"${DEVOPS_DB_USER}" "${@}" | sed 's/[ ]\+/ /g' >&5
psql -h "${DEVOPS_DB_HOST}" -U "${DEVOPS_DB_USER}" -c "${1}" >&5
}
RunPSQL() {
# Args: $@ - SQL statement
printf " Run: psql -h %s -U %s -t -c \"%s\"\n" "${DEVOPS_DB_HOST}" \
"${DEVOPS_DB_USER}" "${@}" | sed 's/[ ]\+/ /g' >&5
res=$(psql -h "${DEVOPS_DB_HOST}" -U "${DEVOPS_DB_USER}" -t -c "${@}")
echo "${res}" | sed 's/[[:space:]]//'
}
GetItemsFromDB() {
# Create comma-separated list of items returned by RunPSQL
# Args: $1 - SQL statement
for item in $(RunPSQL "${1}"); do
items_list="${items_list:-""},${item}"
done
items_list="${items_list##,}"
echo "${items_list}"
}
CheckEnvironmentExists() {
#Check if the environment exist
env_count=$(RunPSQL \
"select count(*) from ${DEVOPS_ENVIRONMENT_TABLE}\
where name='${SYSTEST_ENV}'")
if [ "${env_count}" -eq 0 ]; then
echo "No environments found: ${SYSTEST_ENV}" >&2
exit ${NOENVFOUND_ERR}
fi
# Get environment id in fuel-devops DB
ENVIRONMENT_ID_SQL="select id from ${DEVOPS_ENVIRONMENT_TABLE}\
where name='${SYSTEST_ENV}'"
env_id=$(GetItemsFromDB "${ENVIRONMENT_ID_SQL}")
# Get domain names created in the envionment from fuel-devops DB
DOMAIN_NAMES_SQL="select concat('${SYSTEST_ENV}_', name) \
from ${DEVOPS_NODE_TABLE}\
where environment_id='${env_id}'"
domain_names=$(GetItemsFromDB "${DOMAIN_NAMES_SQL}")
}
CheckSnapshotExists() {
# Check the requested snapshot is available across all the domains
# from the environment. Abort otherwise.
OLDIFS="${IFS}"
IFS=","
for domain in ${domain_names}; do
for snapshot_name in ${SNAPSHOT_NAMES}; do
if ! CheckSnapshot "${domain}" "${snapshot_name}"; then
echo "No '${snapshot_name}' snapshot found for '${domain}'" >&2
exit ${NOSNAPSHOTFOUND_ERR}
fi
done
done
IFS="${OLDIFS}"
}
ExportEnvironmentRecords() {
# Export environment record from fuel-devops DB
echo "=== Export environment records from DB" >&4
mkdir -p "${FUELDEVOPS_EXPORT_DIR}"
ENVIRONMENT_EXPORT_SQL="\copy \
(select * from ${DEVOPS_ENVIRONMENT_TABLE}\
where name='${SYSTEST_ENV}') \
to ${FUELDEVOPS_EXPORT_DIR}/environment.psql"
ExportItemsFromDB "${ENVIRONMENT_EXPORT_SQL}"
}
ExportNetworkRecords() {
# Export network records from fuel-devops DB
echo "=== Export network records from DB" >&4
NETWORKS_EXPORT_SQL="\copy \
(select * from ${DEVOPS_NETWORK_TABLE}\
where environment_id='${env_id}') \
to ${FUELDEVOPS_EXPORT_DIR}/networks.psql"
ExportItemsFromDB "${NETWORKS_EXPORT_SQL}"
}
ExportDomainRecords() {
# Export domain records from fuel-devops DB
echo "=== Export domain records from DB" >&4
DOMAINS_EXPORT_SQL="\copy \
(select * from ${DEVOPS_NODE_TABLE} where environment_id='${env_id}')\
to ${FUELDEVOPS_EXPORT_DIR}/domains.psql"
ExportItemsFromDB "${DOMAINS_EXPORT_SQL}"
# Get list of domains ids in the environment
DOMAINS_IDS_SQL="select id from ${DEVOPS_NODE_TABLE}\
where environment_id='${env_id}'"
domains_list=$(GetItemsFromDB "${DOMAINS_IDS_SQL}")
}
ExportInterfaceRecords() {
# Export interface records from fuel-devops DB
echo "=== Export interface records from DB" >&4
INTERFACES_EXPORT_SQL="\copy \
(select * from ${DEVOPS_INTERFACE_TABLE} \
where node_id in (${domains_list})) \
to ${FUELDEVOPS_EXPORT_DIR}/interfaces.psql"
ExportItemsFromDB "${INTERFACES_EXPORT_SQL}"
# Get list of interfaces in the environment
INTERFACES_IDS_SQL="select id from ${DEVOPS_INTERFACE_TABLE}\
where node_id in (${domains_list})"
interfaces_list=$(GetItemsFromDB "${INTERFACES_IDS_SQL}")
}
ExportAddressRecords() {
# Export address records from fuel-devops DB
echo "=== Export address records from DB" >&4
ADDRESSES_EXPORT_SQL="\copy \
(select * from ${DEVOPS_ADDRESS_TABLE} \
where interface_id in (${interfaces_list})) \
to ${FUELDEVOPS_EXPORT_DIR}/addresses.psql"
ExportItemsFromDB "${ADDRESSES_EXPORT_SQL}"
}
ExportDiskDeviceRecords() {
# Export diskdevice records from fuel-devops DB
echo "=== Export disk device records from DB" >&4
DISKDEVICES_EXPORT_SQL="\copy \
(select * from ${DEVOPS_DISKDEVICE_TABLE} \
where node_id in (${domains_list})) \
to ${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql"
ExportItemsFromDB "${DISKDEVICES_EXPORT_SQL}"
}
GetDiskDevicesFromSnapshot() {
# Takes snapshot XML definition, parses it
# and returns |-separated list of strings like
# <disk_name>\t<path_to_volume_file>
# Args: $1 - name of the domain, $2 - name of the snapshot
sed -nr -f - <(SnapshotGetXML "${1}" "${2}") <<-'SEDSCRIPT'
/<disks>/{
/<disks>/n
:loop
N
s/(.*)<disk name='([a-z]*)'(.*)<source file='(.*)'(.*)/\2\t\4|/
thold
bnext
:hold
H
:next
/<\/disks>/!bloop
x
s/|$//g
s/\n//g
p
}
SEDSCRIPT
}
SetCurrentVMState() {
# Set VM state to the state from the snapshot which is the first one
# in the list of snapshots to export
# Args: $1 - name of the domain, $2 - name of the snapshot
diskdevices=$(GetDiskDevicesFromSnapshot "${1}" "${2}")
# Get id for the domain name from exported domain records
dom_id=$(awk -v name="${1#${SYSTEST_ENV}_}" '
{ if ($2 == name) {
print $1
}
}' "${FUELDEVOPS_EXPORT_DIR}/domains.psql")
SNAPSHOT_VOLUME_IDS=""
local OLDIFS="${IFS}"
IFS="|"
# Iterate over |- separated list of diskdevices gotten from snapshot
for disk in ${diskdevices}; do
d_dev=$(echo "$disk" | awk '{ print $1 }')
d_path=$(echo "$disk" | awk '{ print $2 }')
# Get volume id connected to the disk of the domain from diskdevices
# records. (7th column)
vol_id=$(awk -v dev="${d_dev}" -v id="${dom_id}" 'BEGIN { FS=" " }
{ if (($2 == "disk") && ($5 == dev) &&
($6 == id)) {
print $7
}
}' "${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql")
# Get escaped path of the volume file
current_volpath=$(awk -v id="${vol_id}" '
{ if ($1 == id) { gsub(/\//, "\\/"); print $3 }}
' "${FUELDEVOPS_EXPORT_DIR}/volumes_all.psql")
# Escaped path of the volume gotten from snapshot XML
new_volpath=$(echo "$disk" | awk '{ gsub(/\//, "\\/"); print $2 }')
# Update domain XML definition
sed "s/${current_volpath}/${new_volpath}/g"\
"${LIBVIRT_EXPORT_DEFS_DIR}/${1}.xml" >\
"${LIBVIRT_EXPORT_DEFS_DIR}/${1}.xml.bak"
/bin/mv "${LIBVIRT_EXPORT_DEFS_DIR}/${1}.xml.bak"\
"${LIBVIRT_EXPORT_DEFS_DIR}/${1}.xml"
# Get volume id of the volume from snapshot XML
new_volid=$(awk -v uuid="${d_path}" '
{ if ($3 == uuid) { print $1 }}
' "${FUELDEVOPS_EXPORT_DIR}/volumes_all.psql")
# Update exported diskdevices records with new volume_id
awk -v dev="${d_dev}" -v id="${dom_id}" -v volid="${new_volid}" '
BEGIN { OFS="\t" }
{ if (($2 == "disk") && ($5 == dev) &&
($6 == id)) {
$7 = volid;
}
print > ARGV[1];
}' "${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql"
# Save list of recently processed volumes
SNAPSHOT_VOLUME_IDS="${SNAPSHOT_VOLUME_IDS},${new_volid}"
done
IFS="${OLDIFS}"
echo "${SNAPSHOT_VOLUME_IDS}"
}
ExportAllVolumeRecords() {
echo "=== Export all volume records from DB" >&4
VOLUMES_EXPORT_SQL="\copy \
(select * from ${DEVOPS_VOLUME_TABLE} \
where environment_id='${env_id}') \
to ${FUELDEVOPS_EXPORT_DIR}/volumes_all.psql"
ExportItemsFromDB "${VOLUMES_EXPORT_SQL}"
}
ExportVolumeRecords() {
# Export volume records requested by ids
# Args: $1 - comma-separated list of volumes ids
echo "=== Export volume records from DB" >&4
VOLUMES_EXPORT_SQL="\copy \
(select * from ${DEVOPS_VOLUME_TABLE} \
where id in (${1})) \
to ${FUELDEVOPS_EXPORT_DIR}/volumes.psql"
ExportItemsFromDB "${VOLUMES_EXPORT_SQL}"
}
ProcessSnapshotChain() {
# Recursive function. Walks through the snapshots chain and creates
# comma-separated list of all unique snapshots to SNAPSHOT_EXPORT_LIST
# Args: $1 - domain name, $2 - snapshot name
if [ "${CURRENT_SNAPSHOT}" = "${2}" ]; then
echo "First snapshot should be the tip"\
" of the exported snapshot chain" >&2
exit ${INVALIDSNAPSHOTORDER_ERR}
fi
grep -Eq "(^|,)${2}(,|$)" <<< "${SNAPSHOT_EXPORT_LIST}" || \
SNAPSHOT_EXPORT_LIST="${SNAPSHOT_EXPORT_LIST},${2}"
parent=$(SnapshotGetParent "${1}" "${2}") || return 0
ProcessSnapshotChain "${1}" "${parent}"
}
ProcessVolumeChain() {
# Recursive function. Walks through the volumes chain and creates
# comma-separated list of all volumes ids to VOLUME_EXPORT_LIST
# Args: $1 - volume id
backingStore=$(awk -v id="${1}" '
{ if (($1 == id) && ($6 != "\\N")) { print $6 }}
' "${FUELDEVOPS_EXPORT_DIR}/volumes_all.psql")
[ -z "${backingStore}" ] && return
echo "${VOLUME_EXPORT_LIST}" | grep -q "\<${backingStore}\>" || \
VOLUME_EXPORT_LIST="${VOLUME_EXPORT_LIST},${backingStore}"
ProcessVolumeChain "${backingStore}"
}
ExportInterfaceNetworkFilterDefinitions() {
# Export network filters assigned to interfaces of provided domain
# Args: $1 - domain name
local SAVEDIFS="${IFS}"
IFS=$' \t\n'
local nwfilter_list=$(sed -nr -e \
"s/\s*<filterref\s*filter=('|\")(.*)('|\")\s*\/>/\2/p" \
"${LIBVIRT_EXPORT_DEFS_DIR}/${1}.xml")
for nwfilter in ${nwfilter_list}; do
NetworkFilterGetXML "${nwfilter}" "${LIBVIRT_EXPORT_DEFS_DIR}"
done
IFS="${SAVEDIFS}"
}
ExportNetworkFilterDefinitions() {
# Export parent network filters for interface filter definitions.
# Such filters are assigned to whole networks
local nwfilter_dump_list=""
local SAVEDIFS="${IFS}"
IFS=$' \t\n'
for nwfilter_file in "${LIBVIRT_EXPORT_DEFS_DIR}/nwfilter-"*; do
if grep -Eq "<filterref" "${nwfilter_file}"; then
nwlist=$(sed -nr -e \
"s/\s*<filterref\s*filter=('|\")(.*)('|\")\s*\/>/\2/p" \
"${nwfilter_file}")
for nwfilter in ${nwlist}; do
grep -Eq "(^|,)${nwfilter}(,|$)" <<< "${nwfilter_dump_list}"|| \
nwfilter_dump_list="${nwfilter_dump_list},${nwfilter}"
done
fi
done
IFS=","
for nwfilter in ${nwfilter_dump_list##,}; do
NetworkFilterGetXML "${nwfilter}" "${LIBVIRT_EXPORT_DEFS_DIR}"
done
IFS="${SAVEDIFS}"
}
ProcessDomains() {
mkdir -p "${LIBVIRT_EXPORT_DEFS_DIR}"
OLDIFS="${IFS}"
IFS=","
EMPTY_VOLUME_LIST=""
# Create comma-separated list of volume ids for disks of cdrom type
VOLUME_CDROM_EXPORT_LIST=$(awk 'BEGIN { resStr = "" }
{ if ($2 == "cdrom") { resStr = resStr","$7 }}
END { print resStr }' \
"${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql")
VOLUME_EXPORT_LIST="${VOLUME_CDROM_EXPORT_LIST}"
# Iterate over comma-separated list of domains names
for domain in ${domain_names}; do
unset CURRENT_SNAPSHOT
HEAD_VOLUME_IDS=""
SNAPSHOT_EXPORT_LIST=""
DomainGetXML "${domain}" "${LIBVIRT_EXPORT_DEFS_DIR}"
# Set to 1 if using host-passthrough cpu mode
HOST_PASSTHROUGH_CPU_MODE=$(\
grep -Ec "cpu mode=('|\")host-passthrough('|\")" \
"${LIBVIRT_EXPORT_DEFS_DIR}/${domain}.xml")
ExportInterfaceNetworkFilterDefinitions "${domain}"
# Iterate over list of requested snapshots to export
for SNAPSHOT_NAME in ${SNAPSHOT_NAMES}; do
if [ "$(SnapshotGetCurrent "${domain}")" != "${SNAPSHOT_NAME}" ] &&\
[ -z "${CURRENT_SNAPSHOT}" ]; then
# If requested snapshot is not set as a current on exported
# environment and it is the first in the list to export
# we set VM state to the state from this snapshot
echo "=== Changing ${domain} VM state" >&4
NEW_CURRENT_VOLUME_IDS=$(SetCurrentVMState "${domain}"\
"${SNAPSHOT_NAME}")
HEAD_VOLUME_IDS="${HEAD_VOLUME_IDS}${NEW_CURRENT_VOLUME_IDS}"
else
# Otherwise we collect volumes ids to export in
# HEAD_VOLUME_IDS variable
diskdevices=$(GetDiskDevicesFromSnapshot "${domain}"\
"${SNAPSHOT_NAME}" | \
awk 'BEGIN {RS="|"; ORS=","} { print $2 }')
for disk in ${diskdevices%%,}; do
new_volid=$(awk -v uuid="${disk}" '
{ if ($3 == uuid) { print $1 }}
' "${FUELDEVOPS_EXPORT_DIR}/volumes_all.psql")
HEAD_VOLUME_IDS="${HEAD_VOLUME_IDS},${new_volid}"
done
fi
# Get snapshot names to export from the chain
ProcessSnapshotChain "${domain}" "${SNAPSHOT_NAME}"
# We do not export volumes of upper levels if it is not requested,
# as these volumes does not relate to the state from snapshot
if [ -n "${CURRENT_SNAPSHOT}" ] || [ "${EXPORT_CHANGES}" -eq 0 ];
then
EMPTY_VOLUME_LIST="${EMPTY_VOLUME_LIST}${HEAD_VOLUME_IDS}"
else
VOLUME_EXPORT_LIST="${VOLUME_EXPORT_LIST}${HEAD_VOLUME_IDS}"
fi
CURRENT_SNAPSHOT="${CURRENT_SNAPSHOT:-${SNAPSHOT_NAME}}"
done
# Get volume ids to export from the chain
for volume in ${HEAD_VOLUME_IDS##,}; do
ProcessVolumeChain "${volume}"
done
done
ExportNetworkFilterDefinitions
IFS="${OLDIFS}"
}
ExportStoragePoolDefinition() {
echo "=== Export definition of storage pool" >&4
StoragePoolGetXML "${STORAGE_POOL}" >\
"${LIBVIRT_EXPORT_DEFS_DIR}/pool-${STORAGE_POOL}.xml"
}
ExportNetworkDefinitions() {
echo "=== Export definitions of networks" >&4
awk -v env="${SYSTEST_ENV}" '{ print env"_"$2 }' \
"${FUELDEVOPS_EXPORT_DIR}/networks.psql" | while read network; do
NetworkGetXML "${network}" "${LIBVIRT_EXPORT_DEFS_DIR}"
done
}
ExportVolumeDefinitions() {
echo "=== Export definitions of volumes" >&4
# Get volumes XML definitions
awk -v env="${SYSTEST_ENV}" '{ print env"_"$2 }' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql" |
while read -r volume; do
VolumeGetXML "${volume}" "${LIBVIRT_EXPORT_DEFS_DIR}" "${STORAGE_POOL}"
done
# Create |-separated list of volumes ids as this list can act as a regexp
VOLUME_CDROM_LIST_REGEX=$(echo "${VOLUME_CDROM_EXPORT_LIST##,}" | \
sed 's/,/|/g')
# Get volumes names for the volumes of cdrom type
cdrom_volumes=$(awk -v env="${SYSTEST_ENV}"\
-v id="^(${VOLUME_CDROM_LIST_REGEX})$" '
{ if ($1 ~ id) {print env"_"$2} }' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql")
# Correct type for volumes of cdrom type in XML definition
# It must be 'raw' but in the exported XML we have 'iso'
for volume in ${cdrom_volumes}; do
sed "s/type='iso'/type='raw'/g" \
"${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml" > \
"${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml.new"
/bin/mv "${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml.new"\
"${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml"
done
# Create file with names of volumes for which we do not export
# content
echo > "${FUELDEVOPS_EXPORT_DIR}/volumes_empty.txt"
if [ -n "${EMPTY_VOLUME_LIST}" ]; then
VOLUMES_EMPTY_SQL="select concat('${SYSTEST_ENV}_', name)\
from ${DEVOPS_VOLUME_TABLE} \
where id in (${EMPTY_VOLUME_LIST##,});"
RunPSQL "${VOLUMES_EMPTY_SQL}" > \
"${FUELDEVOPS_EXPORT_DIR}/volumes_empty.txt"
fi
}
ExportVolumesContent() {
mkdir -p "${LIBVIRT_EXPORT_VOLUMES_DIR}"
echo "=== Download volumes" >&4
VOLUMES_DOWNLOAD_SQL="select name from ${DEVOPS_VOLUME_TABLE} \
where id in (${VOLUME_EXPORT_LIST##,});"
volume_names=$(RunPSQL "${VOLUMES_DOWNLOAD_SQL}")
for volume in ${volume_names}; do
if [ "${WORKAROUND}" -eq 0 ]; then
VolumeDownload "${SYSTEST_ENV}_${volume}"\
"${LIBVIRT_EXPORT_VOLUMES_DIR}" "${STORAGE_POOL}"
else
# Workaround for case when libvirt vol-download shows poor
# performance. It has 2 options: run w/ sudo and w/o sudo.
# bug: https://bugzilla.redhat.com/show_bug.cgi?id=1026136
volpath=$(awk -v vol="${volume}" '{
if ($2 == vol) { print $3 } }' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql")
if [ "${WORKAROUND}" -eq 1 ]; then
if [ "${DEFS_ONLY}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " /bin/cp -f ${volpath} ${LIBVIRT_EXPORT_VOLUMES_DIR}"
fi
if [ "${DEFS_ONLY}" -eq 0 ]; then
/bin/cp -f "${volpath}" "${LIBVIRT_EXPORT_VOLUMES_DIR}"
fi
else
if [ "${DEFS_ONLY}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " sudo /bin/cp -f ${volpath} ${LIBVIRT_EXPORT_VOLUMES_DIR}"
fi
if [ "${DEFS_ONLY}" -eq 0 ]; then
sudo /bin/cp -f "${volpath}" "${LIBVIRT_EXPORT_VOLUMES_DIR}"
fi
fi
fi
done
}
ExportSnapshotDefinitions() {
echo "=== Export definitions of snapshots and memory state files" >&4
OLDIFS="${IFS}"
IFS=","
mkdir -p "${LIBVIRT_EXPORT_MEMSTATE_DIR}"
for domain in ${domain_names}; do
domain_dir="${LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR}/${domain}"
mkdir -p "${domain_dir}"
SNAPSHOT_ACTIVE=1
if [ "${HOST_PASSTHROUGH_CPU_MODE}" -eq 1 ]; then
# Get CPU configuration from domain definitions in case of
# host-passthrough cpu mode and change all new lines to
# \n symbols to be able to use this in sed.
DOMAIN_CPU_NEWLINE=""
while IFS='' read -r line; do
DOMAIN_CPU_NEWLINE="${DOMAIN_CPU_NEWLINE}${line}\n"
done < <(sed -n "/^\s*<cpu mode/,/^\s*<\/cpu>/p"\
"${LIBVIRT_EXPORT_DEFS_DIR}/${domain}.xml")
fi
for snapshot in ${SNAPSHOT_EXPORT_LIST##,}; do
SnapshotGetXML "${domain}" "${snapshot}" >\
"${domain_dir}/${snapshot}.xml"
# Add active XML node to snapshot definition and set it to 1
# if snapshot is current (first in the list), otherwise to 0
sed "s/<\/domainsnapshot>/ <active>${SNAPSHOT_ACTIVE}<\/active>\n<\/domainsnapshot>/g"\
"${domain_dir}/${snapshot}.xml" > \
"${domain_dir}/${snapshot}.xml.new"
if [ "${HOST_PASSTHROUGH_CPU_MODE}" -eq 1 ]; then
# Replace CPU configuration in snapshot definition with one
# from domain definition.
# First, we have to escape all forward slashes
SEDREADY_DOMAIN_CPU="${DOMAIN_CPU_NEWLINE//\//\\/}"
sed -n '/\s*<cpu mode/{:loop;N;/\s*<\/cpu>/!bloop;N;s/.*\n/'"${SEDREADY_DOMAIN_CPU}"'/};p' \
"${domain_dir}/${snapshot}.xml.new" > "${domain_dir}/${snapshot}.xml"
/bin/rm "${domain_dir}/${snapshot}.xml.new"
else
/bin/mv "${domain_dir}/${snapshot}.xml.new"\
"${domain_dir}/${snapshot}.xml"
fi
memstate_file=$(sed -nre\
"s/\s+<memory\s+snapshot=('|\")external('|\")\s+(file=('|\")(.*)('|\")\/>)?/\5/gp"\
"${domain_dir}/${snapshot}.xml")
if [ -n "${memstate_file}" ]; then
if [ "${DEFS_ONLY}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " sudo /bin/cp --no-preserve=mode ${memstate_file}"\
" ${LIBVIRT_EXPORT_MEMSTATE_DIR}"
fi
if [ "${DEFS_ONLY}" -eq 0 ]; then
sudo /bin/cp --no-preserve=mode "${memstate_file}"\
"${LIBVIRT_EXPORT_MEMSTATE_DIR}" || \
exit ${MEMORYSTATEEXPORT_ERR}
fi
fi
SNAPSHOT_ACTIVE=0
done
done
IFS="${OLDIFS}"
}
GetoptsVariables "${@}"
GlobalVariables
. "${baseDir}/libvirt_functions.sh"
echo "=== Check environment" >&4
CheckEnvironmentExists
CheckSnapshotExists
ExportEnvironmentRecords
ExportNetworkRecords
ExportDomainRecords
ExportInterfaceRecords
ExportAddressRecords
ExportDiskDeviceRecords
ExportAllVolumeRecords
ProcessDomains
VOLUME_EXPORT_FULL_LIST="${EMPTY_VOLUME_LIST}${VOLUME_EXPORT_LIST}"
ExportVolumeRecords "${VOLUME_EXPORT_FULL_LIST##,}"
ExportStoragePoolDefinition
ExportNetworkDefinitions
ExportVolumeDefinitions
ExportVolumesContent
ExportSnapshotDefinitions

484
bin/env_import.sh Executable file
View File

@ -0,0 +1,484 @@
#!/bin/bash
set -o errexit
# Set xtrace option if run by Jenkins
if [ -n "${WORKSPACE}" ]; then
set -o xtrace
fi
baseDir=$(dirname "$0")
VERBOSITY=0
DRY_RUN=1
DEBUG=0
LIBVIRT_DAEMON_NAME="${LIBVIRT_DAEMON_NAME:-libvirt-bin}"
INVALIDOPTS_ERR=100
FILENOTFOUND_ERR=101
STORAGEPOOLCONF_ERR=102
STORAGEPOOLDEFINE_ERR=103
STORAGEPOOLSTART_ERR=104
STORAGEPOOLAUTOSTART_ERR=105
NETWORKDEFINE_ERR=106
NETWORKSTART_ERR=107
NETWORKAUTOSTART_ERR=108
VOLUMECREATE_ERR=109
VOLUMEUPLOAD_ERR=110
DOMAINDEFINE_ERR=111
SNAPSHOTIMPORT_ERR=112
MEMORYSTATEIMPORT_ERR=113
EMULATORNOTFOUND_ERR=114
ShowHelp() {
cat << EOF
Usage: $0 [-hiv[vv]] DUMP_PATH
DUMP_PATH - Path to the directory where exported environment files are stored.
The directory must contain the following data:
psql files with data to copy to fuel-devops DB, text file with
the list of empty volumes (i.e. volumes that have only definitions
but have no appropriate binary files), volume binary files, memory
state files, files with libvirt XML definitions for domains,
networks, volumes, storage pool, network filters and snapshots.
NOTE: Extra privileges are necessary to copy XML files with libvirt snapshot
definitions and restart libvirt daemon with sudo.
The following options are available:
-h - Show this help page.
-i - Import environment.
By default the script imports nothing. It only checks for
necessary files and displays commands to import the environment.
-v - Set verbose log level.
-vv - Increase verbosity level.
-vvv - Set debug log level (print commands as well).
You can override the following variables:
DEVOPS_DB_NAME - fuel-devops DB name
DEVOPS_DB_HOST - fuel-devops DB hostname
DEVOPS_DB_USER - fuel-devops DB username
DEVOPS_DB_PASSWORD - fuel-devops DB password
CONNECTION_STRING - hypervisor connection URI
LIBVIRT_DAEMON_NAME - name of the libvirt daemon (libvirt-bin by default)
STORAGE_POOL_NAME - name of the storage pool to use
SNAPSHOTS_DIR - directory where libvirt stores files with XML definitions
for snapshots on the target host
EOF
}
exec 3>&1
GetoptsVariables() {
while getopts ":hiv" opt; do
case $opt in
h)
ShowHelp
exit 0
;;
i)
DRY_RUN=0
;;
v)
VERBOSITY=$((VERBOSITY+1))
;;
\?)
ShowHelp
exit ${INVALIDOPTS_ERR}
;;
:)
echo "Option -${OPTARG} requires an argument." >&2
ShowHelp
exit ${INVALIDOPTS_ERR}
;;
esac
done
shift $((OPTIND-1))
if [ ${#} -ne 1 ]; then
ShowHelp
exit ${INVALIDOPTS_ERR}
fi
case ${VERBOSITY} in
0)
exec 4> /dev/null
exec 5> /dev/null
;;
1)
exec 4>&1
exec 5> /dev/null
;;
2)
exec 4>&1
exec 5>&1
;;
*)
DEBUG=1
exec 4>&1
exec 5>&1
;;
esac
DUMP_PATH="${1}"
[ "${DRY_RUN}" -eq 0 ] ||
echo "Dry run mode. Nothing will be imported"
}
GlobalVariables() {
echo "Using DB Name: ${DEVOPS_DB_NAME:=fuel_devops}" >&4
echo "Using DB User: ${DEVOPS_DB_USER:=fuel_devops}" >&4
echo "Using DB Host: ${DEVOPS_DB_HOST:=localhost}" >&4
export PGPASSWORD="${DEVOPS_DB_PASSWORD:-fuel_devops}"
FUELDEVOPS_EXPORT_DIR="${DUMP_PATH}/fuel-devops"
LIBVIRT_EXPORT_DIR="${DUMP_PATH}/libvirt"
LIBVIRT_EXPORT_DEFS_DIR="${LIBVIRT_EXPORT_DIR}/definitions"
LIBVIRT_EXPORT_VOLUMES_DIR="${LIBVIRT_EXPORT_DIR}/volumes"
LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR="${LIBVIRT_EXPORT_DIR}/snapshot"
LIBVIRT_EXPORT_MEMSTATE_DIR="${LIBVIRT_EXPORT_DIR}/memstate"
CONNECTION_STRING="${CONNECTION_STRING:-"qemu:///system"}"
SNAPSHOTS_DIR="${SNAPSHOTS_DIR:-"/var/lib/libvirt/qemu/snapshot"}"
STORAGE_POOL="${STORAGE_POOL_NAME:-default}"
# Tables in fuel-devops DB
DEVOPS_ENVIRONMENT_TABLE="devops_environment"
DEVOPS_NETWORK_TABLE="devops_network"
DEVOPS_VOLUME_TABLE="devops_volume"
DEVOPS_NODE_TABLE="devops_node"
DEVOPS_INTERFACE_TABLE="devops_interface"
DEVOPS_ADDRESS_TABLE="devops_address"
DEVOPS_DISKDEVICE_TABLE="devops_diskdevice"
}
ImportItemsToDB() {
# Args: $1 - SQL statement
if [ "${DEBUG}" -ne 0 ] || [ "${DRY_RUN}" -ne 0 ]; then
printf " Run: psql -h %s -U %s -c \"%s\"\n" "${DEVOPS_DB_HOST}" \
"${DEVOPS_DB_USER}" "${@}" | sed 's/[ ]\+/ /g'
fi
if [ "${DRY_RUN}" -eq 0 ]; then
psql -h "${DEVOPS_DB_HOST}" -U "${DEVOPS_DB_USER}" -c "${1}" >&5
fi
}
CheckFile() {
# Check if the file exists, set ERROR_FOUND to 1 otherwise
# Args: $1 - file path
[ -f "${1}" ] || {
ERROR_FOUND=1
echo "No ${1} file found" >&2
}
}
CheckFuelDevopsFiles() {
# Check if the files to import to fuel-devops DB exist.
# Abort script otherwise
ERROR_FOUND=0
echo "=== Checking for exported tables from fuel-devops DB..." >&4
CheckFile "${FUELDEVOPS_EXPORT_DIR}/environment.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/networks.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/domains.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/interfaces.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/addresses.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/volumes.psql"
CheckFile "${FUELDEVOPS_EXPORT_DIR}/volumes_empty.txt"
# Compare ERROR_FOUND to 0 !!!!
[ "${ERROR_FOUND}" -eq 0 ] || exit ${FILENOTFOUND_ERR}
}
CheckNWFilters() {
# Recursive function.
# Check for network filter definition XML files for filters.
# taken from domain and interface network filter XML definitions.
# Top level NW filters (i.e. the filters that have no reference to
# another NW filters) are saved to TOP_NWFILTERS.
# Args: $1 - path to XML file
local nwfilter_list=$(sed -nr -e \
"s/\s*<filterref\s*filter=('|\")(.*)('|\")\s*\/>/\2/p" \
"${1}")
for nwfilter in ${nwfilter_list}; do
local nwfilter_xml="${LIBVIRT_EXPORT_DEFS_DIR}/nwfilter-${nwfilter}.xml"
CheckFile "${nwfilter_xml}"
if [ -f "${nwfilter_xml}" ] && grep -Eq "<filterref" "${nwfilter_xml}"; then
CheckNWFilters "${nwfilter_xml}"
else
grep -Eq "(^|,)${nwfilter}(,|$)" <<< "${TOP_NWFILTERS}" || \
TOP_NWFILTERS="${TOP_NWFILTERS},${nwfilter}"
fi
done
}
CheckEnvFiles() {
# Check for the files for libvirt.
# Abort script otherwise
local ERROR_FOUND=0
local EMULATOR_MISSED=0
TOP_NWFILTERS=""
echo "=== Checking for storage pool definition..." >&4
env_name=$(awk '{ print $2 }' "${FUELDEVOPS_EXPORT_DIR}/environment.psql")
pool_xmlfile="${LIBVIRT_EXPORT_DEFS_DIR}/pool-${STORAGE_POOL}.xml"
CheckFile "${pool_xmlfile}"
echo "=== Checking for domain and network filter definitions..." >&4
domains=$(awk '{ print $2 }' "${FUELDEVOPS_EXPORT_DIR}/domains.psql")
for domain in ${domains}; do
domain_xmlfile="${LIBVIRT_EXPORT_DEFS_DIR}/${env_name}_${domain}.xml"
CheckFile "${domain_xmlfile}"
local emulator=$(sed -nr "s/\s*<emulator>(.*)<\/emulator>/\1/p" \
"${domain_xmlfile}")
if [ ! -f "${emulator}" ]; then
EMULATOR_MISSED=1
echo "No emulator ${emulator} found" >&2
fi
CheckNWFilters "${domain_xmlfile}"
done
echo "=== Checking for network definitions..." >&4
networks=$(awk -v env="${env_name}" '{ print env"_"$2 }' \
"${FUELDEVOPS_EXPORT_DIR}/networks.psql")
for network in ${networks}; do
net_xmlfile="${LIBVIRT_EXPORT_DEFS_DIR}/network-${network}.xml"
CheckFile "${net_xmlfile}"
done
echo "=== Checking for volume definitions and content files..." >&4
volumes=$(awk -v env="${env_name}" '{ print env"_"$2 }' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql")
for volume in ${volumes}; do
vol_xmlfile="${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml"
vol_binfile="${LIBVIRT_EXPORT_VOLUMES_DIR}/${volume}"
CheckFile "${vol_xmlfile}"
if ! grep -q "^${volume}$" "${FUELDEVOPS_EXPORT_DIR}/volumes_empty.txt"
then
[ -f "${vol_binfile}" ] || {
ERROR_FOUND=1
echo "No ${vol_binfile} file found" >&2
}
fi
done
[ "${ERROR_FOUND}" -eq 0 ] || exit ${FILENOTFOUND_ERR}
[ "${EMULATOR_MISSED}" -eq 0 ] || exit ${EMULATORNOTFOUND_ERR}
}
ImportFuelDevopsDB() {
echo "=== Import environment to fuel-devops DB..." >&4
ENVIRONMENTS_IMPORT_SQL="\copy ${DEVOPS_ENVIRONMENT_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/environment.psql"
ImportItemsToDB "${ENVIRONMENTS_IMPORT_SQL}"
echo "=== Import networks to fuel-devops DB..." >&4
NETWORKS_IMPORT_SQL="\copy ${DEVOPS_NETWORK_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/networks.psql"
ImportItemsToDB "${NETWORKS_IMPORT_SQL}"
echo "=== Import domains to fuel-devops DB..." >&4
DOMAINS_IMPORT_SQL="\copy ${DEVOPS_NODE_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/domains.psql"
ImportItemsToDB "${DOMAINS_IMPORT_SQL}"
echo "=== Import interfaces to fuel-devops DB..." >&4
INTERFACES_IMPORT_SQL="\copy ${DEVOPS_INTERFACE_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/interfaces.psql"
ImportItemsToDB "${INTERFACES_IMPORT_SQL}"
echo "=== Import addresses to fuel-devops DB..." >&4
ADDRESSES_IMPORT_SQL="\copy ${DEVOPS_ADDRESS_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/addresses.psql"
ImportItemsToDB "${ADDRESSES_IMPORT_SQL}"
echo "=== Import volumes to fuel-devops DB..." >&4
VOLUMES_IMPORT_SQL="\copy ${DEVOPS_VOLUME_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/volumes.psql"
ImportItemsToDB "${VOLUMES_IMPORT_SQL}"
echo "=== Import diskdevices to fuel-devops DB..." >&4
DISKDEVICES_IMPORT_SQL="\copy ${DEVOPS_DISKDEVICE_TABLE}\
from ${FUELDEVOPS_EXPORT_DIR}/diskdevices.psql"
ImportItemsToDB "${DISKDEVICES_IMPORT_SQL}"
}
PrepareStoragePool() {
echo "=== Checking if storage pool is already exist..." >&4
StoragePoolInfo "${STORAGE_POOL}" > /dev/null
if [ $? -eq 0 ]; then
# Get dir path for the storage pool from the exported definition
srcPoolDir=$(awk '{if ($0 ~ /<path>/) {
gsub(/<\/?path>/, "");
print $1;
}}' "${LIBVIRT_EXPORT_DEFS_DIR}/pool-${STORAGE_POOL}.xml")
# Get dir path from the existing storage pool
targetPoolDir=$(StoragePoolGetXML "${STORAGE_POOL}" |\
awk '{if ($0 ~ /<path>/) {
gsub(/<\/?path>/, "");
print $1;}}')
echo " Checking the storage pool configuration..." >&5
if [ "${srcPoolDir}" != "${targetPoolDir}" ]; then
echo " Default Storage pool: \"${STORAGE_POOL}\" on target system"\
"is set to ${targetPoolDir}, but on source system"\
"${srcPoolDir} is used." >&2
exit ${STORAGEPOOLCONF_ERR}
fi
echo " Storage pool \"${STORAGE_POOL}\" is already configured..." >&5
else
echo " Defining the storage pool..." >&5
StoragePoolDefine "${LIBVIRT_EXPORT_DEFS_DIR}/pool-${STORAGE_POOL}.xml"\
|| exit ${STORAGEPOOLDEFINE_ERR}
fi
echo " Start the storage pool if it is stopped..." >&5
if [ "$(StoragePoolInfo "${STORAGE_POOL}" | awk '{
if ($0 ~ /State/) { print $2 } }')" = "inactive" ]; then
StoragePoolStart "${STORAGE_POOL}" || exit ${STORAGEPOOLSTART_ERR}
echo "Started." >&5
fi
echo " Set autostart to the storage pool if it is not enabled..." >&5
if [ "$(StoragePoolInfo "${STORAGE_POOL}" | awk '{
if ($0 ~ /Autostart/) { print $2 } }')" = "no" ]; then
StoragePoolAutoStart "${STORAGE_POOL}" ||\
exit ${STORAGEPOOLAUTOSTART_ERR}
echo "Autostart enabled." >&5
fi
}
PrepareNetworks() {
echo "=== Defining networks..." >&4
for network in ${networks}; do
echo " Defining network: ${network}" >&5
NetworkDefine "${LIBVIRT_EXPORT_DEFS_DIR}/network-${network}.xml" ||\
exit ${NETWORKDEFINE_ERR}
echo " Set autostart for network: ${network}" >&5
NetworkAutoStart "${network}" ||\
exit ${NETWORKAUTOSTART_ERR}
echo " Start network: ${network}" >&5
NetworkStart "${network}" ||\
exit ${NETWORKSTART_ERR}
done
}
PrepareVolumes() {
# Recursive function. Finds volumes which have the provided volumes as
# backing stores, then defines ones and upload content to libvirt
# Args: $1 - |-separated list of volumes id which are backing stores
# Get volumes id which have the provided backing stores
volumes_ids=$(awk -v env="${env_name}" -v backingStore="^(${1})$" '
BEGIN { resStr = "" }
{ if ($6 ~ backingStore) { resStr = resStr"|"$1 }}
END { print resStr }' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql")
# Get volumes names which have the provided backing stores
volumes=$(awk -v env="${env_name}" -v backingStore="^(${1})$"\
'{ if ($6 ~ backingStore) { print env"_"$2 }}' \
"${FUELDEVOPS_EXPORT_DIR}/volumes.psql")
# Exit if there no volumes found
[ ${#volumes} -eq 0 ] && return
# Define and upload volume content for all volumes from gotten list
for volume in ${volumes}; do
echo " Creating volume: ${volume}" >&5
VolumeCreate "${LIBVIRT_EXPORT_DEFS_DIR}/vol-${volume}.xml"\
"${STORAGE_POOL}" || exit ${VOLUMECREATE_ERR}
# Upload content only for volumes which names are not in the
# volumes_empty file
grep -q "^${volume}$" "${FUELDEVOPS_EXPORT_DIR}/volumes_empty.txt" ||\
VolumeUpload "${volume}" "${LIBVIRT_EXPORT_VOLUMES_DIR}"\
"${STORAGE_POOL}" || exit ${VOLUMEUPLOAD_ERR}
done
PrepareVolumes "${volumes_ids##|}"
}
DefineVolumes() {
# Create all volumes
echo "=== Defining volumes..." >&4
# Start from volumes that have no backing stores (\N - in the volumes.psql)
# It needs to escape backslash twice
PrepareVolumes '\\\\N'
}
DefineNWFilters() {
echo "=== Defining network filters" >&4
echo " Defining top level network filters" >&5
OLDIFS="${IFS}"
IFS=","
for nwfilter in ${TOP_NWFILTERS##,}; do
echo " Defining network filter: ${nwfilter}" >&5
NetworkFilterDefine \
"${LIBVIRT_EXPORT_DEFS_DIR}/nwfilter-${nwfilter}.xml"
done
for nwfilter_xml in "${LIBVIRT_EXPORT_DEFS_DIR}/nwfilter-"*; do
nwf_name=$(basename "${nwfilter_xml}")
nwf_name="${nwf_name##nwfilter-}"
nwf_name="${nwf_name%%.xml}"
grep -Eq "(^|,)${nwf_name}(,|$)" <<< "${TOP_NWFILTERS}" && continue
echo " Defining network filter: ${nwf_name}" >&5
NetworkFilterDefine "${nwfilter_xml}"
done
IFS="${OLDIFS}"
}
DefineDomains() {
echo "=== Defining domains..." >&4
for domain in ${domains}; do
echo " Defining domain: ${domain}" >&5
DomainDefine "${LIBVIRT_EXPORT_DEFS_DIR}/${env_name}_${domain}.xml" ||\
exit ${DOMAINDEFINE_ERR}
done
}
ImportSnapshots() {
echo "=== Importing snapshots and memory state files..." >&4
echo " Import snapshot definitions" >&5
if [ "${DRY_RUN}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " sudo /bin/cp -fr ${LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR}/*" \
"${SNAPSHOTS_DIR}"
fi
if [ "${DRY_RUN}" -eq 0 ]; then
sudo /bin/cp -fr "${LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR}"/* \
"${SNAPSHOTS_DIR}" || exit ${SNAPSHOTIMPORT_ERR}
fi
echo " Import memory state files" >&5
for snapshot_path in "${LIBVIRT_EXPORT_SNAPSHOTS_DEFS_DIR}"/*/*; do
memstate_path=$(sed -nre\
"s/\s+<memory\s+snapshot=('|\")external('|\")\s+(file=('|\")(.*)('|\")\/>)?/\5/gp"\
"${snapshot_path}")
if [ -n "${memstate_path}" ]; then
echo " Domain: $(basename "$(dirname "${snapshot_path}")")" \
" snapshot: $(basename "${snapshot_path}" | sed 's/\.xml$//g')" >&5
memstate_dir=$(dirname "${memstate_path}")
memstate_file=$(basename "${memstate_path}")
mkdir -p "${memstate_dir}"
if [ "${DRY_RUN}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " /bin/cp -f ${LIBVIRT_EXPORT_MEMSTATE_DIR}/${memstate_file} "\
"${memstate_path}"
fi
if [ "${DRY_RUN}" -eq 0 ]; then
/bin/cp -f "${LIBVIRT_EXPORT_MEMSTATE_DIR}/${memstate_file}" \
"${memstate_path}" || exit ${MEMORYSTATEIMPORT_ERR}
fi
fi
done
echo "=== Restarting ${LIBVIRT_DAEMON_NAME} daemon..." >&4
if [ "${DRY_RUN}" -eq 1 ] || [ "${DEBUG}" -eq 1 ]; then
echo " sudo /usr/sbin/service ${LIBVIRT_DAEMON_NAME} restart"
fi
if [ "${DRY_RUN}" -eq 0 ]; then
sudo /usr/sbin/service "${LIBVIRT_DAEMON_NAME}" restart >&5
fi
}
GetoptsVariables "${@}"
GlobalVariables
. "${baseDir}/libvirt_functions.sh"
CheckFuelDevopsFiles
CheckEnvFiles
ImportFuelDevopsDB
PrepareStoragePool
PrepareNetworks
DefineVolumes
DefineNWFilters
DefineDomains
ImportSnapshots

143
bin/libvirt_functions.sh Normal file
View File

@ -0,0 +1,143 @@
#!/bin/bash
PrintCommand() {
echo " virsh -c ${CONNECTION_STRING} ${*}" | sed 's/,/ /g' >&3
}
RunCommand() {
if [ "${DEBUG}" -ne 0 ]; then
PrintCommand "${@}"
fi
virsh -c "${CONNECTION_STRING}" "${@}"
}
CheckSnapshot() {
RunCommand snapshot-list "${1}" --name | grep -q "^${2}$"
}
DomainDefine() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand define "${1}"
else
RunCommand define "${1}" >&5
fi
}
DomainGetXML() {
RunCommand dumpxml "${1}" --update-cpu > "${2}/${1}.xml"
}
NetworkAutoStart() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand net-autostart "${@}"
else
RunCommand net-autostart "${@}" >&5
fi
}
NetworkDefine() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand net-define "${1}"
else
RunCommand net-define "${1}" >&5
fi
}
NetworkFilterGetXML() {
RunCommand nwfilter-dumpxml "${1}" |
sed -r "/<uuid>(.*)<\/uuid>/d" > "${2}/nwfilter-${1}.xml"
}
NetworkFilterDefine() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand nwfilter-define "${1}"
else
RunCommand nwfilter-define "${1}" >&5
fi
}
NetworkGetXML() {
RunCommand net-dumpxml "${1}" > "${2}/network-${1}.xml"
}
NetworkStart() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand net-start "${1}"
else
RunCommand net-start "${1}" >&5
fi
}
SnapshotGetCurrent() {
RunCommand snapshot-current "${1}" --name 2>&5
}
SnapshotGetParent() {
RunCommand snapshot-parent "${1}" --snapshotname "${2}" 2>&5
}
SnapshotGetXML() {
RunCommand snapshot-dumpxml "${1}" --snapshotname "${2}" 2>&5
}
StoragePoolAutoStart() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand pool-autostart "${@}"
else
RunCommand pool-autostart "${@}" >&5
fi
}
StoragePoolDefine() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand pool-define "${@}"
else
RunCommand pool-define "${@}" >&5
fi
}
StoragePoolGetXML() {
RunCommand pool-dumpxml "${@}"
}
StoragePoolInfo() {
RunCommand pool-info "${@}"
}
StoragePoolStart() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand pool-start "${@}"
else
RunCommand pool-start "${@}" >&5
fi
}
VolumeCreate() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand vol-create "${2:-default}" "${1}"
else
RunCommand vol-create "${2:-default}" "${1}" >&5
fi
}
VolumeDownload() {
if [ "${DEFS_ONLY:-0}" -ne 0 ]; then
PrintCommand vol-download "${1}" "${2}/${1}" --pool "${3:-default}"
else
RunCommand vol-download "${1}" "${2}/${1}" --pool "${3:-default}"
fi
}
VolumeUpload() {
if [ "${DRY_RUN:-0}" -ne 0 ]; then
PrintCommand vol-upload "${1}" "${2}/${1}" --pool "${3:-default}"
else
RunCommand vol-upload "${1}" "${2}/${1}" --pool "${3:-default}" >&5
fi
}
VolumeGetXML() {
RunCommand vol-dumpxml "${1}" --pool "${3:-default}" > "${2}/vol-${1}.xml"
}

View File

@ -32,7 +32,9 @@ setup(
data_files=[
(os.path.expanduser('~/.devops'), ['devops/log.yaml']),
(os.path.expanduser('~/.devops/log'), [])],
scripts=['bin/dos.py'],
scripts=['bin/dos.py',
'bin/env_export.sh', 'bin/env_import.sh',
'bin/libvirt_functions.sh'],
install_requires=[
'xmlbuilder',
'netaddr',