Adds support for LVM ephemeral storage in Nova

DevStack currently lacks support for LVM ephemeral storage in Nova.
This support is important for testing of Nova's LVM backend. The
proposed change adds a default volume group, to be shared by Cinder
and Nova. It also adds a configuration option NOVA_BACKEND, which
must be LVM if it is set, that determines whether Nova should be
configured to use LVM ephemeral storage.

Change-Id: I4eb9afff3536fbcd563939f2d325efbb845081bb
This commit is contained in:
Daniel Genin 2014-10-31 15:01:29 -04:00
parent 62002dd88b
commit d470867f08
7 changed files with 150 additions and 125 deletions

View File

@ -110,8 +110,8 @@ follows:
* Global configuration that may be referenced in ``local.conf``, i.e. ``DEST``, ``DATA_DIR`` * Global configuration that may be referenced in ``local.conf``, i.e. ``DEST``, ``DATA_DIR``
* Global service configuration like ``ENABLED_SERVICES`` * Global service configuration like ``ENABLED_SERVICES``
* Variables used by multiple services that do not have a clear owner, i.e. * Variables used by multiple services that do not have a clear owner, i.e.
``VOLUME_BACKING_FILE_SIZE`` (nova-volumes and cinder) or ``PUBLIC_NETWORK_NAME`` ``VOLUME_BACKING_FILE_SIZE`` (nova-compute, nova-volumes and cinder) or
(nova-network and neutron) ``PUBLIC_NETWORK_NAME`` (nova-network and neutron)
* Variables that can not be cleanly declared in a project file due to * Variables that can not be cleanly declared in a project file due to
dependency ordering, i.e. the order of sourcing the project files can dependency ordering, i.e. the order of sourcing the project files can
not be changed for other reasons but the earlier file needs to dereference a not be changed for other reasons but the earlier file needs to dereference a

View File

@ -76,9 +76,9 @@ CINDER_MULTI_LVM_BACKEND=$(trueorfalse False CINDER_MULTI_LVM_BACKEND)
# configuration and for the volume type name. Multiple backends are # configuration and for the volume type name. Multiple backends are
# comma-separated. # comma-separated.
if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then if [[ $CINDER_MULTI_LVM_BACKEND == "False" ]]; then
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1} CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-}}
else else
CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:lvmdriver-1,lvm:lvmdriver-2} CINDER_ENABLED_BACKENDS=${CINDER_ENABLED_BACKENDS:-lvm:${DEFAULT_VOLUME_GROUP_NAME##*-},lvm:cinder}
fi fi

View File

@ -14,6 +14,7 @@
# CINDER_CONF # CINDER_CONF
# DATA_DIR # DATA_DIR
# VOLUME_GROUP_NAME
# clean_cinder_backend_lvm - called from clean_cinder() # clean_cinder_backend_lvm - called from clean_cinder()
# configure_cinder_backend_lvm - called from configure_cinder() # configure_cinder_backend_lvm - called from configure_cinder()
@ -25,157 +26,44 @@ MY_XTRACE=$(set +o | grep xtrace)
set +o xtrace set +o xtrace
# Defaults
# --------
# Name of the lvm volume groups to use/create for iscsi volumes
# This monkey-motion is for compatibility with icehouse-generation Grenade
# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
# on ``VOLUME_GROUP_NAME`` that includes the backend name
# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
VOLUME_GROUP=${VOLUME_GROUP:-}
VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
# TODO: resurrect backing device...need to know how to set values # TODO: resurrect backing device...need to know how to set values
#VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-} #VOLUME_BACKING_DEVICE=${VOLUME_BACKING_DEVICE:-}
VOLUME_NAME_PREFIX=${VOLUME_NAME_PREFIX:-volume-}
# Entry Points # Entry Points
# ------------ # ------------
# Compatibility for getting a volume group name from either ``VOLUME_GROUP`` # cleanup_cinder_backend_lvm - Delete volume group and remove backing file
# or from ``VOLUME_GROUP_NAME`` plus the backend name # cleanup_cinder_backend_lvm $be_name
function get_volume_group_name {
local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$VOLUME_GROUP_NAME
if [[ -z $VOLUME_GROUP ]]; then
volume_group_name+="-$be_name"
fi
echo $volume_group_name
}
function cleanup_cinder_backend_lvm { function cleanup_cinder_backend_lvm {
local be_name=$1 local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
# Campsite rule: leave behind a volume group at least as clean as we found it # Campsite rule: leave behind a volume group at least as clean as we found it
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX clean_lvm_volume_group $VOLUME_GROUP_NAME-$be_name
_clean_lvm_backing_file ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file
} }
# configure_cinder_backend_lvm - Set config files, create data dirs, etc # configure_cinder_backend_lvm - Set config files, create data dirs, etc
# configure_cinder_backend_lvm $name # configure_cinder_backend_lvm $be_name
function configure_cinder_backend_lvm { function configure_cinder_backend_lvm {
local be_name=$1 local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
iniset $CINDER_CONF $be_name volume_backend_name $be_name iniset $CINDER_CONF $be_name volume_backend_name $be_name
iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver" iniset $CINDER_CONF $be_name volume_driver "cinder.volume.drivers.lvm.LVMISCSIDriver"
iniset $CINDER_CONF $be_name volume_group $volume_group_name iniset $CINDER_CONF $be_name volume_group $VOLUME_GROUP_NAME-$be_name
if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then if [[ "$CINDER_SECURE_DELETE" == "False" ]]; then
iniset $CINDER_CONF $be_name volume_clear none iniset $CINDER_CONF $be_name volume_clear none
fi fi
} }
# init_cinder_backend_lvm - Initialize volume group
# init_cinder_backend_lvm $be_name
function init_cinder_backend_lvm { function init_cinder_backend_lvm {
local be_name=$1 local be_name=$1
# Again with the icehouse-generation compatibility
local volume_group_name=$(get_volume_group_name $be_name)
# Start with a clean volume group # Start with a clean volume group
_create_cinder_volume_group ${volume_group_name} $DATA_DIR/${volume_group_name}-backing-file init_lvm_volume_group $VOLUME_GROUP_NAME-$be_name $VOLUME_BACKING_FILE_SIZE
if is_fedora || is_suse; then
# service is not started by default
start_service tgtd
fi
# Remove iscsi targets
sudo tgtadm --op show --mode target | grep $VOLUME_NAME_PREFIX | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
_clean_lvm_lv ${volume_group_name} $VOLUME_NAME_PREFIX
} }
# _clean_lvm_lv removes all cinder LVM volumes
#
# Usage: _clean_lvm_lv volume-group-name $VOLUME_NAME_PREFIX
function _clean_lvm_lv {
local vg=$1
local lv_prefix=$2
# Clean out existing volumes
local lv
for lv in $(sudo lvs --noheadings -o lv_name $vg 2>/dev/null); do
# lv_prefix prefixes the LVs we want
if [[ "${lv#$lv_prefix}" != "$lv" ]]; then
sudo lvremove -f $vg/$lv
fi
done
}
# _clean_lvm_backing_file() removes the backing file of the
# volume group used by cinder
#
# Usage: _clean_lvm_backing_file() volume-group-name backing-file-name
function _clean_lvm_backing_file {
local vg=$1
local backing_file=$2
# if there is no logical volume left, it's safe to attempt a cleanup
# of the backing file
if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
# if the backing physical device is a loop device, it was probably setup by devstack
local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/backing-file/ { print $1}')
if [[ -n "$vg_dev" ]] && [[ -e "$vg_dev" ]]; then
sudo losetup -d $vg_dev
rm -f $backing_file
fi
fi
}
# _create_cinder_volume_group volume-group-name backing-file-name
function _create_cinder_volume_group {
# According to the ``CINDER_MULTI_LVM_BACKEND`` value, configure one or two default volumes
# group called ``stack-volumes`` (and ``stack-volumes2``) for the volume
# service if it (they) does (do) not yet exist. If you don't wish to use a
# file backed volume group, create your own volume group called ``stack-volumes``
# and ``stack-volumes2`` before invoking ``stack.sh``.
#
# The two backing files are ``VOLUME_BACKING_FILE_SIZE`` in size, and they are stored in
# the ``DATA_DIR``.
local vg_name=$1
local backing_file=$2
if ! sudo vgs $vg_name; then
# TODO: fix device handling
if [ -z "$VOLUME_BACKING_DEVICE" ]; then
# Only create if the file doesn't already exists
[[ -f $backing_file ]] || truncate -s $VOLUME_BACKING_FILE_SIZE $backing_file
local vg_dev=`sudo losetup -f --show $backing_file`
# Only create if the loopback device doesn't contain $VOLUME_GROUP
if ! sudo vgs $vg_name; then
sudo vgcreate $vg_name $vg_dev
fi
else
sudo vgcreate $vg_name $VOLUME_BACKING_DEVICE
fi
fi
}
# Restore xtrace # Restore xtrace
$MY_XTRACE $MY_XTRACE

124
lib/lvm Normal file
View File

@ -0,0 +1,124 @@
# lib/lvm
# Configure the default LVM volume group used by Cinder and Nova
# Dependencies:
#
# - ``functions`` file
# - ``cinder`` configurations
# DATA_DIR
# clean_default_volume_group - called from clean()
# configure_default_volume_group - called from configure()
# init_default_volume_group - called from init()
# Save trace setting
MY_XTRACE=$(set +o | grep xtrace)
set +o xtrace
# Defaults
# --------
# Name of the lvm volume groups to use/create for iscsi volumes
# This monkey-motion is for compatibility with icehouse-generation Grenade
# If ``VOLUME_GROUP`` is set, use it, otherwise we'll build a VG name based
# on ``VOLUME_GROUP_NAME`` that includes the backend name
# Grenade doesn't use ``VOLUME_GROUP2`` so it is left out
VOLUME_GROUP_NAME=${VOLUME_GROUP:-${VOLUME_GROUP_NAME:-stack-volumes}}
DEFAULT_VOLUME_GROUP_NAME=$VOLUME_GROUP_NAME-default
# Backing file name is of the form $VOLUME_GROUP$BACKING_FILE_SUFFIX
BACKING_FILE_SUFFIX=-backing-file
# Entry Points
# ------------
# _clean_lvm_volume_group removes all default LVM volumes
#
# Usage: clean_lvm_volume_group $vg
function _clean_lvm_volume_group {
local vg=$1
# Clean out existing volumes
sudo lvremove -f $vg
}
# _clean_lvm_backing_file() removes the backing file of the
# volume group
#
# Usage: _clean_lvm_backing_file() $backing_file
function _clean_lvm_backing_file {
local backing_file=$1
# if the backing physical device is a loop device, it was probably setup by devstack
if [[ -n "$backing_file" ]] && [[ -e "$backing_file" ]]; then
local vg_dev=$(sudo losetup -j $backing_file | awk -F':' '/'$BACKING_FILE_SUFFIX'/ { print $1}')
sudo losetup -d $vg_dev
rm -f $backing_file
fi
}
# clean_lvm_volume_group() cleans up the volume group and removes the
# backing file
#
# Usage: clean_lvm_volume_group $vg
function clean_lvm_volume_group {
local vg=$1
_clean_lvm_volume_group $vg
# if there is no logical volume left, it's safe to attempt a cleanup
# of the backing file
if [[ -z "$(sudo lvs --noheadings -o lv_name $vg 2>/dev/null)" ]]; then
_clean_lvm_backing_file $DATA_DIR/$vg$BACKING_FILE_SUFFIX
fi
}
# _create_volume_group creates default volume group
#
# Usage: _create_lvm_volume_group() $vg $size
function _create_lvm_volume_group {
local vg=$1
local size=$2
local backing_file=$DATA_DIR/$vg$BACKING_FILE_SUFFIX
if ! sudo vgs $vg; then
# Only create if the file doesn't already exists
[[ -f $DATA_DIR/$backing_file ]] || truncate -s $size $backing_file
local vg_dev=`sudo losetup -f --show $backing_file`
# Only create volume group if it doesn't already exist
if ! sudo vgs $vg; then
sudo vgcreate $vg $vg_dev
fi
fi
}
# init_lvm_volume_group() initializes the volume group creating the backing
# file if necessary
#
# Usage: init_lvm_volume_group() $vg
function init_lvm_volume_group {
local vg=$1
local size=$2
# Start with a clean volume group
_create_lvm_volume_group $vg $size
if is_fedora || is_suse; then
# service is not started by default
start_service tgtd
fi
# Remove iscsi targets
sudo tgtadm --op show --mode target | grep Target | cut -f3 -d ' ' | sudo xargs -n1 tgt-admin --delete || true
_clean_lvm_volume_group $vg
}
# Restore xtrace
$MY_XTRACE
# mode: shell-script
# End:

View File

@ -547,6 +547,11 @@ function create_nova_conf {
iniset $NOVA_CONF DEFAULT ec2_workers "$API_WORKERS" iniset $NOVA_CONF DEFAULT ec2_workers "$API_WORKERS"
iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS" iniset $NOVA_CONF DEFAULT metadata_workers "$API_WORKERS"
if [[ "$NOVA_BACKEND" == "LVM" ]]; then
iniset $NOVA_CONF libvirt images_type "lvm"
iniset $NOVA_CONF libvirt images_volume_group $DEFAULT_VOLUME_GROUP_NAME
fi
if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then if is_ssl_enabled_service glance || is_service_enabled tls-proxy; then
iniset $NOVA_CONF DEFAULT glance_protocol https iniset $NOVA_CONF DEFAULT glance_protocol https
fi fi

View File

@ -523,6 +523,7 @@ source $TOP_DIR/lib/tls
source $TOP_DIR/lib/infra source $TOP_DIR/lib/infra
source $TOP_DIR/lib/oslo source $TOP_DIR/lib/oslo
source $TOP_DIR/lib/stackforge source $TOP_DIR/lib/stackforge
source $TOP_DIR/lib/lvm
source $TOP_DIR/lib/horizon source $TOP_DIR/lib/horizon
source $TOP_DIR/lib/keystone source $TOP_DIR/lib/keystone
source $TOP_DIR/lib/glance source $TOP_DIR/lib/glance
@ -939,6 +940,10 @@ init_service_check
# A better kind of sysstat, with the top process per time slice # A better kind of sysstat, with the top process per time slice
start_dstat start_dstat
# Initialize default LVM volume group
# -----------------------------------
init_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME $VOLUME_BACKING_FILE_SIZE
# Start Services # Start Services
# ============== # ==============

View File

@ -55,6 +55,7 @@ source $TOP_DIR/lib/tls
source $TOP_DIR/lib/infra source $TOP_DIR/lib/infra
source $TOP_DIR/lib/oslo source $TOP_DIR/lib/oslo
source $TOP_DIR/lib/stackforge source $TOP_DIR/lib/stackforge
source $TOP_DIR/lib/lvm
source $TOP_DIR/lib/horizon source $TOP_DIR/lib/horizon
source $TOP_DIR/lib/keystone source $TOP_DIR/lib/keystone
source $TOP_DIR/lib/glance source $TOP_DIR/lib/glance
@ -177,3 +178,5 @@ if [[ -n "$SCREEN" ]]; then
screen -X -S $SESSION quit screen -X -S $SESSION quit
fi fi
fi fi
clean_lvm_volume_group $DEFAULT_VOLUME_GROUP_NAME