Fix tons of bashate issues for bug 1534660

Fixing https://bugs.launchpad.net/osops/+bug/1534660. Some of these
fixes are not pretty and I've not been able to test the tools still
work.

I think the bashate rules should be relaxed for operations tools ... or
people shouldn't use bash for such tools. Sometimes it's pretty
difficult to shorten lines and still have readable code.

Co-Authored-By: Peter Jenkins <mail@peter-jenkins.com>
Co-Authored-By: Mike Dorman <mdorman@godaddy.com>
Change-Id: I70cfc2420cc9a2a4ec553ab7b7ca43a7fc38a9f0
This commit is contained in:
Peter Jenkins 2016-01-19 12:03:01 +02:00 committed by Mike Dorman
parent b9da55d0f8
commit 6c67ad007e
9 changed files with 112 additions and 54 deletions

View File

@ -16,7 +16,8 @@ final_report=$(mktemp)
echo -en "Retrieving list of all volumes...\r"
# oh cinder...
for volume in `cinder list --all-tenants | tail -n +4 | awk '{print $2}'`; do
for line in `cinder show $volume | grep 'os-vol-tenant-attr:tenant_id\| id ' | awk '{print $4}'`; do
for line in `cinder show $volume | \
grep 'os-vol-tenant-attr:tenant_id\| id ' | awk '{print $4}'`; do
echo -en " $line" >> $volume_ids
done
echo "" >> $volume_ids
@ -25,17 +26,24 @@ awk '{print $2}' < $volume_ids | sort -u > $cinder_reported_tenants
# get a list of all tenants, as reported by keystone
echo -en "Retrieving list of all tenants...\r"
keystone tenant-list | tail -n +4 | awk '{print $2}' | sort -u > $keystone_tenants
keystone tenant-list | tail -n +4 | awk '{print $2}' | \
sort -u > $keystone_tenants
# some rough/poor formatting
echo "Comparing outputs to locate orphaned volumes...\r"
echo "+--------------------------------------+-----------------------------------+----------------------------+--------------+------+--------+"
echo "| volume_id | tenant_id | created_at | display_name | size | status |"
echo "+--------------------------------------+-----------------------------------+----------------------------+--------------+------+--------+"
for tenant_id in `comm --nocheck-order -13 $keystone_tenants $cinder_reported_tenants`; do
echo "+--------------------------------------+--------------------------------\
---+----------------------------+--------------+------+--------+"
echo "| volume_id | tenant_id \
| created_at | display_name | size | status |"
echo "+--------------------------------------+--------------------------------\
---+----------------------------+--------------+------+--------+"
for tenant_id in `comm --nocheck-order -13 \
$keystone_tenants $cinder_reported_tenants`; do
for volume_id in `grep $tenant_id $volume_ids | awk '{print $1}'`; do
echo -en "| $volume_id | $tenant_id |"
for attr in `cinder show $volume_id | grep ' status \| size \| display_name \| created_at ' | awk '{print $4}'`; do
for attr in `cinder show $volume_id |\
grep ' status \| size \| display_name \| created_at ' |\
awk '{print $4}'`; do
echo -en " $attr |"
done
echo ""

View File

@ -1,7 +1,8 @@
#!/bin/bash
#
# This script will look at the configured vm's and will check to make sure that their disk drive still exists,
# If not then it will remove the vm from libvirt. This fixes the nova errors about disks missing from VM's
# This script will look at the configured vm's and will check to make sure that
# their disk drive still exists, If not then it will remove the vm from
# libvirt. This fixes the nova errors about disks missing from VM's
#
# Author: Kris Lindgren <klindgren@godaddy.com>
@ -23,7 +24,8 @@ removeorphan(){
}
for i in /etc/libvirt/qemu/*.xml; do
disklocation=$( grep /var/lib/nova/instances $i | grep disk | cut -d"'" -f2,2)
disklocation=$( grep /var/lib/nova/instances $i | grep disk | \
cut -d"'" -f2,2)
if [ ! -e $disklocation ]; then
orphan=$(echo $i | cut -d"/" -f5,5 | cut -d"." -f1,1)
echo "$orphan does not have a disk located at: $disklocation"

View File

@ -3,7 +3,8 @@ source /root/scripts/stackrc
echo "$(date): Tenant quota correction - started"
for x in $(keystone --insecure tenant-list | awk -F' | ' '!/^\+/ && !/\ id\ / {print $2}'); do
for x in $(keystone --insecure tenant-list | awk -F' |\
' '!/^\+/ && !/\ id\ / {print $2}'); do
echo "Correcting quota for tenant $x"
python /root/scripts/auto-fix-quota.py --tenant $x
done

3
nova/backfillconfigdrive.sh Normal file → Executable file
View File

@ -13,7 +13,8 @@ mkdir -p blank
mkisofs -o blank.iso blank/ >/dev/null 2>&1
rmdir blank
for i in `ls /var/lib/nova/instances | grep -E '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}'`; do
for i in `ls /var/lib/nova/instances | \
grep -E '[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}'`; do
ls -l /var/lib/nova/instances/$i/disk.config
if [ ! -s /var/lib/nova/instances/$i/disk.config ]; then
echo "$i config drive doesn't exist, or is size zero."

3
nova/list-vms-by-host.sh Normal file → Executable file
View File

@ -5,7 +5,8 @@
#
# Author: Mike Dorman <mdorman@godaddy.com>
for i in `nova list --all-tenants | grep -v '^+-' | grep -v '^| ID' | awk '{print $2 "," $4 "," $6;}'`; do
for i in `nova list --all-tenants | grep -v '^+-' | grep -v '^| ID' |\
awk '{print $2 "," $4 "," $6;}'`; do
ID=`echo $i | cut -d, -f 1`
NAME=`echo $i | cut -d, -f 2`
STATUS=`echo $i | cut -d, -f 3`

View File

@ -14,10 +14,13 @@ echo "done."
echo "Getting a list of tenants from keystone...."
keystoneraw=$( keystone tenant-list )
echo "done."
novatenants=$( echo "$novavmsraw" | awk '{print $6}' | sort | uniq | grep -v Tenant )
novatenants=$( echo "$novavmsraw" | awk '{print $6}' | sort | uniq |\
grep -v Tenant )
echo "Starting to list vm's that are no longer attached to a tenant..."
echo "Fields are:"
echo "| VM ID | VM Name | Tenant Id | User Id |"
echo "| VM ID | \
VM Name | Tenant Id | \
User Id |"
for i in $novatenants; do
tmp=$( echo "$keystoneraw" | grep $i )
if [ $? -eq 0 ]; then

View File

@ -16,8 +16,8 @@
# License for the specific language governing permissions and limitations
# under the License.
#
# purge tables of "deleted" records by archiveing them in sensible chunks to the shadow tables
# this work was started in PAASINFRA-206
# purge tables of "deleted" records by archiveing them in sensible chunks to
# the shadow tables this work was started in PAASINFRA-206
#
# default to archiving all records flagged as deleted,
@ -26,15 +26,19 @@ unset DRY_RUN
# tables to arhive deleted records from
DATABASE=nova
TABLES="security_group_rules security_group_instance_association security_groups instance_info_caches instances reservations compute_node_stats"
FKTABLES="block_device_mapping instance_metadata instance_system_metadata instance_actions instance_faults virtual_interfaces fixed_ips security_group_instance_association migrations instance_extra"
TABLES="security_group_rules security_group_instance_association \
security_groups instance_info_caches instances reservations compute_node_stats"
FKTABLES="block_device_mapping instance_metadata instance_system_metadata \
instance_actions instance_faults virtual_interfaces fixed_ips \
security_group_instance_association migrations instance_extra"
TABLES="${TABLES} ${FKTABLES}"
## process the command line arguments
while getopts "hnad:H:u:p:" opt; do
case $opt in
h)
echo "openstack_db_archive.sh - archive records flagged as deleted into the shadow tables."
echo "openstack_db_archive.sh - archive records flagged as deleted\
into the shadow tables."
echo "Records are archived from the following tables:"
echo
for TABLE in ${TABLES}; do
@ -43,7 +47,8 @@ while getopts "hnad:H:u:p:" opt; do
echo
echo "Options:"
echo " -n dry run mode - pass --dry-run to pt-archiver"
echo " -a no safe auto increment - pass --nosafe-auto-increment to pt-archiver"
echo " -a no safe auto increment - pass --nosafe-auto-increment" \
"to pt-archiver"
echo " -d db name"
echo " -H db hostname"
echo " -u db username"
@ -55,7 +60,7 @@ while getopts "hnad:H:u:p:" opt; do
DRY_RUN="--dry-run"
;;
a)
NOSAI="--nosafe-auto-increment"
NOSAI="--nosafe-auto-increment"
;;
d)
DATABASE=${OPTARG}
@ -88,54 +93,77 @@ echo `date` "OpenStack Database Archiver starting.."
echo
echo `date` "Purging nova.instance_actions_events of deleted instance data"
# this is back to front (on delete if you can find a record in instances flagged for deletion)
# --where 'EXISTS(SELECT * FROM instance_actions, instances WHERE instance_actions.id=instance_actions_events.action_id AND instance_actions.instance_uuid=instances.uuid AND instances.deleted!=0)'
# this is back to front (on delete if you can find a record in instances
# flagged for deletion)
# --where 'EXISTS(SELECT * FROM instance_actions, instances WHERE
# instance_actions.id=instance_actions_events.action_id AND
# instance_actions.instance_uuid=instances.uuid AND instances.deleted!=0)'
TABLE=instance_actions_events
SHADOW_TABLE="shadow_${TABLE}"
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --sleep-coef 0.75 --progress 100 --commit-each --limit 10 \
--source D=${DATABASE},t=${TABLE}${HOSTPT}${USERPT}${PASSPT} --no-check-charset \
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --sleep-coef 0.75 --progress 100 \
--commit-each --limit 10 \
--source D=${DATABASE},t=${TABLE}${HOSTPT}${USERPT}${PASSPT} \
--no-check-charset \
--dest D=${DATABASE},t=${SHADOW_TABLE}${HOSTPT}${USERPT}${PASSPT} \
--where 'EXISTS(SELECT * FROM instance_actions, instances WHERE instance_actions.id=instance_actions_events.action_id AND instance_actions.instance_uuid=instances.uuid AND instances.deleted!=0)'
--where 'EXISTS(SELECT * FROM instance_actions, instances WHERE \
instance_actions.id=instance_actions_events.action_id AND \
instance_actions.instance_uuid=instances.uuid AND instances.deleted!=0)'
for TABLE in ${FKTABLES}; do
echo `date` "Purging nova.${TABLE} of deleted instance data"
# this is back to front (on delete if you can find a record in instances flagged for deletion)
# --where 'EXISTS(SELECT * FROM instances WHERE deleted!=0 AND uuid='${TABLE}'.instance_uuid)'
# to delete where there is no active record:
# --where 'NOT EXISTS(SELECT * FROM instances WHERE deleted=0 AND uuid='${TABLE}'.instance_uuid)'
echo `date` "Purging nova.${TABLE} of deleted instance data"
# this is back to front (on delete if you can find a record in instances
# flagged for deletion)
# --where 'EXISTS(SELECT * FROM instances WHERE deleted!=0 AND \
# uuid='${TABLE}'.instance_uuid)'
# to delete where there is no active record:
# --where 'NOT EXISTS(SELECT * FROM instances WHERE deleted=0 AND \
# uuid='${TABLE}'.instance_uuid)'
SHADOW_TABLE="shadow_${TABLE}"
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --sleep-coef 0.75 --progress 100 --commit-each --limit 10 \
--source D=${DATABASE},t=${TABLE}${HOSTPT}${USERPT}${PASSPT} --no-check-charset \
--dest D=${DATABASE},t=${SHADOW_TABLE}${HOSTPT}${USERPT}${PASSPT} \
--where 'EXISTS(SELECT * FROM instances WHERE deleted!=0 AND uuid='${TABLE}'.instance_uuid)'
SHADOW_TABLE="shadow_${TABLE}"
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --sleep-coef 0.75 \
--progress 100 --commit-each --limit 10 \
--source D=${DATABASE},t=${TABLE}${HOSTPT}${USERPT}${PASSPT} \
--no-check-charset \
--dest D=${DATABASE},t=${SHADOW_TABLE}${HOSTPT}${USERPT}${PASSPT} \
--where 'EXISTS(SELECT * FROM instances WHERE deleted!=0 AND \
uuid='${TABLE}'.instance_uuid)'
done
for TABLE in ${TABLES}; do
SHADOW_TABLE="shadow_${TABLE}"
ACTIVE_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e "select count(id) from ${DATABASE}.${TABLE} where deleted=0" | tail -1`
DELETED_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e "select count(id) from ${DATABASE}.${TABLE} where deleted!=0" | tail -1`
ACTIVE_RECORDS=`mysql ${HOST} ${USER} ${PASS} \
-B -e "select count(id) from ${DATABASE}.${TABLE} where deleted=0" \
| tail -1`
DELETED_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"select count(id) from ${DATABASE}.${TABLE} where deleted!=0" | tail -1`
LOCAL_ABORTS=`mysql ${HOST} ${USER} ${PASS} -B -e "SHOW STATUS LIKE 'wsrep_%'" | grep -e wsrep_local_bf_aborts -e wsrep_local_cert_failures`
LOCAL_ABORTS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"SHOW STATUS LIKE 'wsrep_%'" | \
grep -e wsrep_local_bf_aborts -e wsrep_local_cert_failures`
echo
echo
echo `date` "Archiving ${DELETED_RECORDS} records to ${SHADOW_TABLE} from ${TABLE}, leaving ${ACTIVE_RECORDS}"
echo `date` "Archiving ${DELETED_RECORDS} records to ${SHADOW_TABLE} from \
${TABLE}, leaving ${ACTIVE_RECORDS}"
echo `date` "LOCAL_ABORTS before"
echo ${LOCAL_ABORTS}
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --progress 100 --commit-each --limit 10 \
pt-archiver ${DRY_RUN} ${NOSAI} --statistics --progress 100 \
--commit-each --limit 10 \
--source D=${DATABASE},t=${TABLE}${HOSTPT}${USERPT}${PASSPT} \
--dest D=${DATABASE},t=${SHADOW_TABLE}${HOSTPT}${USERPT}${PASSPT} \
--ignore --no-check-charset --sleep-coef 0.75 \
--where "deleted!=0"
echo `date` "Finished archiving ${DELETED_RECORDS} to ${SHADOW_TABLE} from ${TABLE}"
echo `date` "Finished archiving ${DELETED_RECORDS} to ${SHADOW_TABLE} from\
${TABLE}"
echo `date` "LOCAL_ABORTS before"
echo ${LOCAL_ABORTS}
LOCAL_ABORTS=`mysql ${HOST} ${USER} ${PASS} -B -e "SHOW STATUS LIKE 'wsrep_%'" | grep -e wsrep_local_bf_aborts -e wsrep_local_cert_failures`
LOCAL_ABORTS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"SHOW STATUS LIKE 'wsrep_%'" | \
grep -e wsrep_local_bf_aborts -e wsrep_local_cert_failures`
echo `date` "LOCAL_ABORTS after"
echo ${LOCAL_ABORTS}
echo

View File

@ -19,11 +19,14 @@
# Report on the current state of unarchived records in the main nova.* tables
DATABASE=nova
FKTABLES="block_device_mapping instance_metadata instance_system_metadata instance_actions instance_faults virtual_interfaces fixed_ips security_group_instance_association migrations instance_extra"
FKTABLES="block_device_mapping instance_metadata instance_system_metadata \
instance_actions instance_faults virtual_interfaces fixed_ips \
security_group_instance_association migrations instance_extra"
TABLES="${TABLES} ${FKTABLES}"
function usage {
echo "$0: Report on the current state of unarchived records in the main nova.* tables"
echo "$0: Report on the current state of unarchived records in the\
main nova.* tables"
echo "Usage: $0 -d [database] -H [hostname] -u [username] -p [password]"
}
@ -51,10 +54,16 @@ done
for TABLE in ${TABLES}; do
SHADOW_TABLE="shadow_${TABLE}"
ACTIVE_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e "select count(id) from ${DATABASE}.${TABLE} where deleted=0" | tail -1`
DELETED_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e "select count(id) from ${DATABASE}.${TABLE} where deleted!=0" | tail -1`
SHADOW_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e "select count(id) from ${DATABASE}.${SHADOW_TABLE}" | tail -1`
ACTIVE_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"select count(id) from ${DATABASE}.${TABLE} where deleted=0" | tail -1`
DELETED_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"select count(id) from ${DATABASE}.${TABLE} where deleted!=0" | \
tail -1`
SHADOW_RECORDS=`mysql ${HOST} ${USER} ${PASS} -B -e \
"select count(id) from ${DATABASE}.${SHADOW_TABLE}" | tail -1`
TOTAL_RECORDS=`expr $ACTIVE_RECORDS + $DELETED_RECORDS + $SHADOW_RECORDS`
echo `date` "${DATABASE}.${TABLE} has ${ACTIVE_RECORDS}, ${DELETED_RECORDS} ready for archiving and ${SHADOW_RECORDS} already in ${SHADOW_TABLE}. Total records is ${TOTAL_RECORDS}"
echo `date` "${DATABASE}.${TABLE} has ${ACTIVE_RECORDS}," \
"${DELETED_RECORDS} ready for archiving and ${SHADOW_RECORDS}" \
"already in ${SHADOW_TABLE}. Total records is ${TOTAL_RECORDS}"
done

View File

@ -12,7 +12,8 @@ keystone_tenants=$(mktemp)
# get a list of all VMs in the cluster and who they belong to
echo -en "Retrieving list of all VMs...\r"
nova list --all-tenants --fields tenant_id | tail -n +4 | awk '{print $4}' | sort -u > $vm_tenants
nova list --all-tenants --fields tenant_id | tail -n +4 | awk '{print $4}' |\
sort -u > $vm_tenants
total_vms=$(cat $vm_tenants | wc -l)
if [ $total_vms == 0 ]; then
echo "Zero VMs found. Exiting..."
@ -22,7 +23,8 @@ fi
# get a list of all tenants/projects in the cluster
echo -en "Retrieving list of all tenants...\r"
keystone tenant-list | tail -n +4 | awk '{print $2}' | sort -u > $keystone_tenants
keystone tenant-list | tail -n +4 | awk '{print $2}' |\
sort -u > $keystone_tenants
total_tenants=$(cat $keystone_tenants | wc -l)
if [ $total_tenants == 0 ]; then
echo "Zero tenants found. Exiting..."
@ -36,10 +38,13 @@ echo -en "Comparing outputs to locate orphaned VMs....\r"
iter=0
for tenant_id in `comm --nocheck-order -13 $keystone_tenants $vm_tenants`; do
if [[ $iter == 0 ]]; then
nova list --all-tenants --tenant=$tenant_id --fields tenant_id,name,status,created,updated | head -n -1
nova list --all-tenants --tenant=$tenant_id \
--fields tenant_id,name,status,created,updated | head -n -1
let "iter++"
else
nova list --all-tenants --tenant=$tenant_id --fields tenant_id,name,status,created,updated | tail -n +4 | head -n -1
nova list --all-tenants --tenant=$tenant_id \
--fields tenant_id,name,status,created,updated | \
tail -n +4 | head -n -1
fi
done