From f0eb0b75822194f2f9c25d45cbedc1cab55d62a6 Mon Sep 17 00:00:00 2001
From: "Crank, Daniel" <dan.no@att.com>
Date: Wed, 22 Jan 2020 15:57:09 -0600
Subject: [PATCH] [ad-hoc] Fix test case exit conditions

While working on another change, I discovered conditions
in many test cases that echoed fail messages but did not
actually exit, so the gate could succeed even though some
tests failed. This patchset aims to fix those problems, and
then fix the problems masked by those problems:

1) fix bug in revert function of file permissions module
preventing permissions from being reverted.
2) fix various syntax and logic problems in test script
3) add wait_for_tiller_ready function to avoid race condition
with test script using helm too early
4) add install for ethtool in test script
5) ignore ethtool pod failures (see note #1 in [0])
6) make logging of test results more uniform
7) Fix error message logic in perm.sh
8) Fix case in _shcommon.tpl where error message was not
logged, causing test script to unnecessarily wait for
container timeout

[0]: https://review.opendev.org/676010

Change-Id: I22182d35250c37c96e73d9f5f49abfb2246f2a35
---
 divingbell/templates/bin/_apt.sh.tpl      |   8 +-
 divingbell/templates/bin/_perm.sh.tpl     |  12 +-
 divingbell/templates/bin/_shcommon.tpl    |   3 +-
 doc/source/index.rst                      |   4 +-
 tools/gate/scripts/020-test-divingbell.sh | 596 ++++++++++++++--------
 5 files changed, 386 insertions(+), 237 deletions(-)

diff --git a/divingbell/templates/bin/_apt.sh.tpl b/divingbell/templates/bin/_apt.sh.tpl
index b6cb5b9..2d787f6 100644
--- a/divingbell/templates/bin/_apt.sh.tpl
+++ b/divingbell/templates/bin/_apt.sh.tpl
@@ -59,7 +59,7 @@ load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
 
 {{- if hasKey .Values.conf "apt" }}
 {{- if hasKey .Values.conf.apt "repositories" }}
-echo -n "" > /etc/apt/trusted.gpg.d/divindbell_temp.gpg
+echo -n "" > /etc/apt/trusted.gpg.d/divingbell_temp.gpg
 echo "#The list of repositories managed by Divingbell" > /etc/apt/sources.list.divingbell
   {{- range .Values.conf.apt.repositories }}
     {{- $url := .url }}
@@ -75,16 +75,16 @@ echo "{{ printf "deb %s %s-%s %s" $url $distribution . $components }}" >>/etc/ap
       {{- end }}
     {{- end }}
     {{- if hasKey . "gpgkey" }}
-apt-key --keyring /etc/apt/trusted.gpg.d/divindbell_temp.gpg add - <<"ENDKEY"
+apt-key --keyring /etc/apt/trusted.gpg.d/divingbell_temp.gpg add - <<"ENDKEY"
 {{ .gpgkey }}
 ENDKEY
     {{- end }}
   {{- end }}
 mv /etc/apt/sources.list.divingbell /etc/apt/sources.list
 rm -rf /etc/apt/sources.list.d/*
-mv /etc/apt/trusted.gpg.d/divindbell_temp.gpg /etc/apt/trusted.gpg.d/divindbell.gpg
+mv /etc/apt/trusted.gpg.d/divingbell_temp.gpg /etc/apt/trusted.gpg.d/divingbell.gpg
 rm -f /etc/apt/trusted.gpg
-find /etc/apt/trusted.gpg.d/ -type f ! -name 'divindbell.gpg' -exec rm {{ "{}" }} \;
+find /etc/apt/trusted.gpg.d/ -type f ! -name 'divingbell.gpg' -exec rm {{ "{}" }} \;
 DEBIAN_FRONTEND=noninteractive apt-get update
 {{- end }}
 {{- if hasKey .Values.conf.apt "packages" }}
diff --git a/divingbell/templates/bin/_perm.sh.tpl b/divingbell/templates/bin/_perm.sh.tpl
index ba6ebb8..d07baf4 100644
--- a/divingbell/templates/bin/_perm.sh.tpl
+++ b/divingbell/templates/bin/_perm.sh.tpl
@@ -97,11 +97,11 @@ revert_perm(){
     for o_perm in ${revert_list}; do
       first=1
       while IFS=' ' read -r a1 a2; do
-        if [ "$first" -eq 1 && -e "$a2"]; then
-          $(chmod "$a1" "$a2")
+        if [ "$first" -eq 1 ] && [ -e "$a2" ]; then
+          chmod "$a1" "$a2"
           first=0
-        elif [ -e "$a2"]; then
-          $(chown "$a1" "$a2")
+        elif [ -e "$a2" ]; then
+          chown "$a1" "$a2"
         else
           log.WARN "Unable to revert permissions on $a2"
           continue
@@ -180,7 +180,7 @@ if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
   fi
  # write timestamp at beginning of execution
  log.INFO 'All permissions successfully applied on this node.'
- echo $(date +"%s") > "${hash}/last_run_timestamp"
+ date +"%s" > "${hash}/last_run_timestamp"
 
  {{- range $perm := .Values.conf.perm.paths }}
  add_perm {{ $perm.path | squote }} {{ $perm.owner | squote }} {{ $perm.group | squote }} {{ $perm.permissions | squote }}
@@ -189,7 +189,7 @@ if [[ $hash_check = pass ]] && [[ $interval_check = pass ]]; then
 
  revert_perm
 
- if [ -n "${curr_settings}" ]; then
+ if [ -n "${applied_perm}" ]; then
   log.INFO 'All permissions successfully applied on this node.'
  else
   log.WARN 'No permissions overrides defined for this node.'
diff --git a/divingbell/templates/bin/_shcommon.tpl b/divingbell/templates/bin/_shcommon.tpl
index 5d90070..72e5fe8 100644
--- a/divingbell/templates/bin/_shcommon.tpl
+++ b/divingbell/templates/bin/_shcommon.tpl
@@ -75,7 +75,6 @@ exec >& >(while read line; do
           done)
 
 die(){
-  set +x
   # write to stderr any passed error message
   if [[ $@ = *[!\ ]* ]]; then
     log.ERROR "$@"
@@ -85,6 +84,8 @@ die(){
     log.TRACE $(caller $i)
   done
   echo __EXIT_MARKER__
+  log.ERROR "Exiting with unspecified error."
+  set +x
   # Exit after pipe closes to ensure all output is flushed first
   while : ; do
     echo "Waiting on exit..." || exit 1
diff --git a/doc/source/index.rst b/doc/source/index.rst
index 5f0be70..5be7eff 100644
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -207,7 +207,7 @@ Used to execute scripts on nodes in ``systemd`` (PID=1) namespace, for ex.::
       002-script2.sh:
         data: |
           #!/bin/bash
-          echo ${BASH_SOURCE}
+          echo ${BASH_SOURCE[0]}
       001-script1.sh:
         blocking_policy: foreground_halt_pod_on_failure
         env:
@@ -218,7 +218,7 @@ Used to execute scripts on nodes in ``systemd`` (PID=1) namespace, for ex.::
         - arg2
         data: |
           #!/bin/bash
-          echo script name: ${BASH_SOURCE}
+          echo script name: ${BASH_SOURCE[0]}
           echo args: $@
           echo env: $env1 $env2 $env3
 
diff --git a/tools/gate/scripts/020-test-divingbell.sh b/tools/gate/scripts/020-test-divingbell.sh
index 0cba92c..990edaa 100755
--- a/tools/gate/scripts/020-test-divingbell.sh
+++ b/tools/gate/scripts/020-test-divingbell.sh
@@ -12,7 +12,7 @@ traceback(){
 trap 'traceback' ERR
 
 check_if_running(){
-  script_name="$(basename ${BASH_SOURCE})"
+  script_name="$(basename ${BASH_SOURCE[0]})"
   script_running=false
   for pid in $(pidof -x $script_name); do
     if [ $pid != $$ ]; then
@@ -22,10 +22,12 @@ check_if_running(){
 }
 check_if_running
 if [[ $script_running = 'true' ]]; then
-  echo "$(basename ${BASH_SOURCE}) already running on this node. Please run recheck for your PS."
+  echo "[FAIL] $(basename ${BASH_SOURCE[0]}) already running on this node. Please run recheck for your PS." >> "${TEST_RESULTS}"
   exit 2
 fi
 
+DEFAULT_IFS=$IFS
+
 NAME=divingbell
 : ${LOGS_DIR:=/tmp/artifacts}
 : ${LOGS_SUBDIR:=${LOGS_DIR}/${NAME}/$(date +"%m-%d-%y_%H:%M:%S")}
@@ -95,7 +97,7 @@ APT_REPOSITORY1="http://us.archive.ubuntu.com/ubuntu/"
 APT_DISTRIBUTIONS1="[ xenial ]"
 APT_COMPONENTS1="[ main, universe, restricted, multiverse ]"
 APT_SUBREPOS1="[ backports, updates ]"
-APT_GPGKEYID1="437D05B5"
+APT_GPGKEYID1="437D 05B5"
 APT_GPGKEY1="-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
 
@@ -177,7 +179,7 @@ APT_REPOSITORY2="http://security.ubuntu.com/ubuntu/"
 APT_DISTRIBUTIONS2="[ xenial ]"
 APT_COMPONENTS2="[ main, universe, restricted, multiverse ]"
 APT_SUBREPOS2="[ security ]"
-APT_GPGKEYID2="C0B21F32"
+APT_GPGKEYID2="C0B2 1F32"
 APT_GPGKEY2="-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
 
@@ -245,7 +247,7 @@ DjfegedFClqpn37b
 APT_REPOSITORY3="https://download.ceph.com/debian-mimic/"
 APT_DISTRIBUTIONS3="[ xenial ]"
 APT_COMPONENTS3="[ main ]"
-APT_GPGKEYID3="460F3994"
+APT_GPGKEYID3="460F 3994"
 APT_GPGKEY3="-----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
 
@@ -295,7 +297,11 @@ for line in ${nic_info}; do
     break
   fi
 done
-[ -n "${DEVICE}" ] || (echo Could not find physical NIC for tesing; exit 1)
+IFS=$DEFAULT_IFS
+if [ -z "${DEVICE}" ]; then
+    echo "[FAIL] Could not find physical NIC for testing." >> "${TEST_RESULTS}"
+    exit 1
+fi
 # Not all hardware has the same NIC tunables to use for testing
 if [[ $(/sbin/ethtool -k "${DEVICE}" | grep "${ETHTOOL_KEY3}:") =~ .*fixed.* ]]; then
   ETHTOOL_KEY3="${ETHTOOL_KEY3_BACKUP}"
@@ -308,8 +314,24 @@ exec >& >(while read line; do echo "${line}" | sudo tee -a ${LOG_NAME}; done)
 
 set -x
 
+wait_for_tiller_ready(){
+  local helm_error
+  local retries=5
+  while [ $retries -gt 0 ]; do
+    # the message typically returned before tiller is ready is
+    # 'Error: could not find a ready tiller pod'
+    helm_error="$(helm list ${NAME} 2>&1 | grep 'Error')"
+    if [ -z "${helm_error}" ]; then return 0; fi
+    sleep 10
+    (( retries-- ))
+  done
+  echo "[FAIL] Tiller pod not ready or not available." >> "${TEST_RESULTS}"
+  exit 1
+}
+
 purge_containers(){
-  local chart_status="$(helm list ${NAME})"
+  local chart_status
+  chart_status="$(helm list ${NAME})"
   if [ -n "${chart_status}" ]; then
     helm delete --purge ${NAME}
   fi
@@ -366,8 +388,7 @@ _reset_account(){
 }
 
 init_default_state(){
-  # TODO (dc6350) this needs retry logic to avoid race condition where tiller is not ready yet
-  sleep 30 # temporary fix for race condition
+  wait_for_tiller_ready
   purge_containers
   clean_persistent_files
   # set sysctl original vals
@@ -400,11 +421,16 @@ dry_run(){
   helm install --name="${NAME}" --dry-run --debug "${NAME}" --namespace="${NAME}" "$@"
 }
 
+# parameter 1 to get_container_status is the module name (e.g., "apt")
+# parameter 2 is optional and can be "expect_failure" to return success
+# on container failure and failure on container success, or "ignore_failure"
+# to return success regardless of container status.
 get_container_status(){
   local deployment="${1}"
   local log_connect_timeout=60
   local log_connect_sleep_interval=2
   local wait_time=0
+  local status
   while : ; do
     container="$(kubectl get pods --namespace="${NAME}" | grep ${NAME}-${deployment} | grep -v Terminating | cut -d' ' -f1)"
     kubectl logs "${container}" --namespace="${NAME}" > /dev/null && break || \
@@ -412,8 +438,12 @@ get_container_status(){
       wait_time=$((${wait_time} + ${log_connect_sleep_interval})) && \
       sleep ${log_connect_sleep_interval}
     if [ ${wait_time} -ge ${log_connect_timeout} ]; then
-      echo "Hit timeout while waiting for container logs to become available."
-      exit 1
+      if [ "${2}" = 'ignore_failure' ]; then
+        echo "Hit timeout while waiting for container logs to become available (ignored)."
+      else
+        echo "[FAIL] Hit timeout while waiting for container logs to become available." >> "${TEST_RESULTS}"
+        exit 1
+      fi
     fi
   done
   local container_runtime_timeout=210
@@ -421,24 +451,32 @@ get_container_status(){
   wait_time=0
   while : ; do
     CLOGS="$(kubectl logs --namespace="${NAME}" "${container}" 2>&1)" || true
-    local status="$(echo "${CLOGS}" | tail -1)"
+    # the test below now looks at the last ten lines of the log rather than
+    # just the last line, since there are cases where other things have to get
+    # logged after the error / success message.
+    # also: trying printf here to avoid arcane SIGTERM/SIGPIPE problems with
+    # builtin echo.
+    status="$(printf '%s' "${CLOGS}" | tail -n 10)"
     if [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *ERROR* ]] ||
        [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *TRACE* ]]; then
       if [ "${2}" = 'expect_failure' ]; then
         echo 'Pod exited as expected'
         break
+      elif [ "${2}" = 'ignore_failure' ]; then
+        echo 'Pod exited with error status (ignored).'
+        break
       else
-        echo 'Expected pod to complete successfully, but pod reported errors'
-        echo 'pod logs:'
-        echo "${CLOGS}"
+        echo '[FAIL] Expected pod to complete successfully, but pod reported errors' >> "${TEST_RESULTS}"
+        echo 'pod logs:' >> "${TEST_RESULTS}"
+        printf '%s' "${CLOGS}" >> "${TEST_RESULTS}"
         exit 1
       fi
     elif [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *'INFO Putting the daemon to sleep'* ]] ||
     [[ $(echo -e ${status} | tr -d '[:cntrl:]') = *'DEBUG + exit 0'* ]]; then
       if [ "${2}" = 'expect_failure' ]; then
-        echo 'Expected pod to die with error, but pod completed successfully'
-        echo 'pod logs:'
-        echo "${CLOGS}"
+        echo '[FAIL] Expected pod to die with error, but pod completed successfully' >> "${TEST_RESULTS}"
+        echo 'pod logs:' >> "${TEST_RESULTS}"
+        printf '%s' "${CLOGS}" >> "${TEST_RESULTS}"
         exit 1
       else
         echo 'Pod completed without errors.'
@@ -456,13 +494,19 @@ get_container_status(){
 }
 
 _test_sysctl_default(){
-  test "$(/sbin/sysctl "${1}" | cut -d'=' -f2 | tr -d '[:space:]')" = "${2}"
+  if [ "$(/sbin/sysctl "${1}" | cut -d'=' -f2 | tr -d '[:space:]')" != "${2}" ]; then
+    echo "[FAIL] Expected kernel parameter ${1} to be set to ${2}, but it was not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 _test_sysctl_value(){
   _test_sysctl_default "${1}" "${2}"
   local key="${1//\//.}"
-  test "$(cat /etc/sysctl.d/60-${NAME}-${key}.conf)" = "${key}=${2}"
+  if [ "$(cat /etc/sysctl.d/60-${NAME}-${key}.conf)" != "${key}=${2}" ]; then
+    echo "[FAIL] Expected kernel parameter ${key}=${2} to be persisted in /etc/sysctl.d, but it was not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 _test_exec_match(){
@@ -470,10 +514,10 @@ _test_exec_match(){
   exec_testfile="$2"
   testID="$3"
   if [[ $expected_result != $(cat $exec_testfile) ]]; then
-    echo "[FAIL] exec $testID failed. Expected:"
-    echo $expected_result
-    echo but got:
-    echo $(cat $exec_testfile)
+    echo "[FAIL] exec $testID failed. Expected:" >> "${TEST_RESULTS}"
+    echo $expected_result >> "${TEST_RESULTS}"
+    echo "but got:" >> "${TEST_RESULTS}"
+    echo $(cat $exec_testfile) >> "${TEST_RESULTS}"
     exit 1
   fi
   rm $exec_testfile
@@ -483,19 +527,20 @@ _test_exec_count(){
   script_location="${1}"
   script_name="${2}"
   script_expected_run_count="${3}"
-  script_run_count=$(cat "${script_location}" | wc -l)
+  script_run_count=$(wc -l "${script_location}")
   if [[ ${script_run_count} -ne ${script_expected_run_count} ]]; then
-    echo "[FAIL] Expected '${script_name}' to run '${script_expected_run_count}' times, but instead it ran '$script_run_count' times"
+    echo "[FAIL] Expected '${script_name}' to run '${script_expected_run_count}' times, but instead it ran '$script_run_count' times" >> "${TEST_RESULTS}"
     exit 1
   fi
 }
 
 _test_clog_msg(){
-  [[ $CLOGS = *${1}* ]] ||
-    (echo "Did not find expected string: '${1}'"
-     echo "in container logs:"
-     echo "${CLOGS}"
-     exit 1)
+  if [[ $CLOGS != *${1}* ]]; then
+    echo "[FAIL] Did not find expected string: '${1}'" >> "${TEST_RESULTS}"
+    echo "in container logs:" >> "${TEST_RESULTS}"
+    printf '%s' "${CLOGS}" >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 alias install_base="install ${BASE_VALS}"
@@ -510,7 +555,7 @@ shopt -s expand_aliases
 
 test_sysctl(){
   # Test the first set of values
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   local val1=0
   local val2=1
   local val3=0
@@ -530,7 +575,7 @@ test_sysctl(){
   echo '[SUCCESS] sysctl test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test an updated set of values
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   val1=1
   val2=0
   val3=1
@@ -559,7 +604,7 @@ test_sysctl(){
   echo '[SUCCESS] sysctl test3 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid key
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid1.yaml
   echo "conf:
   sysctl:
     this.is.a.bogus.key: 1" > "${overrides_yaml}"
@@ -569,7 +614,7 @@ test_sysctl(){
   echo '[SUCCESS] sysctl test4 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid val
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid2.yaml
   echo "conf:
   sysctl:
     $SYSCTL_KEY1: bogus" > "${overrides_yaml}"
@@ -592,7 +637,7 @@ _test_limits_value(){
 }
 
 test_limits(){
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}.yaml
   echo "conf:
   limits:
     limit1:
@@ -618,12 +663,24 @@ _test_perm_value(){
   local owner=${2}
   local group=${3}
   local perm=${4}
-  local r_owner="$(stat -c %U ${file})"
-  local r_group="$(stat -c %G ${file})"
-  local r_perm="$(stat -c %a ${file})"
-  [ "${perm}"=="${r_perm}" ] && echo "+" || (echo "File ${file} permissions ${r_perm} but expected ${perm}"; exit 1)
-  [ "${owner}"=="${r_owner}" ] && echo "+" || (echo "File ${file} owner ${r_owner} but expected ${owner}"; exit 1)
-  [ "${group}"=="${r_group}" ] && echo "+" || (echo "File ${file} group ${r_group} but expected ${group}"; exit 1)
+  local r_owner
+  local r_group
+  local r_perm
+  r_owner="$(stat -c %U ${file})"
+  r_group="$(stat -c %G ${file})"
+  r_perm="$(stat -c %a ${file})"
+  if [ "${perm}" != "${r_perm}" ]; then
+    echo "[FAIL] File ${file} has permissions ${r_perm} but expected ${perm}" >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  if [ "${owner}" != "${r_owner}" ]; then
+    echo "[FAIL] File ${file} has owner ${r_owner} but expected ${owner}" >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  if [ "${group}" != "${r_group}" ]; then
+    echo "[FAIL] File ${file} has group ${r_group} but expected ${group}" >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 _perm_init_one(){
@@ -662,7 +719,7 @@ _perm_teardown(){
 
 test_perm(){
   _perm_init
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}.yaml
   echo "conf:
   perm:
     paths:
@@ -704,8 +761,10 @@ test_perm(){
       owner: 'root'
       group: 'shadow'
       permissions: '0640'" > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. Got' || \
-    (echo "[FAIL] perm test invalid rerun_interval value did not receive expected 'BAD .rerun_interval. Got' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" |& grep 'BAD .rerun_interval. Got')" ]; then
+    echo "[FAIL] perm test invalid rerun_interval value did not receive expected 'BAD .rerun_interval. Got' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] perm test invalid rerun_interval passed successfully' >> "${TEST_RESULTS}"
   # Test invalid rerun_interval combination
   echo "conf:
@@ -718,8 +777,10 @@ test_perm(){
       owner: 'root'
       group: 'shadow'
       permissions: '0640'" > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
-    (echo "[FAIL] perm invalid rerun_interval combination did not receive expected 'BAD COMBINATION' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" |& grep 'BAD COMBINATION')" ]; then
+    echo "[FAIL] perm invalid rerun_interval combination did not receive expected 'BAD COMBINATION' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] perm invalid rerun_interval combination passed successfully' >> "${TEST_RESULTS}"
   # test rerun_interval
   echo "conf:
@@ -736,33 +797,42 @@ test_perm(){
   sleep 72
   get_container_status perm
   _test_perm_value ${p_test_file1} root shadow 640
-  echo '[SUCCESS] perm rerun_interval passed successfully' >> "${TEST_RESULTS}"
+  echo '[SUCCESS] perm rerun_interval passed successfully.' >> "${TEST_RESULTS}"
   _perm_teardown
 }
 
 _test_if_mounted_positive(){
-  mountpoint "${1}" || (echo "Expect ${1} to be mounted, but was not"; exit 1)
-  df -h | grep "${1}" | grep "${2}" ||
-    (echo "Did not find expected mount size of ${2} in mount table"; exit 1)
+  if ! mountpoint "${1}"; then
+    echo "[FAIL] Expected ${1} to be mounted, but it is not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  if [ -z "$(df -h | grep ${1} | grep ${2})" ]; then
+    echo "[FAIL] Expected to find mount size of ${2} for mountpoint ${1} in mount table, but did not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
   __set_systemd_name "${1}" mount
-  systemctl is-enabled "${SYSTEMD_NAME}" ||
-    (echo "Expect ${SYSTEMD_NAME} to be flagged to start on boot, but is not"
-     exit 1)
+
+  if ! systemctl is-enabled "${SYSTEMD_NAME}"; then
+    echo "[FAIL] Expected ${SYSTEMD_NAME} to be flagged to start on boot, but it is not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 _test_if_mounted_negative(){
-  mountpoint "${1}" &&
-    (echo "Expect ${1} not to be mounted, but was"
-     exit 1) || true
+  if mountpoint "${1}"; then
+    echo "[FAIL] Expected ${1} not to be mounted, but it was." >> "${TEST_RESULTS}"
+    exit 1
+  fi
   __set_systemd_name "${1}" mount
-  systemctl is-enabled "${SYSTEMD_NAME}" &&
-    (echo "Expect ${SYSTEMD_NAME} not to be flagged to start on boot, but was"
-     exit 1) || true
+  if systemctl is-enabled "${SYSTEMD_NAME}"; then
+    echo "[FAIL] Expected ${SYSTEMD_NAME} not to be flagged to start on boot, but it is." >> "${TEST_RESULTS}"
+    exit 1
+  fi
 }
 
 test_mounts(){
   # Test the first set of values
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   local mount_size=32M
   echo "conf:
   mounts:
@@ -791,7 +861,7 @@ test_mounts(){
   echo '[SUCCESS] mounts test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test an updated set of values
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   mount_size=30M
   echo "conf:
   mounts:
@@ -828,7 +898,7 @@ test_mounts(){
   echo '[SUCCESS] mounts test3 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid mount
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid1.yaml
   echo "conf:
   mounts:
     mnt:
@@ -852,8 +922,15 @@ _test_ethtool_value(){
 }
 
 test_ethtool(){
+  # On certain opendev hardware, it's not possible to change the
+  # ethtool tunables, or the expected tunables are unavailable.
+  # Until we have a mechanism to schedule to the right hardware,
+  # we will just issue a warning whenever these tests fail instead
+  # of failing the gate (thus the "ignore_failure" on get_container_status
+  # calls in this section)
+
   # Test the first set of values
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   local val2=on
   local val3=off
   [ -n "${ETHTOOL_KEY3}" ] && local line2_1="${ETHTOOL_KEY3}: $val3"
@@ -865,14 +942,14 @@ test_ethtool(){
       $line2_1
       $ETHTOOL_KEY4: $val4" > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
-  get_container_status ethtool
+  get_container_status ethtool ignore_failure
   _test_ethtool_value $ETHTOOL_KEY2 $val2
   _test_ethtool_value "$ETHTOOL_KEY3" $val3
   _test_ethtool_value $ETHTOOL_KEY4 $val4
   echo '[SUCCESS] ethtool test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test an updated set of values
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   val2=off
   val3=on
   [ -n "${ETHTOOL_KEY3}" ] && local line2_2="${ETHTOOL_KEY3}: $val3"
@@ -884,32 +961,32 @@ test_ethtool(){
       $line2_2
       $ETHTOOL_KEY4: $val4" > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
-  get_container_status ethtool
+  get_container_status ethtool ignore_failure
   _test_ethtool_value $ETHTOOL_KEY2 $val2 && \
-    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY2:$val2 passed successfully" || \
+    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY2:$val2 passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
   _test_ethtool_value "$ETHTOOL_KEY3" $val3 && \
-    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY3:$val3 passed successfully" || \
+    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY3:$val3 passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
   _test_ethtool_value $ETHTOOL_KEY4 $val4 && \
-    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY4:$val4 passed successfully" || \
+    echo "[SUCCESS] ethtool test2 $ETHTOOL_KEY4:$val4 passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
 
   # Test revert/rollback functionality
   install_base
-  get_container_status ethtool
+  get_container_status ethtool ignore_failure
   _test_ethtool_value $ETHTOOL_KEY2 $ETHTOOL_VAL2_DEFAULT && \
-    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY2:$ETHTOOL_VAL2_DEFAULT passed successfully" || \
+    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY2:$ETHTOOL_VAL2_DEFAULT passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
   _test_ethtool_value "$ETHTOOL_KEY3" $ETHTOOL_VAL3_DEFAULT && \
-    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY3:$ETHTOOL_VAL3_DEFAULT passed successfully" || \
+    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY3:$ETHTOOL_VAL3_DEFAULT passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
   _test_ethtool_value $ETHTOOL_KEY4 $ETHTOOL_VAL4_DEFAULT && \
-    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY4:$ETHTOOL_VAL4_DEFAULT passed successfully" || \
+    echo "[SUCCESS] ethtool test3 $ETHTOOL_KEY4:$ETHTOOL_VAL4_DEFAULT passed successfully" >> "${TEST_RESULTS}" || \
     ethtool_opendev_warn
 
   # Test invalid key
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid1.yaml
   echo "conf:
   ethtool:
     ${DEVICE}:
@@ -920,7 +997,7 @@ test_ethtool(){
   echo '[SUCCESS] ethtool test4 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid val
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid2.yaml
   echo "conf:
   ethtool:
     ${DEVICE}:
@@ -931,7 +1008,7 @@ test_ethtool(){
   echo '[SUCCESS] ethtool test5 passed successfully' >> "${TEST_RESULTS}"
 
   # Test fixed (unchangeable) ethtool param
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid3.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid3.yaml
   echo "conf:
   ethtool:
     ${DEVICE}:
@@ -942,7 +1019,7 @@ test_ethtool(){
   echo '[SUCCESS] ethtool test6 passed successfully' >> "${TEST_RESULTS}"
 
   # Test ethtool settings conflict
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid4.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid4.yaml
   echo "conf:
   ethtool:
     ${DEVICE}:
@@ -959,16 +1036,25 @@ _test_user_enabled(){
   username=$1
   user_enabled=$2
 
+  # verify the user exists
+  if ! getent passwd "${username}" >& /dev/null; then
+    echo "[FAIL] Expected user ${username} to exist, but it does not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
   if [ "${user_enabled}" = "true" ]; then
-    # verify the user is there and not set to expire
-    getent passwd $username >& /dev/null
-    test "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
-            tr -d '[:space:]')" = "never"
+    # verify the user is not set to expired
+    if [ "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
+            tr -d '[:space:]')" != "never" ]; then
+      echo "[FAIL] Expected user ${username} to be set to non-expired, but it was not." >> "${TEST_RESULTS}"
+      exit 1
+    fi
   else
-    # Verify user is not non-expiring
-    getent passwd $username >& /dev/null
-    test "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
-            tr -d '[:space:]')" != "never"
+    # Verify user is set to expired (@ t=1, 2 Jan 1970)
+    if [ "$(chage -l ${username} | grep 'Account expires' | cut -d':' -f2 |
+            tr -d '[:space:]')" != "Jan02,1970" ]; then
+      echo "[FAIL] Expected user ${username} to be set to expired, but it was not." >> "${TEST_RESULTS}"
+      exit 1
+    fi
   fi
 }
 
@@ -976,25 +1062,30 @@ _test_user_purged(){
   username=$1
 
   # Verify user is no longer defined
-  getent passwd $username >& /dev/null && \
-    echo "Error: User '$username' exists, but was expected it to be purged" && \
-    return 1
+  if getent passwd "${username}" >& /dev/null; then
+    echo "[FAIL] Expected user ${username} to be purged, but it was not."  >> "${TEST_RESULTS}"
+    exit 1
+  fi
 
-  if [ -d /home/$username ]; then
-    echo "Error: User '$username' home dir exists; expected it to be purged"
-    return 1
+  if [ -d "/home/${username}" ]; then
+    echo "[FAIL] Expected home directory for user ${username} to be removed, but it was not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 }
 
 _test_sudo_enabled(){
   username=$1
   sudo_enable=$2
-  sudoers_file=/etc/sudoers.d/*$username*
+  local keyword='divingbell'
+  sudoers_file=$(ls -l /etc/sudoers.d | grep "${keyword}-${username}-sudo" | head -n 1)
 
-  if [ "${sudo_enable}" = "true" ]; then
-    test -f $sudoers_file
-  else
-    test ! -f $sudoers_file
+  if [ "${sudo_enable}" = "true" ] && [ -z "$sudoers_file" ]; then
+    echo "[FAIL] Expected user ${username} to have a file named ${keyword}-${username}-sudo in the sudoers directory, but it does not." >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  if [ "${sudo_enable}" != "true" ] && [ -n "$sudoers_file" ]; then
+    echo "[FAIL] Expected user ${username} to have no file named ${keyword}-${username}-sudo in the sudoers directory, but it has one." >> "${TEST_RESULTS}"
+    exit 1
   fi
 }
 
@@ -1004,9 +1095,15 @@ _test_ssh_keys(){
   ssh_file=/home/$username/.ssh/authorized_keys
 
   if [ "$sshkey" = "false" ]; then
-    test ! -f "${ssh_file}"
+    if [ -f "${ssh_file}" ]; then
+      echo "[FAIL] Expected user ${username} to have no .ssh/authorized_keys file, but it has one." >> "${TEST_RESULTS}"
+      exit 1
+    fi
   else
-    grep "$sshkey" "${ssh_file}"
+    if [ -z "$(grep ${sshkey} ${ssh_file})" ]; then
+      echo "[FAIL] Expected user ${username} to have ssh key ${sshkey}, but it does not." >> "${TEST_RESULTS}"
+      exit 1
+    fi
   fi
 }
 
@@ -1015,14 +1112,14 @@ _test_user_passwd(){
   crypt_passwd="$2"
 
   if [ "$crypt_passwd" != "$(getent shadow $username | cut -d':' -f2)" ]; then
-    echo "Error: User '$username' passwd did not match expected val '$crypt_passwd'"
-    return 1
+    echo "[FAIL] Expected user ${username} to have password ${crypt_passwd}, but it did not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 }
 
 test_uamlite(){
   # Test the first set of values
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   echo "conf:
   uamlite:
     users:
@@ -1063,7 +1160,7 @@ test_uamlite(){
   echo '[SUCCESS] uamlite test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test an updated set of values
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   uname1_sudo=false
   uname2_sudo=true
   uname3_sudo=false
@@ -1119,7 +1216,7 @@ test_uamlite(){
   echo '[SUCCESS] uamlite test3 passed successfully' >> "${TEST_RESULTS}"
 
   # Test purge users flag
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set4.yaml
   echo "conf:
   uamlite:
     purge_expired_users: true" > "${overrides_yaml}"
@@ -1132,19 +1229,21 @@ test_uamlite(){
   echo '[SUCCESS] uamlite test4 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid password
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set5.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set5.yaml
   user2_crypt_passwd_invalid='plaintextPassword'
   echo "conf:
   uamlite:
     users:
     - user_name: ${USERNAME2}
       user_crypt_passwd: ${user2_crypt_passwd_invalid}" > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD PASSWORD' || \
-    (echo "[FAIL] uamlite test5 did not receive expected 'BAD PASSWORD' error" && exit 1)
-  echo '[SUCCESS] uamlite test5 passed successfully' >> "${TEST_RESULTS}"
+  if [ -z "$(install_base "--values=${overrides_yaml}" |& grep 'BAD PASSWORD')" ]; then
+    echo "[FAIL] uamlite test5 did not receive expected 'BAD PASSWORD' error." >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] uamlite test5 passed successfully.' >> "${TEST_RESULTS}"
 
   # Test invalid SSH key
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set6.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set6.yaml
   user2_bad_sshkey='AAAAB3NzaC1yc2EAAAABIwAAAQEAklOUpkDHrfHY17SbrmT key-comment'
   echo "conf:
   uamlite:
@@ -1154,29 +1253,31 @@ test_uamlite(){
       - ${USERNAME2_SSHKEY1}
       - ${user2_bad_sshkey}
       - ${USERNAME2_SSHKEY3}" > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD SSH KEY' || \
-    (echo "[FAIL] uamlite test6 did not receive expected 'BAD SSH KEY' error" && exit 1)
-  echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}"
+  if [ -z "$(install_base "--values=${overrides_yaml}" |& grep 'BAD SSH KEY')" ]; then
+    echo "[FAIL] uamlite test6 did not receive expected 'BAD SSH KEY' error." >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] uamlite test6 passed successfully.' >> "${TEST_RESULTS}"
 }
 
 _test_apt_package_version(){
   local pkg_name=$1
   local pkg_ver=$2
-  if [ ${pkg_ver} = "none" ]; then
+  if [ "${pkg_ver}" = "none" ]; then
     # Does not include residual-config
-    if [[ $(dpkg -l | grep ${pkg_name} | grep -v ^rc) ]]; then
-      echo "[FAIL] Package ${pkg_name} should not be installed" >> "${TEST_RESULTS}"
-      return 1
+    if [ -n "$(dpkg -l | grep ${pkg_name} | grep -v ^rc)" ]; then
+      echo "[FAIL] Expected package ${pkg_name} not to be installed, but it was." >> "${TEST_RESULTS}"
+      exit 1
     fi
-  elif [ ${pkg_ver} = "any" ]; then
-    if [[ ! $(dpkg -l | grep ${pkg_name}) ]]; then
-      echo "[FAIL] Package ${pkg_name} should be installed" >> "${TEST_RESULTS}"
-      return 1
+  elif [ "${pkg_ver}" = "any" ]; then
+    if [ -z "$(dpkg -l | grep ${pkg_name})" ]; then
+      echo "[FAIL] Expected package ${pkg_name} to be installed, but it wasn't." >> "${TEST_RESULTS}"
+      exit 1
     fi
   else
     if [ $(dpkg -l | awk "/[[:space:]]${pkg_name}[[:space:]]/"'{print $3}') != "${pkg_ver}" ]; then
-      echo "[FAIL] Package ${pkg_name} should be of version ${pkg_ver}" >> "${TEST_RESULTS}"
-      return 1
+      echo "[FAIL] Expected package ${pkg_name} version ${pkg_ver} to be installed, but it wasn't." >> "${TEST_RESULTS}"
+      exit 1
     fi
   fi
 }
@@ -1188,36 +1289,49 @@ _test_apt_repositories(){
   do
     if ! grep -qrh "$repository" /etc/apt/sources.list /etc/apt/sources.list.d/*
     then
-      echo "[FAIL] The repository (${repository}) was not added."
-      #return 1
+      echo "[FAIL] Expected repository ${repository} to be added, but it wasn't." >> "${TEST_RESULTS}"
+      exit 1
     fi
   done
-  remaining_repos=$(grep -rh "^deb" /etc/apt/sources.list /etc/apt/sources.list.d/* | sort -u | grep -v "${repositories// /\\|}" | awk '{print$2}')
+  remaining_repos=$(grep -qrh "^deb" /etc/apt/sources.list /etc/apt/sources.list.d/* | sort -u | grep -v "${repositories// /\\|}" | awk '{print$2}')
   for repo in $remaining_repos
   do
-    echo "[FAIL] Repository ${repo} should not be added."
+    echo "[FAIL] Expected repository ${repo} not to be added, but it was." >> "${TEST_RESULTS}"
+    exit 1
   done
 }
 
 _test_apt_keys(){
+  # NOTE: this is somewhat brittle as the output format of apt-key has been known to change.
+  # current code is written for the format:
+  # /etc/apt/trusted.gpg.d/some-key.gpg
+  # ------------------------------------------------------
+  # pub   rsa4096 2001-01-01 [SC]
+  #       0000 0000 0000 0000 0000  0000 0000 0000 0000 0000
+  # uid           [ unknown] words words words (2001) <somebody@somewhere.com>
+  # The keys specified at the top of the script ($APT_GPGKEYID1 etc.) should match the
+  # first 4-digit hex value listed.
+
   local keys=$1
   for key in $keys
   do
-    if ! apt-key list | grep -q "$key"
+    if [ -z "$(apt-key list | grep "$key")" ]
     then
-      echo "[FAIL] The gpg key (${key}) was not installed"
+      echo "[FAIL] The gpg key starting with (${key}) was not installed." >> "${TEST_RESULTS}"
+      exit 1
     fi
   done
-  remaining_keys=$(apt-key list | grep "^pub" | grep -v "${keys// /\\|}" | awk '{print$2}')
+  remaining_keys=$(apt-key list | grep -A 1 "^pub" | grep -v "^pub" | grep -v -x "\-\-" | grep -v "${keys// /\\|}" | awk '{print$1}')
   for rkey in $remaining_keys
   do
-    echo "[FAIL] The gpg key (${rkey}) should not be installed"
+    echo "[FAIL] The gpg key starting with (${rkey}) should not be installed." >> "${TEST_RESULTS}"
+    exit 1
   done
 }
 
 test_apt(){
   # Test the valid set of packages
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   echo "conf:
   apt:
     allow_downgrade: true
@@ -1232,7 +1346,7 @@ test_apt(){
   echo '[SUCCESS] apt test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test removal of one package and install of one new package
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   echo "conf:
   apt:
     packages:
@@ -1252,15 +1366,15 @@ test_apt(){
   _test_apt_package_version $APT_PACKAGE2 any
   # Each entry in passwords.dat contains question value in Name and Template
   # field, so grepping root_password should return 4 lines
-  if [[ $(grep root_password /var/cache/debconf/passwords.dat | wc -l) != 4 ]]; then
+  if [[ $(grep -c root_password /var/cache/debconf/passwords.dat) != 4 ]]; then
     echo "[FAIL] Package $APT_PACKAGE2 should have debconf values configured" >> "${TEST_RESULTS}"
-    return 1
+    exit 1
   fi
   _test_apt_package_version $APT_PACKAGE3 $APT_VERSION3
   echo '[SUCCESS] apt test2 passed successfully' >> "${TEST_RESULTS}"
 
   # Test removal of all installed packages and install of one that already exists
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set3.yaml
   echo "conf:
   apt:
     packages:
@@ -1272,7 +1386,7 @@ test_apt(){
   echo '[SUCCESS] apt test3 passed successfully' >> "${TEST_RESULTS}"
 
   # Test package not installed by divingbell not removed
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set4.yaml
   echo "conf:
   apt:
     packages:
@@ -1284,7 +1398,7 @@ test_apt(){
   echo '[SUCCESS] apt test4 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid package name
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-invalid1.yaml
   echo "conf:
   apt:
     packages:
@@ -1296,7 +1410,7 @@ test_apt(){
   echo '[SUCCESS] apt test5 passed successfully' >> "${TEST_RESULTS}"
 
   # Test blacklistpkgs
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set5.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set5.yaml
   echo "conf:
   apt:
     packages:
@@ -1309,7 +1423,7 @@ test_apt(){
   echo '[SUCCESS] apt test6 passed successfully' >> "${TEST_RESULTS}"
 
   # Test add several repositories with gpg keys
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set6.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set6.yaml
   echo "conf:
   apt:
     repositories:
@@ -1341,7 +1455,7 @@ $(printf '%s' "$APT_GPGKEY3" | awk '{printf "          %s\n", $0}')" > "${overri
   echo '[SUCCESS] apt test7 passed successfully' >> "${TEST_RESULTS}"
 
   # Test add same gpg key two times
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set7.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set7.yaml
   echo "conf:
   apt:
     repositories:
@@ -1366,7 +1480,7 @@ $(printf '%s' "$APT_GPGKEY1" | awk '{printf "          %s\n", $0}')" > "${overri
   echo '[SUCCESS] apt test8 passed successfully' >> "${TEST_RESULTS}"
 
   # Test groups of packages using a map
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set8.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set8.yaml
   echo "conf:
   apt:
     packages:
@@ -1381,7 +1495,7 @@ $(printf '%s' "$APT_GPGKEY1" | awk '{printf "          %s\n", $0}')" > "${overri
   echo '[SUCCESS] apt test9 passed successfully' >> "${TEST_RESULTS}"
 
   # Test adding a package in strict mode
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set9.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set9.yaml
   APT_ALL_INSTALLED_PACKAGES="    packages:"
   build_all_packages_yaml $(dpkg -l | awk 'NR>5 {print $2}')
   echo "conf:
@@ -1397,7 +1511,7 @@ $APT_ALL_INSTALLED_PACKAGES
   echo '[SUCCESS] apt test10 passed successfully' >> "${TEST_RESULTS}"
 
   # Test removing a package in strict mode
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set10.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set10.yaml
   # using the same APT_ALL_INSTALLED_PACKAGES from above,
   # which does NOT have APT_PACKAGE1
   echo "conf:
@@ -1415,7 +1529,7 @@ $APT_ALL_INSTALLED_PACKAGES" > "${overrides_yaml}"
 # test exec module
 test_exec(){
   # test script execution ordering, args, and env vars
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set1.yaml
   echo 'conf:
   exec:
     030-script5.sh:
@@ -1430,7 +1544,7 @@ test_exec(){
       - arg3
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
         echo args: "$@" >> exec_testfile
         echo env: "$env1 $env2 $env3" >> exec_testfile
     005-script1.sh:
@@ -1438,20 +1552,20 @@ test_exec(){
       data: |
         #!/bin/bash
         rm exec_testfile 2> /dev/null || true
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
     015-script3.sh:
       blocking_policy: foreground_halt_pod_on_failure
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
     008-script2.sh:
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
     025-script4.sh:
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile' > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status exec
   expected_result='script name: ./005-script1.sh
@@ -1465,7 +1579,7 @@ env: env1-val env2-val env3-val'
   echo '[SUCCESS] exec test1 passed successfully' >> "${TEST_RESULTS}"
 
   # Test blocking_policy
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set2.yaml
   echo 'conf:
   exec:
     030-script5.sh:
@@ -1480,7 +1594,7 @@ env: env1-val env2-val env3-val'
       - arg3
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
         echo args: "$@" >> exec_testfile
         echo env: "$env1 $env2 $env3" >> exec_testfile
     005-script1.sh:
@@ -1488,21 +1602,21 @@ env: env1-val env2-val env3-val'
       data: |
         #!/bin/bash
         rm exec_testfile 2> /dev/null || true
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
     015-script3.sh:
       blocking_policy: foreground_halt_pod_on_failure
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
         false
     008-script2.sh:
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
     025-script4.sh:
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile' > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status exec expect_failure
   expected_result='script name: ./005-script1.sh
@@ -1512,7 +1626,7 @@ script name: ./015-script3.sh'
   echo '[SUCCESS] exec test2 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid rerun_policy
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set3.yaml
   echo 'conf:
   exec:
     030-script5.sh:
@@ -1520,12 +1634,14 @@ script name: ./015-script3.sh'
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_policy. FOR' || \
-    (echo "[FAIL] exec test3 did not receive expected 'BAD .rerun_policy. FOR' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_policy. FOR')" ]; then
+    echo "[FAIL] exec test3 did not receive expected 'BAD .rerun_policy. FOR' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test3 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid blocking_policy
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set4.yaml
   echo 'conf:
   exec:
     030-script5.sh:
@@ -1533,8 +1649,10 @@ script name: ./015-script3.sh'
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .blocking_policy. FOR' || \
-    (echo "[FAIL] exec test4 did not receive expected 'BAD .blocking_policy. FOR' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .blocking_policy. FOR')" ]; then
+    echo "[FAIL] exec test4 did not receive expected 'BAD .blocking_policy. FOR' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test4 passed successfully' >> "${TEST_RESULTS}"
 
   # Test rerun_policies:
@@ -1545,34 +1663,34 @@ script name: ./015-script3.sh'
   # 5. never
 
   # first execution
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set5.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set5.yaml
   echo 'conf:
   exec:
     001-script1.sh:
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> script1
+        echo script name: ${BASH_SOURCE[0]} >> script1
     002-script2.sh:
       rerun_policy: always
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> script2
+        echo script name: ${BASH_SOURCE[0]} >> script2
     003-script3.sh:
       rerun_policy: once_successfully
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> script3
+        echo script name: ${BASH_SOURCE[0]} >> script3
     004-script4.sh:
       rerun_policy: once_successfully
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> script4
+        echo script name: ${BASH_SOURCE[0]} >> script4
         false
     005-script5.sh:
       rerun_policy: never
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> script5
+        echo script name: ${BASH_SOURCE[0]} >> script5
       env:
         env3: env3-val
         env1: env1-val
@@ -1607,7 +1725,7 @@ manifests:
   done
 
   # test timeout
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set17.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set17.yaml
   echo 'conf:
   exec:
     011-timeout.sh:
@@ -1616,12 +1734,12 @@ manifests:
         #!/bin/bash
         sleep 60' > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
-  get_container_status exec
+  get_container_status exec expect_failure
   _test_clog_msg 'timeout waiting for'
   echo '[SUCCESS] exec test17 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid timeout
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set18.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set18.yaml
   echo 'conf:
   exec:
     011-timeout.sh:
@@ -1629,12 +1747,14 @@ manifests:
       data: |
         #!/bin/bash
         sleep 60' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .timeout. FOR' || \
-    (echo "[FAIL] exec test18 did not receive expected 'BAD .timeout. FOR' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .timeout. FOR')" ]; then
+    echo "[FAIL] exec test18 did not receive expected 'BAD .timeout. FOR' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test18 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid rerun_interval (too short)
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set19.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set19.yaml
   echo 'conf:
   exec:
     012-rerun-interval.sh:
@@ -1642,12 +1762,14 @@ manifests:
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. FOR' || \
-    (echo "[FAIL] exec test19 did not receive expected 'BAD .rerun_interval. FOR' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .rerun_interval. FOR')" ]; then
+    echo "[FAIL] exec test19 did not receive expected 'BAD .rerun_interval. FOR' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test19 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid retry_interval (too short)
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set20.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set20.yaml
   echo 'conf:
   exec:
     012-retry-interval.sh:
@@ -1655,12 +1777,14 @@ manifests:
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .retry_interval. FOR' || \
-    (echo "[FAIL] exec test20 did not receive expected 'BAD .retry_interval. FOR' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD .retry_interval. FOR')" ]; then
+    echo "[FAIL] exec test20 did not receive expected 'BAD .retry_interval. FOR' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test20 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid rerun_interval combination
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set21.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set21.yaml
   echo 'conf:
   exec:
     012-rerun-interval.sh:
@@ -1669,12 +1793,14 @@ manifests:
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
-    (echo "[FAIL] exec test21 did not receive expected 'BAD COMBINATION' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION')" ]; then
+    echo "[FAIL] exec test21 did not receive expected 'BAD COMBINATION' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test21 passed successfully' >> "${TEST_RESULTS}"
 
   # Test invalid retry_interval combination
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set22.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set22.yaml
   echo 'conf:
   exec:
     012-retry-interval.sh:
@@ -1683,19 +1809,21 @@ manifests:
       data: |
         #!/bin/bash
         true' > "${overrides_yaml}"
-  install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION' || \
-    (echo "[FAIL] exec test22 did not receive expected 'BAD COMBINATION' error" && exit 1)
+  if [ -z "$(install_base "--values=${overrides_yaml}" 2>&1 | grep 'BAD COMBINATION')" ]; then
+    echo "[FAIL] exec test22 did not receive expected 'BAD COMBINATION' error" >> "${TEST_RESULTS}"
+    exit 1
+  fi
   echo '[SUCCESS] exec test22 passed successfully' >> "${TEST_RESULTS}"
 
   # test rerun_interval
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set23.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set23.yaml
   echo 'conf:
   exec:
     012-rerun-interval.sh:
       rerun_interval: 60
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile' > "${overrides_yaml}"
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile' > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status exec
   sleep 75
@@ -1706,14 +1834,14 @@ script name: ./012-rerun-interval.sh'
   echo '[SUCCESS] exec test23 passed successfully' >> "${TEST_RESULTS}"
 
   # test retry_interval
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set24.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-set24.yaml
   echo 'conf:
   exec:
     012-retry-interval.sh:
       retry_interval: 60
       data: |
         #!/bin/bash
-        echo script name: ${BASH_SOURCE} >> exec_testfile
+        echo script name: ${BASH_SOURCE[0]} >> exec_testfile
         false' > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status exec
@@ -1727,7 +1855,7 @@ script name: ./012-retry-interval.sh'
 
 # test daemonset value overrides for hosts and labels
 test_overrides(){
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-dryrun.yaml
   echo "conf:
   sysctl:
     net.ipv4.ip_forward: 1
@@ -1819,9 +1947,9 @@ test_overrides(){
   tc_output="$(dry_run_base "--values=${overrides_yaml}")"
 
   # Compare against expected number of generated daemonsets
-  daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
+  daemonset_count="$(echo "${tc_output}" | grep -c 'kind: DaemonSet')"
   if [ "${daemonset_count}" != "${EXPECTED_NUMBER_OF_DAEMONSETS}" ]; then
-    echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
+    echo '[FAIL] overrides test 1 failed' >> "${TEST_RESULTS}"
     echo "Expected ${EXPECTED_NUMBER_OF_DAEMONSETS} daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
     exit 1
   else
@@ -1832,7 +1960,7 @@ test_overrides(){
   # ordering.
 
   # Verify generated affinity for another_label
-  echo "${tc_output}" | grep '    spec:
+  affinity_match_2=$(echo "${tc_output}" | grep '    spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -1853,12 +1981,15 @@ test_overrides(){
               - key: kubernetes.io/hostname
                 operator: NotIn
                 values:
-                - "specialhost"' &&
-  echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" ||
-  (echo '[FAILURE] overrides test 2 failed' && exit 1)
+                - "specialhost"')
+  if [ -z "${affinity_match_2}" ]; then
+    echo '[FAIL] overrides test 2 failed' >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}"
 
   # Verify generated affinity for compute_type
-  echo "${tc_output}" | grep '    spec:
+  affinity_match_3=$(echo "${tc_output}" | grep '    spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -1883,12 +2014,15 @@ test_overrides(){
               - key: kubernetes.io/hostname
                 operator: NotIn
                 values:
-                - "specialhost"' &&
-  echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
-  (echo '[FAILURE] overrides test 3 failed' && exit 1)
+                - "specialhost"')
+  if [ -z "${affinity_match_3}" ]; then
+    echo '[FAIL] overrides test 3 failed' >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}"
 
   # Verify generated affinity for compute_type
-  echo "${tc_output}" | grep '    spec:
+  affinity_match_4=$(echo "${tc_output}" | grep '    spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -1918,12 +2052,15 @@ test_overrides(){
               - key: kubernetes.io/hostname
                 operator: NotIn
                 values:
-                - "specialhost"' &&
-  echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
-  (echo '[FAILURE] overrides test 4 failed' && exit 1)
+                - "specialhost"')
+  if [ -z "${affinity_match_4}" ]; then
+    echo '[FAIL] overrides test 4 failed' >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}"
 
   # Verify generated affinity for one of the daemonset hosts
-  echo "${tc_output}" | grep '    spec:
+  affinity_match_5=$(echo "${tc_output}" | grep '    spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -1933,12 +2070,15 @@ test_overrides(){
                 operator: In
                 values:
                 - "soup"
-                - "chips"' &&
-  echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" ||
-  (echo '[FAILURE] overrides test 5 failed' && exit 1)
+                - "chips"')
+  if [ -z "${affinity_match_5}" ]; then
+    echo '[FAIL] overrides test 5 failed' >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}"
 
   # Verify generated affinity for one of the daemonset defaults
-  echo "${tc_output}" | grep '    spec:
+  affinity_match_6=$(echo "${tc_output}" | grep '    spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
@@ -1968,16 +2108,19 @@ test_overrides(){
               - key: another_label
                 operator: NotIn
                 values:
-                - "another_value"' &&
-  echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" ||
-  (echo '[FAILURE] overrides test 6 failed' && exit 1)
+                - "another_value"')
+  if [ -z "${affinity_match_6}" ]; then
+    echo '[FAIL] overrides test 6 failed' >> "${TEST_RESULTS}"
+    exit 1
+  fi
+  echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}"
 
   # The core functional test to ensure that overrides work.
   # fooKey was added to catch a corner case identified by:
   # https://storyboard.openstack.org/#!/story/2005936
   # If fooHost keys are leaking into this host's values, then this test
   # will fail when sysctl attempts to set the non-existant fooKey.
-  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-functional.yaml
   key1_override_val=0
   key2_non_override_val=0
   kube_hostname="$(kubectl describe nodes | grep kubernetes.io/hostname | head -1 | cut -d'=' -f2)" || true
@@ -2004,11 +2147,9 @@ test_overrides(){
             $SYSCTL_KEY1: $key1_override_val" > "${overrides_yaml}"
   install_base "--values=${overrides_yaml}"
   get_container_status sysctl
-  _test_sysctl_default $SYSCTL_KEY1 $key1_override_val || \
-  (echo '[FAILURE] overrides test 7 failed, most likely someone broke openstack-helm-infra/helm-toolkit/templates/utils/_daemonset_overrides.tpl' && exit 1)
+  _test_sysctl_default $SYSCTL_KEY1 $key1_override_val
   _test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
   echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}"
-
 }
 
 _test_apparmor_profile_added(){
@@ -2018,16 +2159,19 @@ _test_apparmor_profile_added(){
   local persist_path='/etc/apparmor.d'
 
   if [ ! -f "${defaults_path}/${profile_file}" ]; then
-    return 1
+    echo "[FAIL] Expected AppArmor profile ${profile_file} to be found on the defaults path ${defaults_path}, but it was not." >> "${TEST_RESULTS}"
+    exit 1
   fi
   if [ ! -L "${persist_path}/${profile_file}" ]; then
-    return 1
+    echo "[FAIL] Expected symlink to AppArmor profile ${profile_file} to be found on the persist path ${persist_path}, but it was not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 
   profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
 
   if [ -z "$profile_loaded" ]; then
-    return 1
+    echo "[FAIL] Expected AppArmor profile ${profile_file} to be loaded, but it is not." >> "${TEST_RESULTS}"
+    exit 1
   fi
   return 0
 }
@@ -2039,29 +2183,33 @@ _test_apparmor_profile_removed(){
   local persist_path='/etc/apparmor.d'
 
   if [ -f "${defaults_path}/${profile_file}" ]; then
-    return 1
+    echo "[FAIL] Expected AppArmor profile ${profile_file} to be removed from the defaults path ${defaults_path}, but it was not." >> "${TEST_RESULTS}"
+    exit 1
   fi
   if [ -L "${persist_path}/${profile_file}" ]; then
-    return 1
+    echo "[FAIL] Expected symlink to AppArmor profile ${profile_file} to be removed from the persist path ${persist_path}, but it was not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 
   profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
 
-  if [ ! -z "$profile_loaded" ]; then
-    return 1
+  if [ -n "$profile_loaded" ]; then
+    echo "[FAIL] Expected AppArmor profile ${profile_file} to be removed, but it is not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 
   reboot_message_present=$(grep $profile_file /var/run/reboot-required.pkgs || : )
 
   if [ -z "$reboot_message_present" ]; then
-    return 1
+    echo "[FAIL] Expected removed AppArmor profile ${profile_file} to be found in the reboot-required.pkgs file, but it is not." >> "${TEST_RESULTS}"
+    exit 1
   fi
 
   return 0
 }
 
 test_apparmor(){
-  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-apparmor.yaml
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME[0]}-apparmor.yaml
 
   #Test1 - check new profile added and loaded
   echo "conf: