[DATABASE] Add verify databases backup
HTK - added verify_databases_backup_in_directory function that is going to be defined inside mariadb/postgresql/etcd charts. Mariadb chart - added verify_databases_backup_archives function implementation. Added mariadb-verify container to mariadb-backup cronjob to run verification process. Added remove backup verification pocess - comparition of local and remote file md5 hashes. PostgreSQL chart - added empty implementation of verify_databases_backup_archives() function. This is a subject for future realization. Change-Id: I361cdb92c66b0b27539997d697adfd1e93c9a29d
This commit is contained in:
Markin, Sergiy (sm515x)
Sergiy Markin
parent
818c475f1d
commit
5c4056ad34
|
|
@ -15,7 +15,7 @@ apiVersion: v1
|
|||
appVersion: v1.0.0
|
||||
description: OpenStack-Helm Helm-Toolkit
|
||||
name: helm-toolkit
|
||||
version: 0.2.47
|
||||
version: 0.2.48
|
||||
home: https://docs.openstack.org/openstack-helm
|
||||
icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
|
||||
sources:
|
||||
|
|
|
|||
|
|
@ -66,6 +66,14 @@
|
|||
# framework will automatically tar/zip the files in that directory and
|
||||
# name the tarball appropriately according to the proper conventions.
|
||||
#
|
||||
# verify_databases_backup_archives [scope]
|
||||
# returns: 0 if no errors; 1 if any errors occurred
|
||||
#
|
||||
# This function is expected to verify the database backup archives. If this function
|
||||
# completes successfully (returns 0), the
|
||||
# framework will automatically starts remote backup upload.
|
||||
#
|
||||
#
|
||||
# The functions in this file will take care of:
|
||||
# 1) Calling "dump_databases_to_directory" and then compressing the files,
|
||||
# naming the tarball properly, and then storing it locally at the specified
|
||||
|
|
@ -90,6 +98,16 @@ log_backup_error_exit() {
|
|||
exit $ERRCODE
|
||||
}
|
||||
|
||||
log_verify_backup_exit() {
|
||||
MSG=$1
|
||||
ERRCODE=${2:-0}
|
||||
log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}"
|
||||
rm -f $ERR_LOG_FILE
|
||||
# rm -rf $TMP_DIR
|
||||
exit $ERRCODE
|
||||
}
|
||||
|
||||
|
||||
log() {
|
||||
#Log message to a file or stdout
|
||||
#TODO: This can be convert into mail alert of alert send to a monitoring system
|
||||
|
|
@ -201,12 +219,36 @@ send_to_remote_server() {
|
|||
log WARN "${DB_NAME}_backup" "Cannot create container object ${FILE}!"
|
||||
return 2
|
||||
fi
|
||||
|
||||
openstack object show $CONTAINER_NAME $FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to retrieve container object $FILE after creation."
|
||||
return 2
|
||||
fi
|
||||
|
||||
# Calculation remote file SHA256 hash
|
||||
REMOTE_FILE=$(mktemp -p /tmp)
|
||||
openstack object save --file ${REMOTE_FILE} $CONTAINER_NAME $FILE
|
||||
if [[ $? -ne 0 ]]; then
|
||||
log WARN "${DB_NAME}_backup" "Unable to save container object $FILE for SHA256 hash verification."
|
||||
rm -rf ${REMOTE_FILE}
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Remote backup verification
|
||||
SHA256_REMOTE=$(cat ${REMOTE_FILE} | sha256sum | awk '{print $1}')
|
||||
SHA256_LOCAL=$(cat ${FILEPATH}/${FILE} | sha256sum | awk '{print $1}')
|
||||
log INFO "${DB_NAME}_backup" "Calculated SHA256 hashes for the file $FILE in container $CONTAINER_NAME."
|
||||
log INFO "${DB_NAME}_backup" "Local SHA256 hash is ${SHA256_LOCAL}."
|
||||
log INFO "${DB_NAME}_backup" "Remote SHA256 hash is ${SHA256_REMOTE}."
|
||||
if [[ "${SHA256_LOCAL}" == "${SHA256_REMOTE}" ]]; then
|
||||
log INFO "${DB_NAME}_backup" "The local backup & remote backup SHA256 hash values are matching for file $FILE in container $CONTAINER_NAME."
|
||||
else
|
||||
log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup sha256 hash values"
|
||||
return 1
|
||||
fi
|
||||
rm -rf ${REMOTE_FILE}
|
||||
|
||||
log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
|
||||
return 0
|
||||
}
|
||||
|
|
@ -382,8 +424,8 @@ remove_old_remote_archives() {
|
|||
|
||||
# Cleanup now that we're done.
|
||||
for fd in ${BACKUP_FILES} ${DB_BACKUP_FILES}; do
|
||||
if [[ -f fd ]]; then
|
||||
rm -f fd
|
||||
if [[ -f ${fd} ]]; then
|
||||
rm -f ${fd}
|
||||
else
|
||||
log WARN "${DB_NAME}_backup" "Can not delete a temporary file ${fd}"
|
||||
fi
|
||||
|
|
@ -444,10 +486,6 @@ backup_databases() {
|
|||
|
||||
cd $ARCHIVE_DIR
|
||||
|
||||
# Remove the temporary directory and files as they are no longer needed.
|
||||
rm -rf $TMP_DIR
|
||||
rm -f $ERR_LOG_FILE
|
||||
|
||||
#Only delete the old archive after a successful archive
|
||||
export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g')
|
||||
if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then
|
||||
|
|
@ -459,6 +497,25 @@ backup_databases() {
|
|||
done
|
||||
fi
|
||||
|
||||
# Local backup verification process
|
||||
|
||||
# It is expected that this function will verify the database backup files
|
||||
if verify_databases_backup_archives ${SCOPE}; then
|
||||
log INFO "${DB_NAME}_backup_verify" "Databases backup verified successfully. Uploading verified backups to remote location..."
|
||||
else
|
||||
# If successful, there should be at least one file in the TMP_DIR
|
||||
if [[ $(ls $TMP_DIR | wc -w) -eq 0 ]]; then
|
||||
cat $ERR_LOG_FILE
|
||||
fi
|
||||
log_verify_backup_exit "Verify of the ${DB_NAME} database backup failed and needs attention."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Remove the temporary directory and files as they are no longer needed.
|
||||
rm -rf $TMP_DIR
|
||||
rm -f $ERR_LOG_FILE
|
||||
|
||||
# Remote backup
|
||||
REMOTE_BACKUP=$(echo $REMOTE_BACKUP_ENABLED | sed 's/"//g')
|
||||
if $REMOTE_BACKUP; then
|
||||
# Remove Quotes from the constants which were added due to reading
|
||||
|
|
@ -490,7 +547,7 @@ backup_databases() {
|
|||
get_backup_prefix $(cat $DB_BACKUP_FILES)
|
||||
for ((i=0; i<${#PREFIXES[@]}; i++)); do
|
||||
echo "Working with prefix: ${PREFIXES[i]}"
|
||||
create_hash_table $(cat $DB_BACKUP_FILES | grep ${PREFIXES[i]})
|
||||
create_hash_table $(cat ${DB_BACKUP_FILES} | grep ${PREFIXES[i]})
|
||||
remove_old_remote_archives
|
||||
done
|
||||
fi
|
||||
|
|
@ -511,4 +568,4 @@ backup_databases() {
|
|||
echo "=================================================================="
|
||||
fi
|
||||
}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
|
@ -12,10 +12,10 @@
|
|||
|
||||
---
|
||||
apiVersion: v1
|
||||
appVersion: v10.2.31
|
||||
appVersion: v10.6.7
|
||||
description: OpenStack-Helm MariaDB
|
||||
name: mariadb
|
||||
version: 0.2.27
|
||||
version: 0.2.28
|
||||
home: https://mariadb.com/kb/en/
|
||||
icon: http://badges.mariadb.org/mariadb-badge-180x60.png
|
||||
sources:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ dump_databases_to_directory() {
|
|||
LOG_FILE=$2
|
||||
SCOPE=${3:-"all"}
|
||||
|
||||
|
||||
MYSQL="mysql \
|
||||
--defaults-file=/etc/mysql/admin_user.cnf \
|
||||
--connect-timeout 10"
|
||||
|
|
@ -113,5 +114,471 @@ dump_databases_to_directory() {
|
|||
fi
|
||||
}
|
||||
|
||||
# functions from mariadb-verifier chart
|
||||
|
||||
get_time_delta_secs () {
|
||||
second_delta=0
|
||||
input_date_second=$( date --date="$1" +%s )
|
||||
if [ -n "$input_date_second" ]; then
|
||||
current_date=$( date +"%Y-%m-%dT%H:%M:%SZ" )
|
||||
current_date_second=$( date --date="$current_date" +%s )
|
||||
((second_delta=current_date_second-input_date_second))
|
||||
if [ "$second_delta" -lt 0 ]; then
|
||||
second_delta=0
|
||||
fi
|
||||
fi
|
||||
echo $second_delta
|
||||
}
|
||||
|
||||
|
||||
check_data_freshness () {
|
||||
archive_file=$(basename "$1")
|
||||
archive_date=$(echo "$archive_file" | cut -d'.' -f 4)
|
||||
SCOPE=$2
|
||||
|
||||
if [[ "${SCOPE}" != "all" ]]; then
|
||||
log "Data freshness check is skipped for individual database."
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "Checking for data freshness in the backups..."
|
||||
# Get some idea of which database.table has changed in the last 30m
|
||||
# Excluding the system DBs and aqua_test_database
|
||||
#
|
||||
changed_tables=$(${MYSQL_LIVE} -e "select TABLE_SCHEMA,TABLE_NAME from \
|
||||
information_schema.tables where UPDATE_TIME >= SUBTIME(now(),'00:30:00') AND TABLE_SCHEMA \
|
||||
NOT IN('information_schema', 'mysql', 'performance_schema', 'sys', 'aqua_test_database');" | \
|
||||
awk '{print $1 "." $2}')
|
||||
|
||||
if [ -n "${changed_tables}" ]; then
|
||||
delta_secs=$(get_time_delta_secs "$archive_date")
|
||||
age_offset={{ .Values.conf.backup.validateData.ageOffset }}
|
||||
((age_threshold=delta_secs+age_offset))
|
||||
|
||||
data_freshness=false
|
||||
skipped_freshness=false
|
||||
|
||||
for table in ${changed_tables}; do
|
||||
tab_schema=$(echo "$table" | awk -F. '{print $1}')
|
||||
tab_name=$(echo "$table" | awk -F. '{print $2}')
|
||||
|
||||
local_table_existed=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select TABLE_SCHEMA,TABLE_NAME from \
|
||||
INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA=\"${tab_schema}\" AND TABLE_NAME=\"${tab_name}\";")
|
||||
|
||||
if [ -n "$local_table_existed" ]; then
|
||||
# TODO: If last updated field of a table structure has different
|
||||
# patterns (updated/timstamp), it may be worth to parameterize the patterns.
|
||||
datetime=$(${MYSQL_LOCAL_SHORT_SILENT} -e "describe ${table};" | \
|
||||
awk '(/updated/ || /timestamp/) && /datetime/ {print $1}')
|
||||
|
||||
if [ -n "${datetime}" ]; then
|
||||
data_ages=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select \
|
||||
time_to_sec(timediff(now(),${datetime})) from ${table} where ${datetime} is not null order by 1 limit 10;")
|
||||
|
||||
for age in $data_ages; do
|
||||
if [ "$age" -le $age_threshold ]; then
|
||||
data_freshness=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
# As long as there is an indication of data freshness, no need to check further
|
||||
if [ "$data_freshness" = true ] ; then
|
||||
break
|
||||
fi
|
||||
else
|
||||
skipped_freshness=true
|
||||
log "No indicator to determine data freshness for table $table. Skipped data freshness check."
|
||||
|
||||
# Dumping out table structure to determine if enhancement is needed to include this table
|
||||
debug_info=$(${MYSQL_LOCAL} --skip-column-names -e "describe ${table};" | awk '{print $2 " " $1}')
|
||||
log "$debug_info" "DEBUG"
|
||||
fi
|
||||
else
|
||||
log "Table $table doesn't exist in local database"
|
||||
skipped_freshness=true
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$data_freshness" = true ] ; then
|
||||
log "Database passed integrity (data freshness) check."
|
||||
else
|
||||
if [ "$skipped_freshness" = false ] ; then
|
||||
log "Local backup database restore failed integrity check." "ERROR"
|
||||
log "The backup may not have captured the up-to-date data." "INFO"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
else
|
||||
log "No tables changed in this backup. Skipped data freshness check as the"
|
||||
log "check should have been performed by previous validation runs."
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
cleanup_local_databases () {
|
||||
old_local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
|
||||
grep -ivE 'information_schema|performance_schema|mysql|sys' || true)
|
||||
|
||||
for db in $old_local_dbs; do
|
||||
${MYSQL_LOCAL_SHORT_SILENT} -e "drop database $db;"
|
||||
done
|
||||
}
|
||||
|
||||
list_archive_dir () {
|
||||
archive_dir_content=$(ls -1R "$ARCHIVE_DIR")
|
||||
if [ -n "$archive_dir_content" ]; then
|
||||
log "Content of $ARCHIVE_DIR"
|
||||
log "${archive_dir_content}"
|
||||
fi
|
||||
}
|
||||
|
||||
remove_remote_archive_file () {
|
||||
archive_file=$(basename "$1")
|
||||
token_req_file=$(mktemp --suffix ".json")
|
||||
header_file=$(mktemp)
|
||||
resp_file=$(mktemp --suffix ".json")
|
||||
http_resp="404"
|
||||
|
||||
HEADER_CONTENT_TYPE="Content-Type: application/json"
|
||||
HEADER_ACCEPT="Accept: application/json"
|
||||
|
||||
cat << JSON_EOF > "$token_req_file"
|
||||
{
|
||||
"auth": {
|
||||
"identity": {
|
||||
"methods": [
|
||||
"password"
|
||||
],
|
||||
"password": {
|
||||
"user": {
|
||||
"domain": {
|
||||
"name": "${OS_USER_DOMAIN_NAME}"
|
||||
},
|
||||
"name": "${OS_USERNAME}",
|
||||
"password": "${OS_PASSWORD}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"scope": {
|
||||
"project": {
|
||||
"domain": {
|
||||
"name": "${OS_PROJECT_DOMAIN_NAME}"
|
||||
},
|
||||
"name": "${OS_PROJECT_NAME}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
JSON_EOF
|
||||
|
||||
http_resp=$(curl -s -X POST "$OS_AUTH_URL/auth/tokens" -H "${HEADER_CONTENT_TYPE}" \
|
||||
-H "${HEADER_ACCEPT}" -d @"${token_req_file}" -D "$header_file" -o "$resp_file" -w "%{http_code}")
|
||||
|
||||
if [ "$http_resp" = "201" ]; then
|
||||
OS_TOKEN=$(grep -i "x-subject-token" "$header_file" | cut -d' ' -f2 | tr -d "\r")
|
||||
|
||||
if [ -n "$OS_TOKEN" ]; then
|
||||
OS_OBJ_URL=$(python3 -c "import json,sys;print([[ep['url'] for ep in obj['endpoints'] if ep['interface']=='public'] for obj in json.load(sys.stdin)['token']['catalog'] if obj['type']=='object-store'][0][0])" < "$resp_file")
|
||||
|
||||
if [ -n "$OS_OBJ_URL" ]; then
|
||||
http_resp=$(curl -s -X DELETE "$OS_OBJ_URL/$CONTAINER_NAME/$archive_file" \
|
||||
-H "${HEADER_CONTENT_TYPE}" -H "${HEADER_ACCEPT}" \
|
||||
-H "X-Auth-Token: ${OS_TOKEN}" -D "$header_file" -o "$resp_file" -w "%{http_code}")
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ "$http_resp" == "404" ] ; then
|
||||
log "Failed to cleanup remote backup. Container object $archive_file is not on RGW."
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$http_resp" != "204" ] ; then
|
||||
log "Failed to cleanup remote backup. Cannot delete container object $archive_file" "ERROR"
|
||||
cat "$header_file"
|
||||
cat "$resp_file"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
handle_bad_archive_file () {
|
||||
archive_file=$1
|
||||
|
||||
if [ ! -d "$BAD_ARCHIVE_DIR" ]; then
|
||||
mkdir -p "$BAD_ARCHIVE_DIR"
|
||||
fi
|
||||
|
||||
# Move the file to quarantine directory such that
|
||||
# file won't be used for restore in case of recovery
|
||||
#
|
||||
log "Moving $i to $BAD_ARCHIVE_DIR..."
|
||||
mv "$i" "$BAD_ARCHIVE_DIR"
|
||||
log "Removing $i from remote RGW..."
|
||||
if remove_remote_archive_file "$i"; then
|
||||
log "File $i has been successfully removed from RGW."
|
||||
else
|
||||
log "FIle $i cannot be removed form RGW." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Atmost only three bad files are kept. Deleting the oldest if
|
||||
# number of files exceeded the threshold.
|
||||
#
|
||||
bad_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l)
|
||||
if [ "$bad_files" -gt 3 ]; then
|
||||
((bad_files=bad_files-3))
|
||||
delete_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | sort | head --lines=$bad_files)
|
||||
for b in $delete_files; do
|
||||
log "Deleting $b..."
|
||||
rm -f "${b}"
|
||||
done
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
cleanup_old_validation_result_file () {
|
||||
clean_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.passed" 2>/dev/null)
|
||||
for d in $clean_files; do
|
||||
archive_file=${d/.passed}
|
||||
if [ ! -f "$archive_file" ]; then
|
||||
log "Deleting $d as its associated archive file $archive_file nolonger existed."
|
||||
rm -f "${d}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
validate_databases_backup () {
|
||||
archive_file=$1
|
||||
SCOPE=${2:-"all"}
|
||||
|
||||
restore_log='/tmp/restore_error.log'
|
||||
tmp_dir=$(mktemp -d)
|
||||
|
||||
rm -f $restore_log
|
||||
cd "$tmp_dir"
|
||||
log "Decompressing archive $archive_file..."
|
||||
if ! tar zxvf - < "$archive_file" 1>/dev/null; then
|
||||
log "Database restore from local backup failed. Archive decompression failed." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
db_list_file="$tmp_dir/db.list"
|
||||
if [[ -e "$db_list_file" ]]; then
|
||||
dbs=$(sort < "$db_list_file" | grep -ivE sys | tr '\n' ' ')
|
||||
else
|
||||
dbs=" "
|
||||
fi
|
||||
|
||||
sql_file="${tmp_dir}/mariadb.${MARIADB_POD_NAMESPACE}.${SCOPE}.sql"
|
||||
|
||||
if [[ "${SCOPE}" == "all" ]]; then
|
||||
grant_file="${tmp_dir}/grants.sql"
|
||||
else
|
||||
grant_file="${tmp_dir}/${SCOPE}_grant.sql"
|
||||
fi
|
||||
|
||||
if [[ -f $sql_file ]]; then
|
||||
if $MYSQL_LOCAL < "$sql_file" 2>$restore_log; then
|
||||
local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
|
||||
grep -ivE 'information_schema|performance_schema|mysql|sys' | sort | tr '\n' ' ')
|
||||
|
||||
if [ "$dbs" = "$local_dbs" ]; then
|
||||
log "Databases restored successful."
|
||||
else
|
||||
log "Database restore from local backup failed. Database mismatched between local backup and local server" "ERROR"
|
||||
log "Databases restored on local server: $local_dbs" "DEBUG"
|
||||
log "Databases in the local backup: $dbs" "DEBUG"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "Database restore from local backup failed. $dbs" "ERROR"
|
||||
cat $restore_log
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ -f $grant_file ]]; then
|
||||
if $MYSQL_LOCAL < "$grant_file" 2>$restore_log; then
|
||||
if ! $MYSQL_LOCAL -e 'flush privileges;'; then
|
||||
log "Database restore from local backup failed. Failed to flush privileges." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
log "Databases permission restored successful."
|
||||
else
|
||||
log "Database restore from local backup failed. Databases permission failed to restore." "ERROR"
|
||||
cat "$restore_log"
|
||||
cat "$grant_file"
|
||||
log "Local DBs: $local_dbs" "DEBUG"
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "Database restore from local backup failed. There is no permission file available" "ERROR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! check_data_freshness "$archive_file" ${SCOPE}; then
|
||||
# Log has already generated during check data freshness
|
||||
return 1
|
||||
fi
|
||||
else
|
||||
log "Database restore from local backup failed. There is no database file available to restore from" "ERROR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# end of functions form mariadb verifier chart
|
||||
|
||||
# Verify all the databases backup archives
|
||||
verify_databases_backup_archives() {
|
||||
SCOPE=${1:-"all"}
|
||||
|
||||
# verification code
|
||||
export DB_NAME="mariadb"
|
||||
export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive
|
||||
export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine
|
||||
export MYSQL_OPTS="--silent --skip-column-names"
|
||||
export MYSQL_LIVE="mysql --defaults-file=/etc/mysql/admin_user.cnf ${MYSQL_OPTS}"
|
||||
export MYSQL_LOCAL_OPTS="--user=root --host=127.0.0.1"
|
||||
export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2"
|
||||
export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}"
|
||||
export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10"
|
||||
|
||||
max_wait={{ .Values.conf.mariadb_server.setup_wait.iteration }}
|
||||
duration={{ .Values.conf.mariadb_server.setup_wait.duration }}
|
||||
counter=0
|
||||
dbisup=false
|
||||
|
||||
log "Waiting for Mariadb backup verification server to start..."
|
||||
|
||||
# During Mariadb init/startup process, a temporary server is startup
|
||||
# and shutdown prior to starting up the normal server.
|
||||
# To avoid prematurely determine server availability, lets snooze
|
||||
# a bit to give time for the process to complete prior to issue
|
||||
# mysql commands.
|
||||
#
|
||||
|
||||
|
||||
while [ $counter -lt $max_wait ]; do
|
||||
if ! $MYSQL_LOCAL_SHORT -e 'select 1' > /dev/null 2>&1 ; then
|
||||
sleep $duration
|
||||
((counter=counter+1))
|
||||
else
|
||||
# Lets sleep for an additional duration just in case async
|
||||
# init takes a bit more time to complete.
|
||||
#
|
||||
sleep $duration
|
||||
dbisup=true
|
||||
counter=$max_wait
|
||||
fi
|
||||
done
|
||||
|
||||
if ! $dbisup; then
|
||||
log "Mariadb backup verification server is not running" "ERROR"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# During Mariadb init process, a test database will be briefly
|
||||
# created and deleted. Adding to the exclusion list for some
|
||||
# edge cases
|
||||
#
|
||||
clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
|
||||
grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true)
|
||||
|
||||
if [[ -z "${clean_db// }" ]]; then
|
||||
log "Clean Server is up and running"
|
||||
else
|
||||
cleanup_local_databases
|
||||
log "Old databases found on the Mariadb backup verification server were cleaned."
|
||||
clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \
|
||||
grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true)
|
||||
|
||||
if [[ -z "${clean_db// }" ]]; then
|
||||
log "Clean Server is up and running"
|
||||
else
|
||||
log "Cannot clean old databases on verification server." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
log "The server is ready for verification."
|
||||
fi
|
||||
|
||||
# Starting with 10.4.13, new definer mariadb.sys was added. However, mariadb.sys was deleted
|
||||
# during init mariadb as it was not on the exclusion list. This corrupted the view of mysql.user.
|
||||
# Insert the tuple back to avoid other similar issues with error i.e
|
||||
# The user specified as a definer ('mariadb.sys'@'localhost') does not exist
|
||||
#
|
||||
# Before insert the tuple mentioned above, we should make sure that the MariaDB version is 10.4.+
|
||||
mariadb_version=$($MYSQL_LOCAL_SHORT -e "status" | grep -E '^Server\s+version:')
|
||||
log "Current database ${mariadb_version}"
|
||||
if [[ ! -z ${mariadb_version} && -z $(grep '10.2' <<< ${mariadb_version}}) ]]; then
|
||||
if [[ -z $(grep 'mariadb.sys' <<< $($MYSQL_LOCAL_SHORT mysql -e "select * from global_priv where user='mariadb.sys'")) ]]; then
|
||||
$MYSQL_LOCAL_SHORT -e "insert into mysql.global_priv values ('localhost','mariadb.sys',\
|
||||
'{\"access\":0,\"plugin\":\"mysql_native_password\",\"authentication_string\":\"\",\"account_locked\":true,\"password_last_changed\":0}');"
|
||||
$MYSQL_LOCAL_SHORT -e 'flush privileges;'
|
||||
fi
|
||||
fi
|
||||
|
||||
# Ensure archive dir existed
|
||||
if [ -d "$ARCHIVE_DIR" ]; then
|
||||
# List archive dir before
|
||||
list_archive_dir
|
||||
|
||||
# Ensure the local databases are clean for each restore validation
|
||||
#
|
||||
cleanup_local_databases
|
||||
|
||||
if [[ "${SCOPE}" == "all" ]]; then
|
||||
archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | sort)
|
||||
for i in $archive_files; do
|
||||
archive_file_passed=$i.passed
|
||||
if [ ! -f "$archive_file_passed" ]; then
|
||||
log "Validating archive file $i..."
|
||||
if validate_databases_backup "$i"; then
|
||||
touch "$archive_file_passed"
|
||||
else
|
||||
if handle_bad_archive_file "$i"; then
|
||||
log "File $i has been removed from RGW."
|
||||
else
|
||||
log "File $i cannot be removed from RGW." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
else
|
||||
archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | grep "${SCOPE}" | sort)
|
||||
for i in $archive_files; do
|
||||
archive_file_passed=$i.passed
|
||||
if [ ! -f "$archive_file_passed" ]; then
|
||||
log "Validating archive file $i..."
|
||||
if validate_databases_backup "${i}" "${SCOPE}"; then
|
||||
touch "$archive_file_passed"
|
||||
else
|
||||
if handle_bad_archive_file "$i"; then
|
||||
log "File $i has been removed from RGW."
|
||||
else
|
||||
log "File $i cannot be removed from RGW." "ERROR"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
|
||||
# Cleanup passed files if its archive file nolonger existed
|
||||
cleanup_old_validation_result_file
|
||||
|
||||
# List archive dir after
|
||||
list_archive_dir
|
||||
fi
|
||||
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# Call main program to start the database backup
|
||||
backup_databases ${SCOPE}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash -ex
|
||||
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
log () {
|
||||
msg_default="Need some text to log"
|
||||
level_default="INFO"
|
||||
component_default="Mariadb Backup Verifier"
|
||||
|
||||
msg=${1:-$msg_default}
|
||||
level=${2:-$level_default}
|
||||
component=${3:-"$component_default"}
|
||||
|
||||
echo "$(date +'%Y-%m-%d %H:%M:%S,%3N') - ${component} - ${level} - ${msg}"
|
||||
}
|
||||
|
||||
log "Starting Mariadb server for backup verification..."
|
||||
MYSQL_ALLOW_EMPTY_PASSWORD=1 nohup bash -x docker-entrypoint.sh mysqld --user=nobody 2>&1
|
||||
|
|
@ -38,6 +38,8 @@ data:
|
|||
{{- if .Values.conf.backup.enabled }}
|
||||
backup_mariadb.sh: |
|
||||
{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
start_verification_server.sh: |
|
||||
{{ tuple "bin/_start_mariadb_verify_server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
restore_mariadb.sh: |
|
||||
{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
|
||||
backup_main.sh: |
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ spec:
|
|||
{{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
|
||||
serviceAccountName: {{ $serviceAccountName }}
|
||||
restartPolicy: OnFailure
|
||||
shareProcessNamespace: true
|
||||
{{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
|
||||
{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
|
||||
{{ end }}
|
||||
|
|
@ -76,10 +77,29 @@ spec:
|
|||
name: pod-tmp
|
||||
- mountPath: {{ .Values.conf.backup.base_path }}
|
||||
name: mariadb-backup-dir
|
||||
- name: verify-perms
|
||||
{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
|
||||
{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
|
||||
command:
|
||||
- chown
|
||||
- -R
|
||||
- "65534:65534"
|
||||
- /var/lib/mysql
|
||||
volumeMounts:
|
||||
- mountPath: /tmp
|
||||
name: pod-tmp
|
||||
- mountPath: /var/lib/mysql
|
||||
name: mysql-data
|
||||
containers:
|
||||
- name: mariadb-backup
|
||||
command:
|
||||
- /tmp/backup_mariadb.sh
|
||||
- /bin/sh
|
||||
args:
|
||||
- -c
|
||||
- >-
|
||||
/tmp/backup_mariadb.sh;
|
||||
/usr/bin/pkill mysqld
|
||||
env:
|
||||
- name: MARIADB_BACKUP_BASE_DIR
|
||||
value: {{ .Values.conf.backup.base_path | quote }}
|
||||
|
|
@ -131,12 +151,62 @@ spec:
|
|||
subPath: admin_user.cnf
|
||||
readOnly: true
|
||||
{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }}
|
||||
- name: mariadb-verify-server
|
||||
{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 14 }}
|
||||
{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_verify_server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
|
||||
env:
|
||||
{{- if $envAll.Values.manifests.certificates }}
|
||||
- name: MARIADB_X509
|
||||
value: "REQUIRE X509"
|
||||
{{- end }}
|
||||
- name: MYSQL_HISTFILE
|
||||
value: /dev/null
|
||||
- name: MARIADB_BACKUP_BASE_DIR
|
||||
value: {{ .Values.conf.backup.base_path | quote }}
|
||||
ports:
|
||||
- name: mysql
|
||||
protocol: TCP
|
||||
containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
|
||||
command:
|
||||
- /tmp/start_verification_server.sh
|
||||
volumeMounts:
|
||||
- name: pod-tmp
|
||||
mountPath: /tmp
|
||||
- name: var-run
|
||||
mountPath: /var/run/mysqld
|
||||
- name: mycnfd
|
||||
mountPath: /etc/mysql/conf.d
|
||||
- name: mariadb-etc
|
||||
mountPath: /etc/mysql/my.cnf
|
||||
subPath: my.cnf
|
||||
readOnly: true
|
||||
- name: mariadb-secrets
|
||||
mountPath: /etc/mysql/admin_user.cnf
|
||||
subPath: admin_user.cnf
|
||||
readOnly: true
|
||||
- name: mysql-data
|
||||
mountPath: /var/lib/mysql
|
||||
- name: mariadb-bin
|
||||
mountPath: /tmp/start_verification_server.sh
|
||||
readOnly: true
|
||||
subPath: start_verification_server.sh
|
||||
restartPolicy: OnFailure
|
||||
serviceAccount: {{ $serviceAccountName }}
|
||||
serviceAccountName: {{ $serviceAccountName }}
|
||||
volumes:
|
||||
- name: pod-tmp
|
||||
emptyDir: {}
|
||||
- name: mycnfd
|
||||
emptyDir: {}
|
||||
- name: var-run
|
||||
emptyDir: {}
|
||||
- name: mariadb-etc
|
||||
configMap:
|
||||
name: mariadb-etc
|
||||
defaultMode: 0444
|
||||
- name: mysql-data
|
||||
emptyDir: {}
|
||||
- name: mariadb-secrets
|
||||
secret:
|
||||
secretName: mariadb-secrets
|
||||
|
|
|
|||
|
|
@ -122,10 +122,17 @@ pod:
|
|||
backup_perms:
|
||||
runAsUser: 0
|
||||
readOnlyRootFilesystem: true
|
||||
verify_perms:
|
||||
runAsUser: 0
|
||||
readOnlyRootFilesystem: true
|
||||
mariadb_backup:
|
||||
runAsUser: 65534
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
mariadb_verify_server:
|
||||
runAsUser: 65534
|
||||
readOnlyRootFilesystem: true
|
||||
allowPrivilegeEscalation: false
|
||||
tests:
|
||||
pod:
|
||||
runAsUser: 999
|
||||
|
|
@ -328,9 +335,15 @@ conf:
|
|||
ingress_conf:
|
||||
worker-processes: "auto"
|
||||
log-format-stream: "\"$remote_addr [$time_local] $protocol $status $bytes_received $bytes_sent $upstream_addr $upstream_connect_time $upstream_first_byte_time $upstream_session_time $session_time\""
|
||||
mariadb_server:
|
||||
setup_wait:
|
||||
iteration: 30
|
||||
duration: 5
|
||||
backup:
|
||||
enabled: false
|
||||
base_path: /var/backup
|
||||
validateData:
|
||||
ageOffset: 120
|
||||
mysqldump_options: >
|
||||
--single-transaction --quick --add-drop-database
|
||||
--add-drop-table --add-locks --databases
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ pod:
|
|||
mariadb-backup:
|
||||
init: runtime/default
|
||||
mariadb-backup: runtime/default
|
||||
mariadb-verify-server: runtime/default
|
||||
mariadb-test:
|
||||
init: runtime/default
|
||||
mariadb-test: runtime/default
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ apiVersion: v1
|
|||
appVersion: v9.6
|
||||
description: OpenStack-Helm PostgreSQL
|
||||
name: postgresql
|
||||
version: 0.1.16
|
||||
version: 0.1.17
|
||||
home: https://www.postgresql.org
|
||||
sources:
|
||||
- https://github.com/postgres/postgres
|
||||
|
|
|
|||
|
|
@ -82,5 +82,13 @@ dump_databases_to_directory() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Verify all the databases backup archives
|
||||
verify_databases_backup_archives() {
|
||||
####################################
|
||||
# TODO: add implementation of local backup verification
|
||||
####################################
|
||||
return 0
|
||||
}
|
||||
|
||||
# Call main program to start the database backup
|
||||
backup_databases ${SCOPE}
|
||||
|
|
|
|||
|
|
@ -54,4 +54,5 @@ helm-toolkit:
|
|||
- 0.2.45 Modify use_external_ingress_controller place in openstack-helm values.yaml
|
||||
- 0.2.46 Fixed for getting kibana ingress value parameters
|
||||
- 0.2.47 Adjusting of kibana ingress value parameters
|
||||
- 0.2.48 Added verify_databases_backup_archives function call to backup process and added remote backup sha256 hash verification
|
||||
...
|
||||
|
|
|
|||
|
|
@ -43,4 +43,5 @@ mariadb:
|
|||
- 0.2.25 Add liveness probe to restart a pod that got stuck in a transfer wsrep_local_state_comment
|
||||
- 0.2.26 Added OCI registry authentication
|
||||
- 0.2.27 Fix broken helmrelease for helmv3
|
||||
- 0.2.28 Added verify_databases_backup_in_directory function implementation
|
||||
...
|
||||
|
|
|
|||
|
|
@ -17,4 +17,5 @@ postgresql:
|
|||
- 0.1.14 Fix invalid fields in values
|
||||
- 0.1.15 Migrated CronJob resource to batch/v1 API version
|
||||
- 0.1.16 Added OCI registry authentication
|
||||
- 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit
|
||||
...
|
||||
|
|
|
|||
Loading…
Reference in New Issue