Add flock in wipe_osds.sh to avoid race condition

When running the script to wipe the OSDs, it sometimes happened
that the second partition was not found, although it was there.

Analyzing the code, it was possible to replicate the problem,
which is caused by a race condition when running the parted
command on the first partition, which causes udev to be reloaded
while the second partition was being processed.

To solve this problem, the “flock” command was used to lock the
entire disk, not just the partition.

Additionally, the use of udevadm has also been removed.

Test Plan:
  - PASS: (AIO-SX) Replace wipe_osds.sh with changes,
          run and check script output

Closes-Bug: 2056765

Change-Id: Icbc351a868b413a51dc8273ca422737d39756b3b
Signed-off-by: Erickson Silva de Oliveira <Erickson.SilvadeOliveira@windriver.com>
This commit is contained in:
Erickson Silva de Oliveira 2024-03-11 11:13:39 -03:00
parent fc5e4aac46
commit e63bfd683e
1 changed files with 46 additions and 11 deletions

View File

@ -1,6 +1,6 @@
#!/bin/sh
#!/bin/bash
#
# Copyright (c) 2020, 2023 Wind River Systems, Inc.
# Copyright (c) 2020, 2023-2024 Wind River Systems, Inc.
#
# SPDX-License-Identifier: Apache-2.0
#
@ -13,7 +13,9 @@
#
# Upstream reports of this:
# - https://github.com/ansible/ansible/issues/70092
sleep 2
if [ -z "${NO_WORKAROUND}" ]; then
sleep 2
fi
CEPH_REGULAR_OSD_GUID="4FBD7E29-9D25-41B8-AFD0-062C0CEFF05D"
CEPH_REGULAR_JOURNAL_GUID="45B0969E-9B03-4F30-B4C6-B4B80CEFF106"
@ -23,7 +25,31 @@ CEPH_MPATH_JOURNAL_GUID="45B0969E-8AE0-4982-BF9D-5A8D867AF560"
# This script is not supposed to fail, print extended logs from ansible if it does.
set -x
wipe_if_ceph_disk() {
unlock_dev() {
local __lock_fd="${lock_fd}"
if [ -z "${__lock_fd}" ]; then
return
fi
flock -u "${__lock_fd}"
exec {__lock_fd}>&-
lock_fd=
trap "" EXIT
}
lock_dev() {
local __dev="${1}"
declare -g lock_fd=
trap "unlock_dev" EXIT
exec {lock_fd}>"${__dev}"
flock -x "${lock_fd}"
}
__wipe_if_ceph_disk() {
__dev="$1"
__osd_guid="$2"
__journal_guid="$3"
@ -31,14 +57,11 @@ wipe_if_ceph_disk() {
ceph_disk="false"
# Based on past experience observing race conditions with udev and updates
# to partition data/metadata we will continue to use flock in accordance
# with: https://systemd.io/BLOCK_DEVICE_LOCKING/
for part in $(flock "${__dev}" sfdisk -q -l "${__dev}" | \
for part in $(sfdisk -q -l "${__dev}" | \
awk '$1 == "Device" {i=1; next}; i {print $1}'); do
part_no=$(udevadm info "$part" | \
grep -oP -m1 'E: PARTN=\K.*|E: DM_PART=\K.*')
guid=$(flock "${__dev}" sfdisk --part-type "$__dev" "$part_no")
part_no="$(echo "${part}" | sed -n -e 's@^.*[^0-9]\([0-9]\+\)$@\1@p')"
guid=$(sfdisk --part-type "$__dev" "$part_no")
if [ "${guid}" = "$__osd_guid" ]; then
echo "Found Ceph OSD partition #${part_no} ${part}, erasing!"
dd if=/dev/zero of="${part}" bs=512 count=34 2>/dev/null
@ -84,7 +107,19 @@ wipe_if_ceph_disk() {
fi
wipefs -a "${__dev}"
fi
}
wipe_if_ceph_disk() {
local __dev="$1"
# Based on past experience observing race conditions with udev and updates
# to partition data/metadata we will continue to use flock in accordance
# with: https://systemd.io/BLOCK_DEVICE_LOCKING/
lock_dev "${__dev}"
__wipe_if_ceph_disk "${@}"
unlock_dev
}
# Only non multipath