fix(update): stop a second update running while one is in progress (#5095)

Adds check_update_lock.sh, which stops a second update, force-update, validate or workshop-update from running while one is in progress. The lock is removed in core_exit.sh, and a lock left by a dead or reused PID is treated as stale.

Also fixes the SteamCMD retry loop: a failing base app no longer retries forever without an error, and the SteamCMD log is cleared before each attempt so a successful retry is not reported as failed.

Fixes #5094

lgsm/modules/check_update_lock.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# LinuxGSM check_update_lock.sh module
+# Author: Daniel Gibbs
+# Contributors: https://linuxgsm.com/contrib
+# Website: https://linuxgsm.com
+# Description: Stops an update or validate from running while another one is running.
+
+moduleselfname="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+updatinglock="${lockdir}/updating.lock"
+
+fn_update_lock_create() {
+	(
+		set -o noclobber
+		echo "$$" > "${updatinglock}"
+	) 2> /dev/null
+}
+
+# Checks the PID in the lockfile is running and started before the lockfile was written.
+fn_update_lock_running() {
+	local lockpid="${1}"
+	local lockage processage
+	if [ -z "${lockpid}" ] || ! kill -0 "${lockpid}" 2> /dev/null; then
+		return 1
+	fi
+	lockage="$(($(date '+%s') - $(stat -c %Y "${updatinglock}" 2> /dev/null || echo 0)))"
+	processage="$(ps -o etimes= -p "${lockpid}" 2> /dev/null | tr -d ' ')"
+	[ -n "${processage}" ] && [ "$((processage + 2))" -ge "${lockage}" ]
+}
+
+if ! fn_update_lock_create; then
+	updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+	if [ -z "${updatelockpid}" ]; then
+		sleep 1
+		updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+	fi
+	if [ "${updatelockpid}" == "$$" ] && [ -n "${updatelocklevel}" ]; then
+		return
+	fi
+	if [ "${updatelockpid}" == "$$" ] || ! fn_update_lock_running "${updatelockpid}"; then
+		fn_script_log_warn "Checking update: Removing stale lockfile from PID ${updatelockpid:-unknown}"
+		rm -f "${updatinglock:?}"
+		fn_update_lock_create
+	fi
+fi
+
+updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+if [ "${updatelockpid}" != "$$" ]; then
+	fn_print_dots "Checking update"
+	fn_print_checking_eol
+	fn_print_info "Checking update: Update is already running (PID ${updatelockpid})"
+	fn_print_info_eol_nl
+	fn_script_log_info "Checking update: Update is already running (PID ${updatelockpid})"
+	# shellcheck disable=SC2034 # Used by core_exit.sh.
+	exitcode=0
+	unset exitbypass
+	core_exit.sh
+fi
+updatelocklevel="${BASH_SUBSHELL}"

lgsm/modules/command_update.sh

@@ -49,6 +49,7 @@ fn_update_apply() {
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 # A postponed update is requested again if players are still online.
 rm -f "${lockdir:?}/${selfname}-update-request.lock"

lgsm/modules/command_validate.sh

@@ -30,6 +30,7 @@ fn_validate() {
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 fn_print_dots "SteamCMD"
 

lgsm/modules/command_workshop_update.sh

@@ -13,6 +13,7 @@ fn_firstcommand_set
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 if [ -z "${workshopmods}" ] && [ ! -f "${datadir}/${selfname}-workshop.txt" ]; then
 	fn_print_info_nl "No Workshop mods set. Add Workshop IDs to workshopmods in ${configdirserver}"

lgsm/modules/core_dl.sh

@@ -75,14 +75,12 @@ fn_dl_steamcmd() {
 	# To do error checking for SteamCMD the output of steamcmd will be saved to a log.
 	steamcmdlog="${lgsmlogdir}/${selfname}-steamcmd.log"
 
-	# clear previous steamcmd log
-	if [ -f "${steamcmdlog}" ]; then
-		rm -f "${steamcmdlog:?}"
-	fi
 	local invalidplatformworkaround=""
 	counter=0
 	while [ "${counter}" -eq 0 ] || [ "${exitcode}" -ne 0 ]; do
 		counter=$((counter + 1))
+		# clear previous steamcmd log
+		rm -f "${steamcmdlog:?}"
 		# Select SteamCMD parameters
 		# If GoldSrc (appid 90) servers. GoldSrc (appid 90) require extra commands.
 		if [ "${appid}" == "90" ]; then
@@ -108,19 +106,18 @@ fn_dl_steamcmd() {
 				fi
 				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${supportdir}" +login "${steamuser}" "${steampass}" +app_update "${baseappid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
 				exitcode="${PIPESTATUS[0]}"
-				if [ "${exitcode}" -ne 0 ]; then
-					continue
-				fi
 			fi
-			if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			elif [ -n "${branch}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			else
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
+			if [ -z "${baseappid}" ] || [ "${exitcode}" -eq 0 ]; then
+				if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				elif [ -n "${branch}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				else
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				fi
 			fi
 		# All other servers.
 		else
@@ -133,25 +130,23 @@ fn_dl_steamcmd() {
 				fi
 				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${supportdir}" +login "${steamuser}" "${steampass}" +app_update "${baseappid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
 				exitcode="${PIPESTATUS[0]}"
-				if [ "${exitcode}" -ne 0 ]; then
-					continue
-				fi
 			fi
-			if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			elif [ -n "${branch}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			else
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
+			if [ -z "${baseappid}" ] || [ "${exitcode}" -eq 0 ]; then
+				if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				elif [ -n "${branch}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				else
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				fi
 			fi
 		fi
 
 		# Error checking for SteamCMD. Some errors will loop to try again and some will just exit.
-		# Check also if we have more errors than retries to be sure that we do not loop to many times and error out.
-		if [ -n "$(grep -i "Error!" "${steamcmdlog}" | tail -1)" ] && [ "$(grep -ic "Error!" "${steamcmdlog}")" -ge "${counter}" ]; then
+		if grep -qi "Error!" "${steamcmdlog}"; then
 			# Not enough space.
 			if [ -n "$(grep "0x202" "${steamcmdlog}" | tail -1)" ]; then
 				fn_print_failure_nl "${commandaction} ${selfname}: ${remotelocation}: Not enough disk space to download server files"

lgsm/modules/core_exit.sh

@@ -27,6 +27,10 @@ if [ "$(whoami)" == "root" ]; then
 	find "${logdir}"/ -group root -prune -exec rm -rf {} + > /dev/null 2>&1
 fi
 
+if [ -z "${exitbypass}" ] && [ "${updatelocklevel}" == "${BASH_SUBSHELL}" ] && [ "$(cat "${lockdir}/updating.lock" 2> /dev/null)" == "$$" ]; then
+	rm -f "${lockdir:?}/updating.lock"
+fi
+
 if [ "${exitbypass}" ]; then
 	unset exitbypass
 elif [ "${exitcode}" -ne 0 ]; then

lgsm/modules/core_modules.sh

@@ -278,6 +278,11 @@ check_tmuxception.sh() {
 	fn_fetch_module
 }
 
+check_update_lock.sh() {
+	modulefile="${FUNCNAME[0]}"
+	fn_fetch_module
+}
+
 check_version.sh() {
 	modulefile="${FUNCNAME[0]}"
 	fn_fetch_module