fix(update): stop a second update running while one is in progress

A second update, force-update, validate or workshop-update started while
another was running (for example the docker entrypoint update and the cron
update) ran SteamCMD into the same serverfiles at the same time and broke
both downloads. The new check_update_lock.sh writes the PID to
lock/updating.lock with noclobber, and a second run prints "Update is already
running" and exits 0. A lock whose PID is dead, or was reused by a process
started after the lock was written, is removed. core_exit.sh removes the lock
when the owning command exits.

fn_dl_steamcmd fixes:
- A failing baseappid download ran continue, which skipped the error report
  and the retry limit, so it retried forever without a message.
- The SteamCMD log was only cleared before the loop and every Error! in it
  was counted against the attempt counter. After an attempt with two
  different errors, a later successful attempt still reported "Update
  required but not completed" and SteamCMD ran again. The log is now cleared
  before every attempt.

lgsm/modules/check_update_lock.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# LinuxGSM check_update_lock.sh module
+# Author: Daniel Gibbs
+# Contributors: https://linuxgsm.com/contrib
+# Website: https://linuxgsm.com
+# Description: Stops an update or validate from running while another one is running.
+
+moduleselfname="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
+
+updatinglock="${lockdir}/updating.lock"
+
+fn_update_lock_create() {
+	(
+		set -o noclobber
+		echo "$$" > "${updatinglock}"
+	) 2> /dev/null
+}
+
+# Checks the PID in the lockfile is running and started before the lockfile was written.
+fn_update_lock_running() {
+	local lockpid="${1}"
+	local lockage processage
+	if [ -z "${lockpid}" ] || ! kill -0 "${lockpid}" 2> /dev/null; then
+		return 1
+	fi
+	lockage="$(($(date '+%s') - $(stat -c %Y "${updatinglock}" 2> /dev/null || echo 0)))"
+	processage="$(ps -o etimes= -p "${lockpid}" 2> /dev/null | tr -d ' ')"
+	[ -n "${processage}" ] && [ "$((processage + 2))" -ge "${lockage}" ]
+}
+
+if ! fn_update_lock_create; then
+	updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+	if [ -z "${updatelockpid}" ]; then
+		sleep 1
+		updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+	fi
+	if [ "${updatelockpid}" == "$$" ] && [ -n "${updatelocklevel}" ]; then
+		return
+	fi
+	if [ "${updatelockpid}" == "$$" ] || ! fn_update_lock_running "${updatelockpid}"; then
+		fn_script_log_warn "Checking update: Removing stale lockfile from PID ${updatelockpid:-unknown}"
+		rm -f "${updatinglock:?}"
+		fn_update_lock_create
+	fi
+fi
+
+updatelockpid="$(cat "${updatinglock}" 2> /dev/null)"
+if [ "${updatelockpid}" != "$$" ]; then
+	fn_print_dots "Checking update"
+	fn_print_checking_eol
+	fn_print_info "Checking update: Update is already running (PID ${updatelockpid})"
+	fn_print_info_eol_nl
+	fn_script_log_info "Checking update: Update is already running (PID ${updatelockpid})"
+	# shellcheck disable=SC2034 # Used by core_exit.sh.
+	exitcode=0
+	unset exitbypass
+	core_exit.sh
+fi
+updatelocklevel="${BASH_SUBSHELL}"

lgsm/modules/command_update.sh

@@ -49,6 +49,7 @@ fn_update_apply() {
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 # A postponed update is requested again if players are still online.
 rm -f "${lockdir:?}/${selfname}-update-request.lock"

lgsm/modules/command_validate.sh

@@ -30,6 +30,7 @@ fn_validate() {
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 fn_print_dots "SteamCMD"
 

lgsm/modules/command_workshop_update.sh

@@ -13,6 +13,7 @@ fn_firstcommand_set
 fn_print_dots ""
 check.sh
 core_logs.sh
+check_update_lock.sh
 
 if [ -z "${workshopmods}" ] && [ ! -f "${datadir}/${selfname}-workshop.txt" ]; then
 	fn_print_info_nl "No Workshop mods set. Add Workshop IDs to workshopmods in ${configdirserver}"

lgsm/modules/core_dl.sh

@@ -75,14 +75,12 @@ fn_dl_steamcmd() {
 	# To do error checking for SteamCMD the output of steamcmd will be saved to a log.
 	steamcmdlog="${lgsmlogdir}/${selfname}-steamcmd.log"
 
-	# clear previous steamcmd log
-	if [ -f "${steamcmdlog}" ]; then
-		rm -f "${steamcmdlog:?}"
-	fi
 	local invalidplatformworkaround=""
 	counter=0
 	while [ "${counter}" -eq 0 ] || [ "${exitcode}" -ne 0 ]; do
 		counter=$((counter + 1))
+		# clear previous steamcmd log
+		rm -f "${steamcmdlog:?}"
 		# Select SteamCMD parameters
 		# If GoldSrc (appid 90) servers. GoldSrc (appid 90) require extra commands.
 		if [ "${appid}" == "90" ]; then
@@ -108,19 +106,18 @@ fn_dl_steamcmd() {
 				fi
 				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${supportdir}" +login "${steamuser}" "${steampass}" +app_update "${baseappid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
 				exitcode="${PIPESTATUS[0]}"
-				if [ "${exitcode}" -ne 0 ]; then
-					continue
-				fi
 			fi
-			if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			elif [ -n "${branch}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			else
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
+			if [ -z "${baseappid}" ] || [ "${exitcode}" -eq 0 ]; then
+				if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				elif [ -n "${branch}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				else
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +@sSteamCmdForcePlatformType windows +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				fi
 			fi
 		# All other servers.
 		else
@@ -133,25 +130,23 @@ fn_dl_steamcmd() {
 				fi
 				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${supportdir}" +login "${steamuser}" "${steampass}" +app_update "${baseappid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
 				exitcode="${PIPESTATUS[0]}"
-				if [ "${exitcode}" -ne 0 ]; then
-					continue
-				fi
 			fi
-			if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			elif [ -n "${branch}" ]; then
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
-			else
-				"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
-				exitcode="${PIPESTATUS[0]}"
+			if [ -z "${baseappid}" ] || [ "${exitcode}" -eq 0 ]; then
+				if [ -n "${branch}" ] && [ -n "${betapassword}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" -betapassword "${betapassword}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				elif [ -n "${branch}" ]; then
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" -beta "${branch}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				else
+					"${unbuffercommand[@]}" "${steamcmdcommandarray[@]}" +force_install_dir "${serverfiles}" +login "${steamuser}" "${steampass}" +app_update "${appid}" "${validateparam[@]}" +quit | fn_dl_steamcmd_output | tee -a "${lgsmlog}" "${steamcmdlog}"
+					exitcode="${PIPESTATUS[0]}"
+				fi
 			fi
 		fi
 
 		# Error checking for SteamCMD. Some errors will loop to try again and some will just exit.
-		# Check also if we have more errors than retries to be sure that we do not loop to many times and error out.
-		if [ -n "$(grep -i "Error!" "${steamcmdlog}" | tail -1)" ] && [ "$(grep -ic "Error!" "${steamcmdlog}")" -ge "${counter}" ]; then
+		if grep -qi "Error!" "${steamcmdlog}"; then
 			# Not enough space.
 			if [ -n "$(grep "0x202" "${steamcmdlog}" | tail -1)" ]; then
 				fn_print_failure_nl "${commandaction} ${selfname}: ${remotelocation}: Not enough disk space to download server files"

lgsm/modules/core_exit.sh

@@ -25,6 +25,10 @@ if [ "$(whoami)" == "root" ]; then
 	find "${logdir}"/ -group root -prune -exec rm -rf {} + > /dev/null 2>&1
 fi
 
+if [ -z "${exitbypass}" ] && [ "${updatelocklevel}" == "${BASH_SUBSHELL}" ] && [ "$(cat "${lockdir}/updating.lock" 2> /dev/null)" == "$$" ]; then
+	rm -f "${lockdir:?}/updating.lock"
+fi
+
 if [ "${exitbypass}" ]; then
 	unset exitbypass
 elif [ "${exitcode}" -ne 0 ]; then

lgsm/modules/core_modules.sh

@@ -273,6 +273,11 @@ check_tmuxception.sh() {
 	fn_fetch_module
 }
 
+check_update_lock.sh() {
+	modulefile="${FUNCNAME[0]}"
+	fn_fetch_module
+}
+
 check_version.sh() {
 	modulefile="${FUNCNAME[0]}"
 	fn_fetch_module