feat(backup): run backups postponed by players once the server is empty (#5029)

* feat(stop): warn players in-game before stop, restart and update, and retry postponed updates

* fix(stop): exit when players prevent a stop during update or backup, keep module modes

* fix(stop): mark variables shared between modules for shellcheck

* refactor(stop): shorten check_players_online module description

* refactor(stop): keep chat commands out of the config, clean up locks when a stop is cancelled, exit on CTRL-C

* feat(backup): run backups postponed by players once the server is empty, and keep long backups' lockfile

* feat(stop): say why the player count is unavailable

* fix(monitor): run postponed updates in the background, keep a long update's lockfile, retry forced updates as forced

* fix(monitor): run postponed backups in the background

* fix(monitor): mark exitcode as used by core_exit for shellcheck

* fix(monitor): mark exitcode as used by core_exit for shellcheck

lgsm/modules/alert.sh

@@ -169,6 +169,17 @@ fn_alert_backup() {
 	alertcolourdec="52480"
 }
 
+fn_alert_backup_postponed() {
+	fn_script_log_warn "Sending alert: ${selfname} backup postponed for over 24 hours"
+	alertaction="Backup Postponed"
+	alertemoji="⏳"
+	alertsound="1"
+	alertmessage="${selfname} has not been backed up for over 24 hours as players are online. The backup will run once the server is empty."
+	# Amber
+	alertcolourhex="#cd8b00"
+	alertcolourdec="13470464"
+}
+
 fn_alert_hytale_login() {
 	fn_script_log_info "Sending alert: ${selfname} needs a Hytale server login"
 	alertaction="Login Required"
@@ -261,6 +272,8 @@ elif [ "${alert}" == "update-linuxgsm" ]; then
 	fn_alert_update_linuxgsm
 elif [ "${alert}" == "backup" ]; then
 	fn_alert_backup
+elif [ "${alert}" == "backup-postponed" ]; then
+	fn_alert_backup_postponed
 elif [ "${alert}" == "hytale-login" ]; then
 	fn_alert_hytale_login
 else

lgsm/modules/command_backup.sh

@@ -32,8 +32,10 @@ fn_backup_trap() {
 
 # Check if a backup is pending or has been aborted using backup.lock.
 fn_backup_check_lockfile() {
-	# Remove stale lockfile.
-	if [ -f "${lockdir}/backup.lock" ]; then
+	# Remove stale lockfile, unless a long backup is still running.
+	local backupsrunning
+	backupsrunning="$(pgrep -fc -u "$(id -u)" "^([^ ]*/)?bash ([^ ]*/)?${selfname} (backup|b)( |$)")"
+	if [ -f "${lockdir}/backup.lock" ] && [ "${backupsrunning:-0}" -le 1 ]; then
 		if [ "$(find "${lockdir}/backup.lock" -mmin +60)" ]; then
 			fn_print_dots "Lockfile found: "
 			fn_print_checking_eol
@@ -167,6 +169,8 @@ fn_backup_compression() {
 		fn_script_log_pass "Backup created: ${backupname}.${compressext}, total size $(du -sh "${backupdir}/${backupname}.${compressext}" | awk '{print $1}')"
 		alert="backup"
 		alert.sh
+		# A postponed backup is complete.
+		rm -f "${lockdir:?}/${selfname}-backup-request.lock"
 	fi
 }
 

lgsm/modules/command_monitor.sh

@@ -166,9 +166,42 @@ fn_monitor_check_update_request() {
 	fi
 }
 
+# A backup postponed because players were online runs once the server is empty, in the background
+# like a requested update. An alert is sent once if it has been postponed for over 24 hours.
+fn_monitor_check_backup_request() {
+	local backuprequest="${lockdir}/${selfname}-backup-request.lock"
+	if [ -f "${backuprequest}" ]; then
+		fn_print_dots "Checking backup: "
+		fn_print_checking_eol
+		fn_print_info "Checking backup: Backup requested: "
+		fn_print_info_eol_nl
+		fn_script_log_info "Checking backup: Backup requested"
+		check_players_online.sh
+		if [ -n "${playersonline}" ]; then
+			fn_print_info_nl "${playersonline} players are on the server: backup postponed"
+			fn_script_log_info "${playersonline} players are on the server: backup postponed"
+			echo "${playersonline}" > "${lockdir:?}/${selfname}-player-numbers.lock"
+			local requested
+			requested="$(head -n 1 "${backuprequest}")"
+			if [[ "${requested}" =~ ^[0-9]+$ ]] && [ "$(($(date '+%s') - requested))" -gt 86400 ] && ! grep -q "alerted" "${backuprequest}"; then
+				alert="backup-postponed"
+				alert.sh
+				echo "alerted" >> "${backuprequest}"
+			fi
+		else
+			fn_print_info_nl "Checking backup: Running requested backup"
+			fn_script_log_info "Checking backup: Running requested backup"
+			setsid "${rootdir}/${selfname}" backup > /dev/null 2>&1 < /dev/null &
+			# shellcheck disable=SC2034 # Used by core_exit.sh.
+			exitcode=0
+			core_exit.sh
+		fi
+	fi
+}
+
 fn_monitor_check_backup() {
-	# Remove stale lockfile.
-	if [ -f "${lockdir}/backup.lock" ]; then
+	# Remove stale lockfile, unless a long backup is still running.
+	if [ -f "${lockdir}/backup.lock" ] && ! fn_monitor_command_running "backup|b"; then
 		if [ "$(find "${lockdir}/backup.lock" -mmin +60)" ]; then
 			fn_print_dots "Checking backup"
 			fn_print_checking_eol
@@ -464,6 +497,7 @@ fn_monitor_check_stopping
 fn_monitor_check_session
 fn_monitor_check_restart_request
 fn_monitor_check_update_request
+fn_monitor_check_backup_request
 
 # Monitor will not continue if session only check.
 if [ "${querymode}" != "1" ]; then

lgsm/modules/command_stop.sh

@@ -29,6 +29,12 @@ fn_stop_players_online() {
 				fi
 				fn_print_info_nl "Update postponed until the server is empty"
 				fn_script_log_info "Update postponed until the server is empty"
+			elif [ "${firstcommandname}" == "BACKUP" ]; then
+				if [ ! -f "${lockdir}/${selfname}-backup-request.lock" ]; then
+					date '+%s' > "${lockdir:?}/${selfname}-backup-request.lock"
+				fi
+				fn_print_info_nl "Backup postponed until the server is empty"
+				fn_script_log_info "Backup postponed until the server is empty"
 			fi
 			fn_stop_cancel_locks
 			# Exit the whole command, including when called by another command (e.g. update) with exitbypass.