LinuxGSM/lgsm/modules/command_monitor.sh

#!/bin/bash
# LinuxGSM command_monitor.sh module
# Author: Daniel Gibbs
# Contributors: https://linuxgsm.com/contrib
# Website: https://linuxgsm.com
# Description: Monitors server by checking for running processes
# then passes to gamedig and gsquery.

commandname="MONITOR"
commandaction="Monitoring"
moduleselfname="$(basename "$(readlink -f "${BASH_SOURCE[0]}")")"
fn_firstcommand_set

# Checks if this instance is running one of the given commands, e.g. "update|u".
# Matches the script run as ./gameserver, /path/to/gameserver or bash gameserver.
fn_monitor_command_running() {
	[ "$(pgrep -fc -u "${USER}" "^([^ ]*/)?bash ([^ ]*/)?${selfname} ($1)( |$)")" != "0" ]
}

fn_monitor_check_monitoring() {
	if [ -f "${lockdir}/${selfname}.lock" ]; then
		# Part of migration to v23.5.0. #4296
		rm -f "${lockdir:?}/${selfname}.lock"
		date '+%s' > "${lockdir:?}/${selfname}-monitoring.lock"
	elif [ ! -f "${lockdir}/${selfname}-monitoring.lock" ]; then
		# Monitor does not run if lockfile is not found.
		fn_print_dots "Checking lockfile"
		fn_print_checking_eol
		fn_script_log_info "Checking lockfile: CHECKING"
		fn_print_error "Checking lockfile: No lockfile found"
		fn_print_error_eol_nl
		fn_script_log_error "Checking lockfile: No lockfile found: ERROR"
		echo -e "* Start ${selfname} to run monitor."
		core_exit.sh
	fi
}

fn_monitor_check_install() {
	if fn_monitor_command_running "install|i|auto-install|ai"; then
		fn_print_dots "Checking installer"
		fn_print_checking_eol
		fn_script_log_info "Checking installer: CHECKING"
		fn_print_info "Checking installer: LinuxGSM is installing"
		fn_print_info_eol_nl
		fn_script_log_pass "Checking installer: LinuxGSM is installing"
		core_exit.sh
	fi
}

fn_monitor_check_debug() {
	if fn_monitor_command_running "debug|d"; then
		fn_print_dots "Checking debug"
		fn_print_checking_eol
		fn_print_info "Checking debug: Debug is running"
		fn_print_info_eol_nl
		fn_script_log_pass "Checking debug: Debug is running"
		core_exit.sh
	fi
}

fn_monitor_check_details() {
	if fn_monitor_command_running "details|dt"; then
		fn_print_dots "Checking details"
		fn_print_checking_eol
		fn_print_info "Checking details: Details is running"
		fn_print_info_eol_nl
		fn_script_log_pass "Checking details: Details is running"
		core_exit.sh
	fi
}

fn_monitor_check_starting() {
	# Remove stale lockfile.
	if [ -f "${lockdir}/${selfname}-starting.lock" ]; then
		if [ "$(find "${lockdir}/${selfname}-starting.lock" -mmin +5)" ]; then
			fn_print_dots "Checking start"
			fn_print_checking_eol
			fn_print_warn "Checking start: Removing stale lockfile"
			fn_print_warn_eol_nl
			fn_script_log_warn "Checking start: Removing stale lockfile"
			rm -f "${lockdir:?}/${selfname}-starting.lock"
		fi
	fi

	if [ -f "${lockdir}/${selfname}-starting.lock" ] && fn_monitor_command_running "start|st"; then
		fn_print_dots "Checking start"
		fn_print_checking_eol
		fn_print_info "Checking start: LinuxGSM is starting"
		fn_print_info_eol_nl
		fn_script_log_info "Checking start: LinuxGSM is starting"
		core_exit.sh
	fi
}

fn_monitor_check_stopping() {
	# Remove stale lockfile.
	if [ -f "${lockdir}/${selfname}-stopping.lock" ]; then
		if [ "$(find "${lockdir}/${selfname}-stopping.lock" -mmin +5)" ]; then
			fn_print_dots "Checking stop"
			fn_print_checking_eol
			fn_print_warn "Checking stop: Removing stale lockfile"
			fn_print_warn_eol_nl
			fn_script_log_warn "Checking stop: Removing stale lockfile"
			rm -f "${lockdir:?}/${selfname}-stopping.lock"
		fi
	fi

	if [ -f "${lockdir}/${selfname}-stopping.lock" ] && fn_monitor_command_running "stop|sp"; then
		fn_print_dots "Checking stop"
		fn_print_checking_eol
		fn_print_info "Checking stop: LinuxGSM is stopping"
		fn_print_info_eol_nl
		fn_script_log_info "Checking stop: LinuxGSM is stopping"
		core_exit.sh
	fi
}

fn_monitor_check_restart_request() {
	if [ -f "${lockdir}/${selfname}-restart-request.lock" ]; then
		fn_print_dots "Checking restart: "
		fn_print_checking_eol
		fn_print_info "Checking restart: Restart requested: "
		fn_print_info_eol_nl
		fn_script_log_info "Checking restart: Restart requested"
		check_players_online.sh
		if [ -n "${playersonline}" ]; then
			fn_print_info_nl "${playersonline} players are on the server: restart postponed"
			fn_script_log_info "${playersonline} players are on the server: restart postponed"
			echo "${playersonline}" > "${lockdir:?}/${selfname}-player-numbers.lock"
			date '+%s' > "${lockdir:?}/${selfname}-restart-request.lock"
			core_exit.sh
		fi
		command_restart.sh
		core_exit.sh
	fi
}

# An update postponed because players were online runs once the server is empty.
# It runs in the background as its own update command, so monitor returns straight away
# (e.g. a healthcheck timeout can't stop it) and other monitor runs see the update in progress.
fn_monitor_check_update_request() {
	local updaterequest="${lockdir}/${selfname}-update-request.lock"
	if [ -f "${updaterequest}" ]; then
		fn_print_dots "Checking update: "
		fn_print_checking_eol
		fn_print_info "Checking update: Update requested: "
		fn_print_info_eol_nl
		fn_script_log_info "Checking update: Update requested"
		check_players_online.sh
		if [ -n "${playersonline}" ]; then
			fn_print_info_nl "${playersonline} players are on the server: update postponed"
			fn_script_log_info "${playersonline} players are on the server: update postponed"
			echo "${playersonline}" > "${lockdir:?}/${selfname}-player-numbers.lock"
		else
			local requestforce=""
			if grep -qx "force" "${updaterequest}"; then
				requestforce="1"
			fi
			fn_print_info_nl "Checking update: Running requested update"
			fn_script_log_info "Checking update: Running requested update"
			forceupdate="${requestforce}" setsid "${rootdir}/${selfname}" update > /dev/null 2>&1 < /dev/null &
			# shellcheck disable=SC2034 # Used by core_exit.sh.
			exitcode=0
			core_exit.sh
		fi
	fi
}

# A backup postponed because players were online runs once the server is empty, in the background
# like a requested update. An alert is sent once if it has been postponed for over 24 hours.
fn_monitor_check_backup_request() {
	local backuprequest="${lockdir}/${selfname}-backup-request.lock"
	if [ -f "${backuprequest}" ]; then
		fn_print_dots "Checking backup: "
		fn_print_checking_eol
		fn_print_info "Checking backup: Backup requested: "
		fn_print_info_eol_nl
		fn_script_log_info "Checking backup: Backup requested"
		check_players_online.sh
		if [ -n "${playersonline}" ]; then
			fn_print_info_nl "${playersonline} players are on the server: backup postponed"
			fn_script_log_info "${playersonline} players are on the server: backup postponed"
			echo "${playersonline}" > "${lockdir:?}/${selfname}-player-numbers.lock"
			local requested
			requested="$(head -n 1 "${backuprequest}")"
			if [[ "${requested}" =~ ^[0-9]+$ ]] && [ "$(($(date '+%s') - requested))" -gt 86400 ] && ! grep -q "alerted" "${backuprequest}"; then
				alert="backup-postponed"
				alert.sh
				echo "alerted" >> "${backuprequest}"
			fi
		else
			fn_print_info_nl "Checking backup: Running requested backup"
			fn_script_log_info "Checking backup: Running requested backup"
			setsid "${rootdir}/${selfname}" backup > /dev/null 2>&1 < /dev/null &
			# shellcheck disable=SC2034 # Used by core_exit.sh.
			exitcode=0
			core_exit.sh
		fi
	fi
}

fn_monitor_check_backup() {
	# Remove stale lockfile, unless a long backup is still running.
	if [ -f "${lockdir}/backup.lock" ] && ! fn_monitor_command_running "backup|b"; then
		if [ "$(find "${lockdir}/backup.lock" -mmin +60)" ]; then
			fn_print_dots "Checking backup"
			fn_print_checking_eol
			fn_print_warn "Checking backup: Removing stale lockfile"
			fn_print_warn_eol
			fn_script_log_warn "Checking backup: Removing stale lockfile"
			rm -f "${lockdir:?}/backup.lock"
		fi
	fi

	if [ -f "${lockdir}/backup.lock" ] && fn_monitor_command_running "backup|b"; then
		fn_print_dots "Checking backup"
		fn_print_checking_eol
		fn_print_info "Checking backup: Backup is running"
		fn_print_info_eol_nl
		fn_script_log_info "Checking backup: Backup is running"
		core_exit.sh
	fi
}

fn_monitor_check_update() {
	# Remove stale lockfile, unless a long update is still running.
	if [ -f "${lockdir}/update.lock" ] && ! fn_monitor_command_running "update|u|validate|v|force-update|fu|update-restart|ur|workshop-update|wu"; then
		if [ "$(find "${lockdir}/update.lock" -mmin +15)" ]; then
			fn_print_dots "Checking update"
			fn_print_checking_eol
			fn_print_warn "Checking update: Removing stale lockfile"
			fn_print_warn_eol_nl
			fn_script_log_warn "Checking update: Removing stale lockfile"
			rm -f "${lockdir:?}/update.lock"
		fi
	fi

	if [ -f "${lockdir}/update.lock" ] && fn_monitor_command_running "update|u|validate|v|force-update|fu|update-restart|ur|workshop-update|wu"; then
		fn_print_dots "Checking update"
		fn_print_checking_eol
		fn_print_info "Checking update: LinuxGSM is updating the game server"
		fn_print_info_eol_nl
		fn_script_log_pass "Checking update: LinuxGSM is updating the game server"
		core_exit.sh
	fi
}

fn_monitor_check_backup_update() {
	fn_monitor_check_backup
	fn_monitor_check_update
}

# Source engine games may display a messages to indicate the server needs restarting.
fn_monitor_check_update_source() {
	if [ -f "${consolelogdir}/${selfname}-console.log" ] && [ "${engine}" == "source" ]; then
		if grep -q "Your server needs to be restarted in order to receive the latest update." "${consolelogdir}/${selfname}-console.log"; then
			fn_print_dots "Checking update"
			fn_print_checking_eol
			fn_script_log_info "Checking update: CHECKING"
			fn_print_ok "Checking update"
			fn_print_ok_eol_nl
			fn_script_log_info "Checking update: ${selfname} has requested a restart for an update to be applied"
			alert="update-restart-request"
			alert.sh
			command_restart.sh
			core_exit.sh
		fi
	fi
}

fn_monitor_check_session() {
	fn_print_dots "Checking session"
	fn_print_checking_eol
	fn_script_log_info "Checking session: CHECKING"
	# Tmux session width and height needs to be reviewed as may no longer be required.
	sessionwidth="80"
	sessionheight="23"
	# Check for PIDS with identical tmux sessions running.
	if [ "$(pgrep -fcx "tmux -L ${socketname} new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}")" -ge "2" ]; then
		fn_print_error "Checking session: There are PIDS with identical tmux sessions running"
		fn_print_error_eol_nl
		fn_script_log_error "Checking session: ERROR"
		fn_script_log_error "Checking session: There are PIDS with identical tmux sessions running"
		fn_script_log_error "Checking session: Killing all tmux sessions with the socketname name ${socketname} and session name ${sessionname}"
		pkill -f "tmux -L ${socketname} new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}"
		fn_monitor_check_backup_update
		command_restart.sh
		core_exit.sh
	# Check for tmux pids with the same tmux session and socket names. This will reduce issues with migration to release v23.5.0. #4296
	elif [ "$(pgrep -fc -u "${USER}" "^tmux -L ${sessionname} new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}( |$)")" != "0" ]; then
		fn_print_error "Checking session: PIDS with the same tmux session and socket names are running"
		fn_print_error_eol_nl
		fn_script_log_error "Checking session: ERROR"
		fn_script_log_error "Checking session: PIDS with the same tmux session and socket names are running"
		fn_script_log_error "Checking session: Killing session with the socketname name ${sessionname} and session name ${sessionname}"
		pkill -f "^tmux -L ${sessionname} new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}( |$)"
		fn_monitor_check_backup_update
		command_restart.sh
		core_exit.sh
	# Check for tmux pids that are using the old type of tmux session. This will reduce issues with migration to release v23.5.0. #4296
	elif [ "$(pgrep -fc -u "${USER}" "^tmux new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}( |$)")" != "0" ]; then
		fn_print_error "Checking session: PIDS with old type tmux session are running"
		fn_print_error_eol_nl
		fn_script_log_error "Checking session: ERROR"
		fn_script_log_error "Checking session: PIDS with old type tmux session are running"
		fn_script_log_error "Checking session: Killing session with the session name ${sessionname}"
		pkill -f "^tmux new-session -d -x ${sessionwidth} -y ${sessionheight} -s ${sessionname}( |$)"
		fn_monitor_check_backup_update
		command_restart.sh
		core_exit.sh
	elif [ "${status}" != "0" ]; then
		fn_print_ok "Checking session"
		fn_print_ok_eol_nl
		fn_script_log_pass "Checking session: OK"
		# send LinuxGSM stats if monitor is OK.
		if [ "${stats}" == "on" ] || [ "${stats}" == "y" ] && [ "${querymode}" == "1" ]; then
			info_stats.sh
		fi
	else
		fn_print_error "Checking session"
		fn_print_fail_eol_nl
		fn_script_log_fail "Checking session: FAIL"
		fn_monitor_check_backup_update
		alert="monitor-session"
		alert.sh
		fn_script_log_info "Checking session: Monitor is restarting ${selfname}"
		command_restart.sh
		core_exit.sh
	fi
}

# Monitor will check queryport is set before continuing.
fn_monitor_check_queryport() {
	if [ -z "${queryport}" ] || [ "${queryport}" == "0" ]; then
		fn_print_dots "Checking port"
		fn_print_checking_eol
		fn_script_log_info "Checking port: CHECKING"
		if [ -n "${rconenabled}" ] && [ "${rconenabled}" != "true" ] && [ "${shortname}" == "av" ]; then
			fn_print_warn "Checking port: Unable to query, rcon is not enabled"
			fn_script_log_warn "Checking port: Unable to query, rcon is not enabled"
		else
			fn_print_error "Checking port: Unable to query, queryport is not set"
			fn_script_log_error "Checking port: Unable to query, queryport is not set"
		fi
		core_exit.sh
	fi
}

fn_query_gsquery() {
	if [ ! -f "${modulesdir}/query_gsquery.py" ]; then
		fn_fetch_file_github "lgsm/modules" "query_gsquery.py" "${modulesdir}" "chmodx" "norun" "noforce" "nohash"
	fi
	"${modulesdir}"/query_gsquery.py -a "${queryip}" -p "${queryport}" -e "${querytype}" > /dev/null 2>&1
	querystatus="$?"
}

fn_query_tcp() {
	bash -c "exec 3<> /dev/tcp/'${queryip}'/'${queryport}'" > /dev/null 2>&1
	querystatus="$?"
}

fn_monitor_query_port_hint() {
	local sockets protocol state recvq sendq address peer details
	local serverpid="" protocols="" process="" pid="" ancestor="" serverprocess="" otherprocess="" hint=""

	command -v ss > /dev/null 2>&1 || return 0
	sockets="$(ss -H -lntup 2> /dev/null)" || return 0
	serverpid="$(TERM=screen tmux -L "${socketname}" list-sessions -F "#{session_name} #{pane_pid}" 2> /dev/null | awk -v session="${sessionname}" '$1 == session {print $2}')"

	while read -r protocol state recvq sendq address peer details; do
		if [ "${address##*:}" != "${queryport}" ]; then
			continue
		fi
		if [[ " ${protocols//\// } " != *" ${protocol} "* ]]; then
			protocols+="${protocols:+/}${protocol}"
		fi
		process="$(sed -nE 's/.*users:\(\("([^"]+)".*/\1/p' <<< "${details}")"
		pid="$(sed -nE 's/.*users:\(\("[^"]+",pid=([0-9]+).*/\1/p' <<< "${details}")"
		ancestor="${pid}"
		while [ -n "${serverpid}" ] && [ -n "${ancestor}" ] && [ "${ancestor}" -gt 1 ]; do
			if [ "${ancestor}" == "${serverpid}" ]; then
				serverprocess="${process}"
				break
			fi
			ancestor="$(ps -o ppid= -p "${ancestor}" 2> /dev/null | tr -d ' ')"
		done
		if [ -z "${serverprocess}" ] && [ -z "${otherprocess}" ]; then
			otherprocess="${process}"
		fi
	done <<< "${sockets}"

	if [ -z "${protocols}" ]; then
		hint="Nothing is listening on port ${queryport}; check the server started and the port settings"
	elif [ -n "${serverprocess}" ]; then
		hint="Port ${queryport}/${protocols} is open by the game server (${serverprocess}) but it did not answer; the server may be hung, or the query port or type may not match"
	elif [ -n "${otherprocess}" ]; then
		hint="Port ${queryport}/${protocols} is in use by another process (${otherprocess}); check for a port conflict"
	else
		hint="Port ${queryport}/${protocols} is in use by a process owned by another user; check for a port conflict"
	fi

	fn_print_warn "Query port: ${hint}"
	fn_print_warn_eol_nl
	fn_script_log_warn "Query port: ${hint}"
}

fn_monitor_query() {
	# Will loop and query up to 5 times every 15 seconds.
	# Query will wait up to 60 seconds to confirm server is down as server can become non-responsive during map changes.
	totalseconds=0
	for queryattempt in {1..5}; do
		for queryip in "${queryips[@]}"; do
			fn_print_dots "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
			fn_print_querying_eol
			fn_script_log_info "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : QUERYING"
			# querydelay
			if [ "$(head -n 1 "${lockdir}/${selfname}-started.lock")" -gt "$(date "+%s" -d "${querydelay} mins ago")" ]; then
				fn_print_ok "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
				fn_print_delay_eol_nl
				fn_script_log_info "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : DELAY"
				fn_script_log_info "Query bypassed: ${gameservername} started less than ${querydelay} minutes ago"
				fn_script_log_info "Server started: $(date -d "@$(head -n 1 "${lockdir}/${selfname}-started.lock")")"
				fn_script_log_info "Current time: $(date)"
				monitorpass=1
				core_exit.sh
			# will use query method selected in fn_monitor_loop
			# gamedig
			elif [ "${querymethod}" == "gamedig" ]; then
				query_gamedig.sh
			# gsquery
			elif [ "${querymethod}" == "gsquery" ]; then
				fn_query_gsquery
			#tcp query
			elif [ "${querymethod}" == "tcp" ]; then
				fn_query_tcp
			fi

			if [ "${querystatus}" == "0" ]; then
				# Server query OK.
				fn_print_ok "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
				fn_print_ok_eol_nl
				fn_script_log_pass "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : OK"
				monitorpass=1
				if [ "${querystatus}" == "0" ]; then
					# Add query data to log.
					if [ "${gdname}" ]; then
						fn_script_log_info "Server name: ${gdname}"
					fi
					if [ "${gdplayers}" ]; then
						fn_script_log_info "Players: ${gdplayers}/${gdmaxplayers}"
					fi
					if [ "${gdbots}" ]; then
						fn_script_log_info "Bots: ${gdbots}"
					fi
					if [ "${gdmap}" ]; then
						fn_script_log_info "Map: ${gdmap}"
					fi
					if [ "${gdgamemode}" ]; then
						fn_script_log_info "Game Mode: ${gdgamemode}"
					fi

					# send LinuxGSM stats if monitor is OK.
					if [ "${stats}" == "on" ] || [ "${stats}" == "y" ]; then
						info_stats.sh
					fi
				fi
				core_exit.sh
			else
				# Server query FAIL.
				fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
				fn_print_fail_eol
				fn_script_log_warn "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : FAIL"
				# Monitor will try gamedig (if supported) for first 30s then gsquery before restarting.
				# gsquery will fail if longer than 60s
				if [ "${totalseconds}" -ge "59" ]; then
					# Monitor will FAIL if over 60s and trigger gane server reboot.
					fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
					fn_print_fail_eol_nl
					fn_script_log_warn "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : FAIL"
					fn_monitor_query_port_hint
					fn_monitor_check_backup_update
					# Send alert if enabled.
					alert="monitor-query"
					alert.sh
					command_restart.sh
					fn_firstcommand_reset
					core_exit.sh
				fi
			fi
		done
		# Second counter will wait for 15s before breaking loop.
		for seconds in {1..15}; do
			fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
			fn_print_wait_eol
			fn_sleep_time_1
			totalseconds=$((totalseconds + 1))
			if [ "${seconds}" == "15" ]; then
				break
			fi
		done
	done
}

fn_monitor_loop() {
	# loop though query methods selected by querymode.
	totalseconds=0
	if [ "${querymode}" == "2" ]; then
		local query_methods_array=(gamedig gsquery)
	elif [ "${querymode}" == "3" ]; then
		local query_methods_array=(gamedig)
	elif [ "${querymode}" == "4" ]; then
		local query_methods_array=(gsquery)
	elif [ "${querymode}" == "5" ]; then
		local query_methods_array=(tcp)
	fi
	for querymethod in "${query_methods_array[@]}"; do
		# Will check if gamedig is installed and bypass if not.
		if [ "${querymethod}" == "gamedig" ]; then
			if [ "$(command -v gamedig 2> /dev/null)" ] || [ -f "${lgsmdir}/node_modules/gamedig/bin/gamedig.js" ] && [ "$(command -v jq 2> /dev/null)" ]; then
				if [ -z "${monitorpass}" ]; then
					fn_monitor_query
				fi
			else
				fn_script_log_info "gamedig is not installed"
				fn_script_log_info "https://docs.linuxgsm.com/requirements/gamedig"
			fi
		else
			# will not query if query already passed.
			if [ -z "${monitorpass}" ]; then
				fn_monitor_query
			fi
		fi
	done
}

fn_print_dots ""
monitorflag=1
# Dont do any monitoring or checks if installer is running.
fn_monitor_check_install
check.sh
core_logs.sh
info_game.sh

# query pre-checks
fn_monitor_check_details
fn_monitor_check_backup
fn_monitor_check_debug
fn_monitor_check_update_source
fn_monitor_check_update
fn_monitor_check_monitoring
fn_monitor_check_starting
fn_monitor_check_stopping
fn_monitor_check_session
fn_monitor_check_restart_request
fn_monitor_check_update_request
fn_monitor_check_backup_request

# Monitor will not continue if session only check.
if [ "${querymode}" != "1" ]; then
	fn_monitor_check_queryport

	# Add a querydelay of 1 min if var missing.
	if [ -z "${querydelay}" ]; then
		querydelay="1"
	fi

	fn_monitor_loop
fi

core_exit.sh