feat(monitor): show local port state after query failures (#5111)

When a monitor query fails, check the local query port with ss and add a hint to the output and log: either nothing is listening on the port, or the port is in use (naming the process when ss knows it). The monitor still retries and restarts exactly as before.

.github/tests/monitor_query_port_hint.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+# The monitor runs when sourced, so load only the helper under test.
+# shellcheck source=/dev/null
+source <(sed -n '/^fn_monitor_query_port_hint() {/,/^}/p' lgsm/modules/command_monitor.sh)
+
+tmp="$(mktemp -d)"
+trap 'rm -rf "$tmp"' EXIT
+fn_print_warn() { printf '%s\n' "$*" >> "$tmp/screen"; }
+fn_print_warn_eol_nl() { :; }
+fn_script_log_warn() { printf '%s\n' "$*" >> "$tmp/log"; }
+queryport=27016
+
+socket_state=absent
+ss() {
+	case "$socket_state" in
+		absent) printf '%s\n' 'udp UNCONN 0 0 0.0.0.0:270160 0.0.0.0:*' ;;
+		used) printf '%s\n' 'tcp LISTEN 0 128 0.0.0.0:27016 0.0.0.0:* users:(("srcds",pid=123,fd=5))' ;;
+		no_process) printf '%s\n' 'udp UNCONN 0 0 0.0.0.0:27016 0.0.0.0:*' ;;
+		unavailable) return 1 ;;
+	esac
+}
+check_output() {
+	grep -Fq "$1" "$tmp/log"
+	grep -Fq "$1" "$tmp/screen"
+	[ "$(wc -l < "$tmp/log")" -eq 1 ]
+	[ "$(wc -l < "$tmp/screen")" -eq 1 ]
+}
+: > "$tmp/log"
+: > "$tmp/screen"
+fn_monitor_query_port_hint
+check_output 'No local TCP/UDP listener on port 27016'
+
+: > "$tmp/log"
+: > "$tmp/screen"
+socket_state=used
+fn_monitor_query_port_hint
+check_output 'Local TCP/UDP port 27016 is in use by srcds'
+
+: > "$tmp/log"
+: > "$tmp/screen"
+socket_state=no_process
+fn_monitor_query_port_hint
+check_output 'Local TCP/UDP port 27016 is in use; check the query port and protocol'
+
+: > "$tmp/log"
+: > "$tmp/screen"
+socket_state=unavailable
+fn_monitor_query_port_hint
+[ ! -s "$tmp/log" ]
+[ ! -s "$tmp/screen" ]
+printf '%s\n' 'monitor query port hints: 4 checks passed'

lgsm/modules/command_monitor.sh

@@ -359,6 +359,37 @@ fn_query_tcp() {
 	querystatus="$?"
 }
 
+# Explain a failed query without changing the monitor's retry/restart decision.
+fn_monitor_query_port_hint() {
+	local sockets protocol state recvq sendq address peer details
+	local listener="" process="" hint=""
+
+	command -v ss > /dev/null 2>&1 || return 0
+	sockets="$(ss -H -lntup 2> /dev/null)" || return 0
+
+	while read -r protocol state recvq sendq address peer details; do
+		if [ "${address##*:}" == "${queryport}" ]; then
+			listener=1
+			process="$(sed -nE 's/.*users:\(\("([^"]+)".*/\1/p' <<< "${details}")"
+			break
+		fi
+	done <<< "${sockets}"
+
+	if [ -z "${listener}" ]; then
+		hint="No local TCP/UDP listener on port ${queryport}; check startup and port settings"
+	else
+		hint="Local TCP/UDP port ${queryport} is in use"
+		if [ -n "${process}" ]; then
+			hint+=" by ${process}"
+		fi
+		hint+="; check the query port and protocol"
+	fi
+
+	fn_print_warn "Query port: ${hint}"
+	fn_print_warn_eol_nl
+	fn_script_log_warn "Query port: ${hint}"
+}
+
 fn_monitor_query() {
 	# Will loop and query up to 5 times every 15 seconds.
 	# Query will wait up to 60 seconds to confirm server is down as server can become non-responsive during map changes.
@@ -432,6 +463,7 @@ fn_monitor_query() {
 					fn_print_fail "Querying port: ${querymethod}: ${queryip}:${queryport} : ${totalseconds}/${queryattempt}"
 					fn_print_fail_eol_nl
 					fn_script_log_warn "Querying port: ${querymethod}: ${queryip}:${queryport} : ${queryattempt} : FAIL"
+					fn_monitor_query_port_hint
 					fn_monitor_check_backup_update
 					# Send alert if enabled.
 					alert="monitor-query"