ci(details-check): run servers in shards from the checkout (#4993)

* ci(details-check): run servers in shards from the checkout

- Check servers in 12 shards instead of one job per server, so jobs no
  longer queue behind the concurrency limit.
- Seed lgsm/modules, config-default and data from the checkout, so each
  server takes about 20s instead of about 80s and the PR's own code is
  what gets checked.
- Run bf1942, bfv, btl and onset in an ubuntu:20.04 container. The
  ubuntu-20.04 runner is retired, so btl and onset never started.
- Changes to the details-check workflow or scripts trigger a full run.

* ci(details-check): use the config filename for the Game-Server-Configs URL

servercfgdefault can include a directory variable (xnt uses
${systemdir}/server.cfg). Game-Server-Configs stores files flat per
game, so use only the filename.

* ci(details-check): keep config subdirectories in the Game-Server-Configs URL

Only drop a leading ${variable}/ (xnt). pc2's config lives in
config_sample/ in Game-Server-Configs.

* ci(details-check): fix the Game-Server-Configs path expansion

Strip a leading ${variable}/ with a pattern test instead of a nested
parameter expansion, which bash mangled. Keeps pc2's config_sample/.

.github/scripts/details-check-generate-matrix.sh

@@ -1,50 +1,50 @@
 #!/bin/bash
+# Splits the servers to check into shards (one job each) and a list of legacy servers.
+# Legacy servers need an older glibc than the hosted runners have, so they run in an ubuntu:20.04 container.
+# Writes shards.json (matrix) and legacy.txt (space-separated shortnames).
 
 ref="${LGSM_REF:-${GITHUB_REF#refs/heads/}}"
-curl "https://raw.githubusercontent.com/GameServerManagers/LinuxGSM/${ref}/lgsm/data/serverlist.csv" | grep -v '^[[:blank:]]*$' > serverlist.csv
+maxshards="${DETAILS_CHECK_SHARDS:-12}"
+legacyservers="bfv bf1942 btl onset"
 
-echo -n "{" > "shortnamearray.json"
-echo -n "\"include\":[" >> "shortnamearray.json"
+if [ -f lgsm/data/serverlist.csv ]; then
+	serverlist="lgsm/data/serverlist.csv"
+else
+	curl -fsS "https://raw.githubusercontent.com/GameServerManagers/LinuxGSM/${ref}/lgsm/data/serverlist.csv" -o serverlist.csv
+	serverlist="serverlist.csv"
+fi
 
-while read -r line; do
-	shortname=$(echo "$line" | awk -F, '{ print $1 }')
-	# If TARGETED_SHORTNAMES is set, skip servers not in the list
-	if [ -n "${TARGETED_SHORTNAMES:-}" ]; then
-		if ! echo "${TARGETED_SHORTNAMES}" | grep -qw "${shortname}"; then
-			continue
-		fi
+shortnames=()
+legacy=()
+while IFS=, read -r shortname _; do
+	[ -z "${shortname}" ] && continue
+	if [ -n "${TARGETED_SHORTNAMES:-}" ] && ! echo "${TARGETED_SHORTNAMES}" | grep -qw "${shortname}"; then
+		continue
 	fi
-	export shortname
-	servername=$(echo "$line" | awk -F, '{ print $2 }')
-	export servername
-	gamename=$(echo "$line" | awk -F, '{ print $3 }')
-	export gamename
-	distro=$(echo "$line" | awk -F, '{ print $4 }')
-	export distro
-	# Legacy servers that require older Ubuntu/Debian versions due to glibc compatibility
-	case "${shortname}" in
-		bfv | bf1942)
-			# Requires Ubuntu <= 22.04 or Debian <= 12 (glibc 2.31 compatible)
-			runner="ubuntu-22.04"
-			;;
-		btl | onset)
-			# Requires Ubuntu <= 20.04 or Debian <= 11 (glibc 2.31 compatible)
-			runner="ubuntu-20.04"
-			;;
-		*)
-			runner="ubuntu-latest"
-			;;
-	esac
-	{
-		echo -n "{"
-		echo -n "\"shortname\":"
-		echo -n "\"${shortname}\""
-		echo -n ",\"runner\":"
-		echo -n "\"${runner}\""
-		echo -n "},"
-	} >> "shortnamearray.json"
-done < <(tail -n +2 serverlist.csv)
-sed -i '$ s/.$//' "shortnamearray.json"
-echo -n "]" >> "shortnamearray.json"
-echo -n "}" >> "shortnamearray.json"
-rm serverlist.csv
+	if echo "${legacyservers}" | grep -qw "${shortname}"; then
+		legacy+=("${shortname}")
+	else
+		shortnames+=("${shortname}")
+	fi
+done < <(tail -n +2 "${serverlist}" | grep -v '^[[:blank:]]*$')
+
+count="${#shortnames[@]}"
+shards=$((count < maxshards ? count : maxshards))
+declare -a shard
+for i in "${!shortnames[@]}"; do
+	shard[i % shards]+="${shortnames[i]} "
+done
+
+{
+	echo -n '{"include":['
+	for ((i = 0; i < shards; i++)); do
+		[ "${i}" -gt 0 ] && echo -n ','
+		echo -n "{\"shard\":\"$((i + 1))/${shards}\",\"shortnames\":\"${shard[i]% }\"}"
+	done
+	echo -n ']}'
+} > shards.json
+
+echo -n "${legacy[*]}" > legacy.txt
+[ "${serverlist}" == "serverlist.csv" ] && rm serverlist.csv
+
+echo "Servers: ${count} in ${shards} shard(s), legacy: ${legacy[*]:-none}"

.github/scripts/details-check-run.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+# Runs the details check for each shortname given, in one shared LinuxGSM directory.
+# Modules, default configs and data come from the checkout, so the code under test is used and nothing is downloaded per server.
+# Usage: details-check-run.sh <shortname>...
+
+src="${GITHUB_WORKSPACE:-${PWD}}"
+workdir="${DETAILS_CHECK_DIR:-${HOME}/details-check}"
+summary="${GITHUB_STEP_SUMMARY:-/dev/null}"
+export LGSM_GITHUBBRANCH="${LGSM_REF:?LGSM_REF must be set}"
+
+rm -rf "${workdir}"
+mkdir -p "${workdir}/lgsm" "${workdir}/serverfiles"
+cp "${src}/linuxgsm.sh" "${workdir}/"
+cp -r "${src}/lgsm/modules" "${src}/lgsm/config-default" "${src}/lgsm/data" "${workdir}/lgsm/"
+cd "${workdir}" || exit 1
+
+result() {
+	if [ "${1}" -eq 0 ]; then echo "✅ pass"; else echo "❌ fail"; fi
+}
+
+failed=()
+{
+	echo "| Server | Config | Details | Parse game details | Query raw |"
+	echo "|--------|--------|---------|--------------------|-----------|"
+} >> "${summary}"
+
+for shortname in "$@"; do
+	gameserver="${shortname}server"
+	echo "::group::${gameserver}"
+	./linuxgsm.sh "${gameserver}"
+	./"${gameserver}" developer
+
+	servercfg=$(sed -n "/^\<servercfgdefault\>/ { s/.*= *\"\?\([^\"']*\)\"\?/\1/p;q }" "lgsm/config-lgsm/${gameserver}/_default.cfg")
+	if [ -n "${servercfg}" ]; then
+		cfgpath="${servercfg}"
+		[[ "${cfgpath}" == \$\{*\}/* ]] && cfgpath="${cfgpath#*\}/}"
+		curl -fsS -o "config-${shortname}" "https://raw.githubusercontent.com/GameServerManagers/Game-Server-Configs/main/${shortname}/${cfgpath}"
+		config=$?
+		[ "${config}" -eq 0 ] && cat "config-${shortname}"
+		configresult=$(result "${config}")
+	else
+		echo "This game server has no config file."
+		config=0
+		configresult="➖ none"
+	fi
+	grep "startparameters" "lgsm/config-default/config-lgsm/${gameserver}/_default.cfg"
+
+	./"${gameserver}" details
+	details=$?
+	./"${gameserver}" parse-game-details
+	parse=$?
+	./"${gameserver}" query-raw
+	query=$?
+	echo "::endgroup::"
+
+	if [ "${config}" -ne 0 ] || [ "${details}" -ne 0 ] || [ "${parse}" -ne 0 ] || [ "${query}" -ne 0 ]; then
+		failed+=("${gameserver}")
+		echo "::error title=${gameserver}::config=${config} details=${details} parse-game-details=${parse} query-raw=${query}"
+	fi
+	echo "| ${gameserver} | ${configresult} | $(result "${details}") | $(result "${parse}") | $(result "${query}") |" >> "${summary}"
+done
+
+if [ "${#failed[@]}" -gt 0 ]; then
+	echo -e "\n**Failed:** ${failed[*]}" >> "${summary}"
+	exit 1
+fi

.github/workflows/action-details-check.yml

@@ -16,18 +16,21 @@ concurrency:
   group: details-check-${{ github.ref_name }}
   cancel-in-progress: true
 
+env:
+  LGSM_REF: ${{ github.event.pull_request.head.sha || github.sha }}
+
 jobs:
   create-matrix:
     if: github.repository_owner == 'GameServerManagers'
     runs-on: ubuntu-latest
-    env:
-      LGSM_REF: ${{ github.event.pull_request.head.sha || github.ref_name }}
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      legacy: ${{ steps.set-matrix.outputs.legacy }}
     steps:
       - name: Checkout
         uses: actions/checkout@v7
         with:
+          ref: ${{ env.LGSM_REF }}
           # Full history so the base branch ref is available for the
           # `git diff origin/<base>...HEAD` comparison below (a shallow
           # checkout has no origin/<base> and the diff fails).
@@ -44,7 +47,7 @@ jobs:
           # Only these modules directly affect what 'details', 'parse-game-details'
           # and 'query-raw' output. Everything else (install, fix, update, check_*,
           # data CSVs, game icons, etc.) is irrelevant to this check.
-          details_core='^(linuxgsm\.sh|lgsm/modules/(info_game|info_messages|command_details|command_dev_parse_game_details|command_dev_query_raw|core_modules)\.sh)$'
+          details_core='^(linuxgsm\.sh|lgsm/modules/(info_game|info_messages|command_details|command_dev_parse_game_details|command_dev_query_raw|core_modules)\.sh|\.github/workflows/action-details-check\.yml|\.github/scripts/details-check-.*\.sh)$'
           if echo "$changed" | grep -qE "${details_core}"; then
             echo "Details-relevant core file changed — using full matrix."
             echo "mode=full" >> "$GITHUB_OUTPUT"
@@ -64,7 +67,7 @@ jobs:
             fi
           fi
 
-      - name: Generate matrix with generate-matrix.sh
+      - name: Generate matrix
         if: github.event_name != 'pull_request' || steps.targeted.outputs.mode != 'skip'
         run: |
           if [ -f targeted_shortnames.txt ]; then
@@ -76,29 +79,30 @@ jobs:
       - name: Set Matrix
         id: set-matrix
         run: |
-          if [ ! -f shortnamearray.json ]; then
+          if [ ! -f shards.json ]; then
             echo 'matrix={"include":[]}' >> "$GITHUB_OUTPUT"
+            echo 'legacy=' >> "$GITHUB_OUTPUT"
           else
-            shortnamearray=$(cat shortnamearray.json)
-            echo "${shortnamearray}"
-            echo -n "matrix=${shortnamearray}" >> "$GITHUB_OUTPUT"
+            cat shards.json
+            echo "matrix=$(cat shards.json)" >> "$GITHUB_OUTPUT"
+            echo "legacy=$(cat legacy.txt)" >> "$GITHUB_OUTPUT"
           fi
 
   details-check:
+    name: details-check (${{ matrix.shard }})
     # Skip (rather than fail the run) when create-matrix found nothing to check.
     if: github.repository_owner == 'GameServerManagers' && needs.create-matrix.outputs.matrix != '{"include":[]}'
     needs: create-matrix
     continue-on-error: true
-    runs-on: ${{ matrix.runner }}
-    env:
-      LGSM_REF: ${{ github.event.pull_request.head.sha || github.ref_name }}
-
+    runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix: ${{ fromJSON(needs.create-matrix.outputs.matrix) }}
-
     steps:
-      - name: Download linuxgsm.sh
-        run: wget "https://raw.githubusercontent.com/GameServerManagers/LinuxGSM/${LGSM_REF}/linuxgsm.sh"; chmod +x linuxgsm.sh
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ env.LGSM_REF }}
 
       - name: Cache apt packages
         uses: actions/cache@v6
@@ -109,74 +113,36 @@ jobs:
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libxml2-utils jq
 
-      - name: Create serverfiles directory
-        run: mkdir -p serverfiles
-
-      - name: Grab server
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./linuxgsm.sh ${{ matrix.shortname }}server
-
-      - name: Enable developer mode
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./${{ matrix.shortname }}server developer
-
-      - name: Generate servercfgname
-        id: sets-servercfgname
-        run: |
-          servercfg=$(sed -n "/^\<servercfgdefault\>/ { s/.*= *\"\?\([^\"']*\)\"\?/\1/p;q }" lgsm/config-lgsm/${{ matrix.shortname }}server/_default.cfg)
-          echo "servercfgname=$servercfg" >> "$GITHUB_OUTPUT"
-
-      - name: Download config
-        run: |
-          if [ "${{ steps.sets-servercfgname.outputs.servercfgname }}" == "" ]; then
-            echo "This game server has no config file."
-          else
-            curl -f -o config "https://raw.githubusercontent.com/GameServerManagers/Game-Server-Configs/main/${{ matrix.shortname }}/${{ steps.sets-servercfgname.outputs.servercfgname }}"
-          fi
+      - name: Details check
+        run: .github/scripts/details-check-run.sh ${{ matrix.shortnames }}
 
-      - name: Pre-load LinuxGSM
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./${{ matrix.shortname }}server details
-
-      - name: Display config
+  details-check-legacy:
+    name: details-check (legacy)
+    if: github.repository_owner == 'GameServerManagers' && needs.create-matrix.outputs.legacy != ''
+    needs: create-matrix
+    continue-on-error: true
+    runs-on: ubuntu-latest
+    container: public.ecr.aws/ubuntu/ubuntu:20.04
+    steps:
+      - name: Install dependencies
+        env:
+          DEBIAN_FRONTEND: noninteractive
         run: |
-          if [ "${{ steps.sets-servercfgname.outputs.servercfgname }}" == "" ]; then
-            echo "This game server has no config file."
-          else
-            cat config
-          fi
-
-      - name: Display parameters
-        run: grep "startparameters" lgsm/config-default/config-lgsm/${{ matrix.shortname }}server/_default.cfg
-
-      - name: Details
-        id: details
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./${{ matrix.shortname }}server details
-
-      - name: Detect details
-        id: detect-details
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./${{ matrix.shortname }}server parse-game-details
+          apt-get update
+          apt-get install -y --no-install-recommends ca-certificates curl wget git sudo iproute2 libxml2-utils jq \
+            bc binutils bsdmainutils bzip2 cpio distro-info file gzip hostname lib32gcc-s1 lib32stdc++6 \
+            netcat-openbsd pigz python3 tar tmux unzip util-linux uuid-runtime xz-utils
 
-      - name: Query Raw
-        id: query-raw
-        run: LGSM_GITHUBBRANCH="${LGSM_REF}" ./${{ matrix.shortname }}server query-raw
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ env.LGSM_REF }}
 
-      - name: Write job summary
-        if: always()
+      - name: Details check
         run: |
-          status_icon() {
-            case "$1" in
-              success)  echo "✅" ;;
-              failure)  echo "❌" ;;
-              skipped)  echo "⏭️" ;;
-              *)         echo "⚠️" ;;
-            esac
-          }
-          details_icon=$(status_icon "${{ steps.details.outcome }}")
-          parse_icon=$(status_icon "${{ steps.detect-details.outcome }}")
-          query_icon=$(status_icon "${{ steps.query-raw.outcome }}")
-          {
-            echo "## ${{ matrix.shortname }}server"
-            echo "| Step | Result |"
-            echo "|------|--------|"
-            echo "| Details | ${details_icon} ${{ steps.details.outcome }} |"
-            echo "| Parse game details | ${parse_icon} ${{ steps.detect-details.outcome }} |"
-            echo "| Query raw | ${query_icon} ${{ steps.query-raw.outcome }} |"
-          } >> "$GITHUB_STEP_SUMMARY"
+          useradd -m linuxgsm
+          echo "linuxgsm ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/linuxgsm
+          chown -R linuxgsm:linuxgsm "${GITHUB_WORKSPACE}"
+          chmod a+w "${GITHUB_STEP_SUMMARY}"
+          sudo -u linuxgsm -H env LGSM_REF="${LGSM_REF}" GITHUB_WORKSPACE="${GITHUB_WORKSPACE}" GITHUB_STEP_SUMMARY="${GITHUB_STEP_SUMMARY}" \
+            "${GITHUB_WORKSPACE}/.github/scripts/details-check-run.sh" ${{ needs.create-matrix.outputs.legacy }}