Fix non-ASCII characters in scripts of fetched HTML pages (#9384)

* Fix non-ASCII characters in scripts of fetched HTML pages

To add a <base> to a fetched HTML page, enforceHtmlBase() re-serialises it
with libxml2 saveHTML(). For a whole document, libxml2 2.9 only knows the
charset from <meta http-equiv="Content-Type">: with <meta charset="UTF-8">
or a byte order mark, it writes every non-ASCII character as an HTML entity.
In text that is harmless, but inside <script> entities are not decoded, so
e.g. the JSON read by HTML + XPath + JSON feeds gets "Wall Street&rsquo;s"
instead of "Wall Street’s", which then shows in titles.

UTF-8 documents are now serialised node by node, which writes UTF-8.
Other charsets keep the previous output.

* Clarify comment about the byte order mark

* Remove comment about the byte order mark

* Minor test syntax preference

---------

Co-authored-by: Alexandre Alapetite <alexandre@alapetite.fr>

app/Utils/httpUtil.php

@@ -258,7 +258,20 @@ final class FreshRSS_http_Util {
 		$htmlPos = stripos($html, '<html');
 		$htmlStart = $htmlPos === false || $htmlPos > 512 ? '' : substr($html, 0, $htmlPos);
 
-		$html = $doc->saveHTML() ?: $html;
+		$encoding = strtolower($doc->encoding ?? '');
+		$bom = str_starts_with($html, "\xEF\xBB\xBF") ? "\xEF\xBB\xBF" : '';
+		if ($encoding === 'utf-8' || ($encoding === '' && $bom !== '')) {
+			// For a whole document, libxml2 saveHTML() only knows the charset from <meta http-equiv="Content-Type">.
+			// With <meta charset="UTF-8"> or a byte order mark, it writes non-ASCII characters as HTML entities,
+			// also inside <script>, where they are not decoded, which corrupts e.g. JSON data. Node by node, it writes UTF-8.
+			$utf8 = '';
+			foreach ($doc->childNodes as $node) {
+				$utf8 .= $doc->saveHTML($node) ?: '';
+			}
+			$html = $utf8 !== '' ? $bom . $utf8 : $html;
+		} else {
+			$html = $doc->saveHTML() ?: $html;
+		}
 		if ($htmlStart !== '' && !str_starts_with($html, $htmlStart)) {
 			// libxml2 saveHTML() risks removing Unicode BOM and XML declaration,
 			// which affects future detection of charset encoding, so manually restore it

tests/app/Utils/httpUtilTest.php

@@ -39,6 +39,28 @@ final class httpUtilTest extends \PHPUnit\Framework\TestCase {
 		];
 	}
 
+	#[DataProvider('provideUtf8Html')]
+	public function test_enforceHtmlBase_keepsUtf8InScripts(string $html): void {
+		$enforceHtmlBase = new ReflectionMethod(FreshRSS_http_Util::class, 'enforceHtmlBase');
+		$result = $enforceHtmlBase->invoke(null, $html, 'https://example.net/page');
+		self::assertIsString($result);
+		self::assertStringContainsString('<base href="https://example.net/page">', $result);
+		self::assertStringContainsString('<script type="application/json">{"title":"Café’s menu — 日本"}</script>', $result);
+		if (str_starts_with($html, "\xEF\xBB\xBF")) {
+			self::assertStringStartsWith("\xEF\xBB\xBF", $result);
+		}
+	}
+
+	/** @return Traversable<string,array{string}> */
+	public static function provideUtf8Html(): Traversable {
+		$head = '<html><head><title>T</title></head>';
+		$body = '<body><p>Café’s menu — 日本</p><script type="application/json">{"title":"Café’s menu — 日本"}</script></body></html>';
+		yield 'meta charset' => ['<!DOCTYPE html><html><head><meta charset="utf-8"><title>T</title></head>' . $body];
+		yield 'byte order mark' => ["\xEF\xBB\xBF" . '<!DOCTYPE html>' . $head . $body];
+		yield 'byte order mark, late <html>' => ["\xEF\xBB\xBF" . '<!DOCTYPE html>' . str_repeat('<!-- padding -->', 40) . $head . $body];
+		yield 'meta http-equiv' => ['<!DOCTYPE html><html><head><meta http-equiv="Content-Type" content="text/html; charset=utf-8"><title>T</title></head>' . $body];
+	}
+
 	#[DataProvider('provideCidrRanges')]
 	public function test_checkCIDR(string $ip, string $range, bool $expected): void {
 		$checkCIDR = new ReflectionMethod(FreshRSS_http_Util::class, 'checkCIDR');