Fix SimplePie support of HTTP trailer headers (#7983)

* Fix SimplePie support of HTTP trailer headers
fix https://github.com/FreshRSS/FreshRSS/discussions/7981
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Trailer
https://developer.mozilla.org/en-US/docs/Web/HTTP/Reference/Headers/Server-Timing
We need to use content-length to know where the body stops, but content-length is wrong is any compression was used.
So let cURL perform the separation of HTTP headers and body instead of using the SimplePie parser.

* Minor whitespace

* Same change for lib_rss

* Move changes to SimplePie repo
https://github.com/FreshRSS/simplepie/pull/55
https://github.com/FreshRSS/simplepie/pull/57

lib/composer.json

@@ -14,7 +14,7 @@
         "marienfressinaud/lib_opml": "0.5.1",
         "phpgt/cssxpath": "v1.3.0",
         "phpmailer/phpmailer": "6.11.1",
-        "simplepie/simplepie": "dev-freshrss#d381ca57e9c57e251a47de2b577846503c3030ce"
+        "simplepie/simplepie": "dev-freshrss#c1bf1a353dae742977dde34d65e4c89b633a9b47"
     },
     "config": {
         "sort-packages": true,

lib/lib_rss.php

@@ -633,13 +633,20 @@ function httpGet(string $url, string $cachePath, string $type = 'html', array $a
 		CURLOPT_CONNECTTIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
 		CURLOPT_TIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
 		CURLOPT_MAXREDIRS => 4,
-		CURLOPT_HEADER => true,
 		CURLOPT_RETURNTRANSFER => true,
 		CURLOPT_FOLLOWLOCATION => true,
 		CURLOPT_ENCODING => '',	//Enable all encodings
 		//CURLOPT_VERBOSE => 1,	// To debug sent HTTP headers
 	]);
 
+	$responseHeaders = '';
+	curl_setopt($ch, CURLOPT_HEADERFUNCTION, function (\CurlHandle $ch, string $header) use (&$responseHeaders) {
+		if (trim($header) !== '') {	// Skip e.g. separation with trailer headers
+			$responseHeaders .= $header;
+		}
+		return strlen($header);
+	});
+
 	curl_setopt_array($ch, FreshRSS_Context::systemConf()->curl_options);
 
 	if (is_array($attributes['curl_params'] ?? null)) {
@@ -666,22 +673,20 @@ function httpGet(string $url, string $cachePath, string $type = 'html', array $a
 
 	curl_setopt_array($ch, $curl_options);
 
-	$response = curl_exec($ch);
+	$body = curl_exec($ch);
 	$c_status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
 	$c_content_type = '' . curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
 	$c_effective_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
 	$c_redirect_count = curl_getinfo($ch, CURLINFO_REDIRECT_COUNT);
 	$c_error = curl_error($ch);
 
-	$body = false;
 	$headers = [];
-	if ($response !== false) {
+	if ($body !== false) {
 		assert($c_redirect_count >= 0);
-		$response = \SimplePie\HTTP\Parser::prepareHeaders(is_string($response) ? $response : '', $c_redirect_count + 1);
-		$parser = new \SimplePie\HTTP\Parser($response);
+		$responseHeaders = \SimplePie\HTTP\Parser::prepareHeaders($responseHeaders, $c_redirect_count + 1);
+		$parser = new \SimplePie\HTTP\Parser($responseHeaders);
 		if ($parser->parse()) {
 			$headers = $parser->headers;
-			$body = $parser->body;
 		}
 	}
 

lib/simplepie/simplepie/src/File.php

@@ -125,7 +125,6 @@ class File implements Response
                     curl_setopt($fp, CURLOPT_ENCODING, '');
                 }
                 curl_setopt($fp, CURLOPT_URL, $url);
-                curl_setopt($fp, CURLOPT_HEADER, 1);
                 curl_setopt($fp, CURLOPT_RETURNTRANSFER, 1);
                 curl_setopt($fp, CURLOPT_FAILONERROR, 1);
                 curl_setopt($fp, CURLOPT_TIMEOUT, $timeout);
@@ -133,27 +132,37 @@ class File implements Response
                 // curl_setopt($fp, CURLOPT_REFERER, \SimplePie\Misc::url_remove_credentials($url)); // FreshRSS removed
                 curl_setopt($fp, CURLOPT_USERAGENT, $useragent);
                 curl_setopt($fp, CURLOPT_HTTPHEADER, $headers2);
+                $responseHeaders = '';
+                curl_setopt($fp, CURLOPT_HEADERFUNCTION, function ($ch, string $header) use (&$responseHeaders) {
+                    if (trim($header) !== '') { // Skip e.g. separation with trailer headers
+                        $responseHeaders .= $header;
+                    }
+                    return strlen($header);
+                });
                 foreach ($curl_options as $curl_param => $curl_value) {
                     curl_setopt($fp, $curl_param, $curl_value);
                 }
 
-                /** @var string|false $responseHeaders */
-                $responseHeaders = curl_exec($fp);
+                /** @var string|false $responseBody */
+                $responseBody = curl_exec($fp);
+                $responseHeaders .= "\r\n";
                 if (curl_errno($fp) === CURLE_WRITE_ERROR || curl_errno($fp) === CURLE_BAD_CONTENT_ENCODING) {
                     $this->error = 'cURL error ' . curl_errno($fp) . ': ' . curl_error($fp); // FreshRSS
-                    $this->on_http_response($responseHeaders);
+                    $this->on_http_response($responseBody === false ? false : $responseHeaders . $responseBody);
                     $this->error = null; // FreshRSS
                     curl_setopt($fp, CURLOPT_ENCODING, 'none');
-                    /** @var string|false $responseHeaders */
-                    $responseHeaders = curl_exec($fp);
+                    $responseHeaders = '';
+                    /** @var string|false $responseBody */
+                    $responseBody = curl_exec($fp);
+                    $responseHeaders .= "\r\n";
                 }
                 $this->status_code = curl_getinfo($fp, CURLINFO_HTTP_CODE);
                 if (curl_errno($fp)) {
                     $this->error = 'cURL error ' . curl_errno($fp) . ': ' . curl_error($fp);
                     $this->success = false;
-                    $this->on_http_response($responseHeaders);
+                    $this->on_http_response($responseBody === false ? false : $responseHeaders . $responseBody);
                 } else {
-                    $this->on_http_response($responseHeaders);
+                    $this->on_http_response($responseBody === false ? false : $responseHeaders . $responseBody);
                     // Use the updated url provided by curl_getinfo after any redirects.
                     if ($info = curl_getinfo($fp)) {
                         $this->url = $info['url'];
@@ -167,8 +176,7 @@ class File implements Response
                     $parser = new \SimplePie\HTTP\Parser($responseHeaders, true);
                     if ($parser->parse()) {
                         $this->set_headers($parser->headers);
-                        $this->body = $parser->body;
-                        $this->status_code = $parser->status_code;
+                        $this->body = $responseBody === false ? null : $responseBody;
                         if ((in_array($this->status_code, [300, 301, 302, 303, 307]) || $this->status_code > 307 && $this->status_code < 400) && ($locationHeader = $this->get_header_line('location')) !== '' && $this->redirects < $redirects) {
                             $this->redirects++;
                             $location = \SimplePie\Misc::absolutize_url($locationHeader, $url);

lib/simplepie/simplepie/src/HTTP/Parser.php

@@ -42,7 +42,7 @@ class Parser
     /**
      * Key/value pairs of the headers
      *
-     * @var (Psr7Compatible is true ? array<string, non-empty-array<string>> : array<string, string>)
+     * @phpstan-var (Psr7Compatible is true ? array<string, non-empty-array<string>> : array<string, string>)
      */
     public $headers = [];