SimplePie: apply HTTPS policy also on GUIDs and permalinks (#8037)

* SimplePie: apply HTTPS policy also on GUIDs and permalinks
Fix WordPress duplicates HTTP vs. HTTPS
fix https://github.com/FreshRSS/FreshRSS/issues/5151

* Move code upstream
* https://github.com/simplepie/simplepie/pull/951
* https://github.com/FreshRSS/simplepie/pull/59

lib/composer.json

@@ -14,7 +14,7 @@
         "marienfressinaud/lib_opml": "0.5.1",
         "phpgt/cssxpath": "v1.3.0",
         "phpmailer/phpmailer": "6.11.1",
-        "simplepie/simplepie": "dev-freshrss#c1bf1a353dae742977dde34d65e4c89b633a9b47"
+        "simplepie/simplepie": "dev-freshrss#24cfb0c6d81f81ef110c8257d3464b2649476c77"
     },
     "config": {
         "sort-packages": true,

lib/simplepie/simplepie/src/File.php

@@ -134,16 +134,13 @@ class File implements Response
                 curl_setopt($fp, CURLOPT_HTTPHEADER, $headers2);
                 $responseHeaders = '';
                 curl_setopt($fp, CURLOPT_HEADERFUNCTION, function ($ch, string $header) use (&$responseHeaders) {
-                    if (trim($header) !== '') { // Skip e.g. separation with trailer headers
-                        $responseHeaders .= $header;
-                    }
+                    $responseHeaders .= $header;
                     return strlen($header);
                 });
                 foreach ($curl_options as $curl_param => $curl_value) {
                     curl_setopt($fp, $curl_param, $curl_value);
                 }
 
-                /** @var string|false $responseBody */
                 $responseBody = curl_exec($fp);
                 $responseHeaders .= "\r\n";
                 if (curl_errno($fp) === CURLE_WRITE_ERROR || curl_errno($fp) === CURLE_BAD_CONTENT_ENCODING) {
@@ -152,31 +149,29 @@ class File implements Response
                     $this->error = null; // FreshRSS
                     curl_setopt($fp, CURLOPT_ENCODING, 'none');
                     $responseHeaders = '';
-                    /** @var string|false $responseBody */
                     $responseBody = curl_exec($fp);
                     $responseHeaders .= "\r\n";
                 }
                 $this->status_code = curl_getinfo($fp, CURLINFO_HTTP_CODE);
-                if (curl_errno($fp)) {
+                if (curl_errno($fp) !== CURLE_OK) {
                     $this->error = 'cURL error ' . curl_errno($fp) . ': ' . curl_error($fp);
                     $this->success = false;
                     $this->on_http_response($responseBody === false ? false : $responseHeaders . $responseBody);
                 } else {
-                    $this->on_http_response($responseBody === false ? false : $responseHeaders . $responseBody);
-                    // Use the updated url provided by curl_getinfo after any redirects.
-                    if ($info = curl_getinfo($fp)) {
-                        $this->url = $info['url'];
+                    // For PHPStan: `curl_exec` returns `false` only on error so the `is_string` check will always pass.
+                    \assert(is_string($responseBody));
+                    if (curl_getinfo($fp, CURLINFO_HTTP_CONNECTCODE) !== 0) {
+                        // TODO: Replace with `CURLOPT_SUPPRESS_CONNECT_HEADERS` once PHP 7.2 support is dropped.
+                        $responseHeaders = \SimplePie\HTTP\Parser::prepareHeaders($responseHeaders);
                     }
-                    // For PHPStan: We already checked that error did not occur.
-                    assert(is_array($info) && $info['redirect_count'] >= 0);
+                    $this->on_http_response($responseHeaders . $responseBody);
                     if (\PHP_VERSION_ID < 80000) {
                         curl_close($fp);
                     }
-                    $responseHeaders = \SimplePie\HTTP\Parser::prepareHeaders((string) $responseHeaders, $info['redirect_count'] + 1);
                     $parser = new \SimplePie\HTTP\Parser($responseHeaders, true);
                     if ($parser->parse()) {
                         $this->set_headers($parser->headers);
-                        $this->body = $responseBody === false ? null : $responseBody;
+                        $this->body = $responseBody;
                         if ((in_array($this->status_code, [300, 301, 302, 303, 307]) || $this->status_code > 307 && $this->status_code < 400) && ($locationHeader = $this->get_header_line('location')) !== '' && $this->redirects < $redirects) {
                             $this->redirects++;
                             $location = \SimplePie\Misc::absolutize_url($locationHeader, $url);

lib/simplepie/simplepie/src/IRI.php

@@ -170,7 +170,7 @@ class IRI
             $return = null;
         }
 
-        if ($return === null && isset($this->normalization[$this->scheme][$name])) {
+        if ($return === null && isset($this->scheme, $this->normalization[$this->scheme][$name])) {
             return $this->normalization[$this->scheme][$name];
         }
 
@@ -623,6 +623,10 @@ class IRI
      */
     protected function scheme_normalization()
     {
+        if ($this->scheme === null) {
+            return;
+        }
+
         if (isset($this->normalization[$this->scheme]['iuserinfo']) && $this->iuserinfo === $this->normalization[$this->scheme]['iuserinfo']) {
             $this->iuserinfo = null;
         }

lib/simplepie/simplepie/src/Item.php

@@ -201,18 +201,23 @@ class Item implements RegistryAware
     public function get_id(bool $hash = false, $fn = 'md5')
     {
         if (!$hash) {
+            $guid = '';
             if ($return = $this->get_item_tags(\SimplePie\SimplePie::NAMESPACE_ATOM_10, 'id')) {
-                return $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
             } elseif ($return = $this->get_item_tags(\SimplePie\SimplePie::NAMESPACE_ATOM_03, 'id')) {
-                return $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
             } elseif ($return = $this->get_item_tags(\SimplePie\SimplePie::NAMESPACE_RSS_20, 'guid')) {
-                return $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
             } elseif ($return = $this->get_item_tags(\SimplePie\SimplePie::NAMESPACE_DC_11, 'identifier')) {
-                return $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
             } elseif ($return = $this->get_item_tags(\SimplePie\SimplePie::NAMESPACE_DC_10, 'identifier')) {
-                return $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($return[0]['data'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
             } elseif (isset($this->data['attribs'][\SimplePie\SimplePie::NAMESPACE_RDF]['about'])) {
-                return $this->sanitize($this->data['attribs'][\SimplePie\SimplePie::NAMESPACE_RDF]['about'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+                $guid = $this->sanitize($this->data['attribs'][\SimplePie\SimplePie::NAMESPACE_RDF]['about'], \SimplePie\SimplePie::CONSTRUCT_TEXT);
+            }
+            if ($guid !== '') {
+                // If the ID looks like a URL, apply HTTPS policy to it.
+                return $this->get_sanitize()->https_url($guid);
             }
         }
         if ($fn === false) {
@@ -877,6 +882,13 @@ class Item implements RegistryAware
                 }
                 $this->data['links'][$key] = array_unique($this->data['links'][$key]);
             }
+
+            // Apply HTTPS policy to all links
+            foreach ($this->data['links'] as &$links) {
+                foreach ($links as &$link) {
+                    $link = $this->get_sanitize()->https_url($link);
+                }
+            }
         }
         if (isset($this->data['links'][$rel])) {
             return $this->data['links'][$rel];
@@ -2353,7 +2365,7 @@ class Item implements RegistryAware
         $this->sanitize = $sanitize;
     }
 
-    protected function get_sanitize(): Sanitize
+    public function get_sanitize(): Sanitize
     {
         if ($this->sanitize === null) {
             $this->sanitize = new Sanitize();

lib/simplepie/simplepie/src/Locator.php

@@ -112,7 +112,7 @@ class Locator implements RegistryAware
     }
 
     /**
-     * @param SimplePie::LOCATOR_* $type
+     * @param int-mask-of<SimplePie::LOCATOR_*> $type
      * @param array<Response>|null $working
      * @return Response|null
      */

lib/simplepie/simplepie/src/SimplePie.php

@@ -568,7 +568,7 @@ class SimplePie
     public $input_encoding = false;
 
     /**
-     * @var self::LOCATOR_* Feed Autodiscovery Level
+     * @var int-mask-of<self::LOCATOR_*> Feed Autodiscovery Level
      * @see SimplePie::set_autodiscovery_level()
      * @access private
      */
@@ -1122,7 +1122,7 @@ class SimplePie
      * @see self::LOCATOR_REMOTE_EXTENSION
      * @see self::LOCATOR_REMOTE_BODY
      * @see self::LOCATOR_ALL
-     * @param self::LOCATOR_* $level Feed Autodiscovery Level (level can be a combination of the above constants, see bitwise OR operator)
+     * @param int-mask-of<self::LOCATOR_*> $level Feed Autodiscovery Level (level can be a combination of the above constants, see bitwise OR operator)
      * @return void
      */
     public function set_autodiscovery_level(int $level = self::LOCATOR_ALL)
@@ -2983,6 +2983,13 @@ class SimplePie
                 }
                 $this->data['links'][$key] = array_unique($this->data['links'][$key]);
             }
+
+            // Apply HTTPS policy to all links
+            foreach ($this->data['links'] as &$links) {
+                foreach ($links as &$link) {
+                    $link = $this->sanitize->https_url($link);
+                }
+            }
         }
 
         if (isset($this->data['headers']['link'])) {

lib/simplepie/simplepie/src/Source.php

@@ -387,6 +387,13 @@ class Source implements RegistryAware
                 }
                 $this->data['links'][$key] = array_unique($this->data['links'][$key]);
             }
+
+            // Apply HTTPS policy to all links
+            foreach ($this->data['links'] as &$links) {
+                foreach ($links as &$link) {
+                    $link = $this->item->get_sanitize()->https_url($link);
+                }
+            }
         }
 
         if (isset($this->data['links'][$rel])) {