$resolve_ok */ private static array $resolve_ok = []; /** @var array $retry_after_domain_wide */ private static array $retry_after_domain_wide = []; private static function getRetryAfterFile(string $url, string $proxy): string { $domain = parse_url($url, PHP_URL_HOST); if (!is_string($domain) || $domain === '') { return ''; } // Once per host, as serverIsPublic() may resolve it $domainWide = self::$retry_after_domain_wide[$domain] ??= Minz_Request::serverIsPublic($url, allowDnsResolve: $proxy === ''); $port = parse_url($url, PHP_URL_PORT); if (is_int($port)) { $domain .= ':' . $port; } return self::RETRY_AFTER_PATH . urlencode($domain) . ($domainWide ? '' : '_' . hash('sha256', $url)) . (empty($proxy) ? '' : '_' . urlencode($proxy)) . '.txt'; } /** * Clean up old Retry-After files */ private static function cleanRetryAfters(): void { if (!is_dir(self::RETRY_AFTER_PATH)) { return; } $files = glob(self::RETRY_AFTER_PATH . '*.txt', GLOB_NOSORT); if ($files === false) { return; } foreach ($files as $file) { if (@filemtime($file) < time()) { @unlink($file); } } } /** * Check whether the URL needs to wait for a Retry-After period. * @return int The timestamp of when the Retry-After expires, or 0 if not set. */ public static function getRetryAfter(string $url, string $proxy): int { if (rand(0, 30) === 1) { // Remove old files once in a while self::cleanRetryAfters(); } $txt = self::getRetryAfterFile($url, $proxy); if ($txt === '') { return 0; } $retryAfter = @filemtime($txt) ?: 0; if ($retryAfter <= 0) { return 0; } if ($retryAfter < time()) { @unlink($txt); return 0; } return $retryAfter; } /** * Store the HTTP Retry-After header value of an HTTP `429 Too Many Requests` or `503 Service Unavailable` response. */ public static function setRetryAfter(string $url, string $proxy, string $retryAfter): int { $txt = self::getRetryAfterFile($url, $proxy); if ($txt === '') { return 0; } $limits = FreshRSS_Context::systemConf()->limits; if (ctype_digit($retryAfter)) { $retryAfter = time() + (int)$retryAfter; } else { $retryAfter = \SimplePie\Misc::parse_date($retryAfter) ?: (time() + max(600, $limits['retry_after_default'] ?? 0)); } $retryAfter = min($retryAfter, time() + max(3600, $limits['retry_after_max'] ?? 0)); @mkdir(self::RETRY_AFTER_PATH); if (!touch($txt, $retryAfter)) { Minz_Log::error('Failed to set Retry-After for ' . $url); return 0; } return $retryAfter; } /** * @param array $curl_params * @return array */ public static function sanitizeCurlParams(array $curl_params): array { $safe_params = [ CURLOPT_COOKIE, CURLOPT_COOKIEFILE, CURLOPT_FOLLOWLOCATION, // We filter this value later, only allowing `false` CURLOPT_HTTPHEADER, CURLOPT_MAXREDIRS, CURLOPT_POST, CURLOPT_POSTFIELDS, CURLOPT_PROXY, CURLOPT_PROXYTYPE, CURLOPT_USERAGENT, ]; foreach ($curl_params as $k => $_) { if (!in_array($k, $safe_params, true)) { unset($curl_params[$k]); continue; } // Allow only an empty value just to enable the libcurl cookie engine if ($k === CURLOPT_COOKIEFILE) { $curl_params[$k] = ''; } // Remove HTTP authentication headers problematic for security if ($k === CURLOPT_HTTPHEADER && is_array($curl_params[$k])) { $curl_params[$k] = array_filter($curl_params[$k], fn($header) => is_string($header) && !preg_match('/^(Remote[-_\s]*User|X[-_\s]*WebAuth[-_\s]*User)\\s*:/i', $header)); } } return $curl_params; } private static function idn_to_puny(string $url): string { if (function_exists('idn_to_ascii')) { $idn = parse_url($url, PHP_URL_HOST); if (is_string($idn) && $idn != '') { $puny = idn_to_ascii($idn); $pos = strpos($url, $idn); if ($puny != false && $pos !== false) { $url = substr_replace($url, $puny, $pos, strlen($idn)); } } } return $url; } public static function checkUrl(string $url, bool $fixScheme = true): string|false { $url = trim($url); if ($url == '') { return ''; } if ($fixScheme && preg_match('#^https?://#i', $url) !== 1) { $url = 'https://' . ltrim($url, '/'); } $url = self::idn_to_puny($url); // https://bugs.php.net/bug.php?id=53474 $urlRelaxed = str_replace('_', 'z', $url); //PHP discussion #64948 Underscore if (is_string(filter_var($urlRelaxed, FILTER_VALIDATE_URL))) { return $url; } else { return false; } } /** * Remove the charset meta information of an HTML document, e.g.: * `` * `` */ private static function stripHtmlMetaCharset(string $html): string { return preg_replace('/]*charset\s*=\s*[^>]+>/i', '', $html, 1) ?? ''; } /** * Set an XML preamble to enforce the HTML content type charset received by HTTP. * @param string $html the raw downloaded HTML content * @param string $contentType an HTTP Content-Type such as 'text/html; charset=utf-8' * @return string an HTML string with XML encoding information for DOMDocument::loadHTML() */ private static function enforceHttpEncoding(string $html, string $contentType = ''): string { $httpCharset = preg_match('/\bcharset=([0-9a-z_-]{2,12})$/i', $contentType, $matches) === 1 ? $matches[1] : ''; if ($httpCharset == '') { // No charset defined by HTTP if (preg_match('/]*charset\s*=[\s\'"]*UTF-?8\b/i', substr($html, 0, 2048))) { // Detect UTF-8 even if declared too deep in HTML for DOMDocument $httpCharset = 'UTF-8'; } else { // Do nothing return $html; } } $httpCharsetNormalized = \SimplePie\Misc::encoding($httpCharset); if (in_array($httpCharsetNormalized, ['windows-1252', 'US-ASCII'], true)) { // Default charset for HTTP, do nothing return $html; } if (substr($html, 0, 3) === "\xEF\xBB\xBF" || // UTF-8 BOM substr($html, 0, 2) === "\xFF\xFE" || // UTF-16 Little Endian BOM substr($html, 0, 2) === "\xFE\xFF" || // UTF-16 Big Endian BOM substr($html, 0, 4) === "\xFF\xFE\x00\x00" || // UTF-32 Little Endian BOM substr($html, 0, 4) === "\x00\x00\xFE\xFF") { // UTF-32 Big Endian BOM // Existing byte order mark, do nothing return $html; } if (preg_match('/^<[?]xml[^>]+encoding\b/', substr($html, 0, 64))) { // Existing XML declaration, do nothing return $html; } if ($httpCharsetNormalized !== 'UTF-8') { // Try to change encoding to UTF-8 using mbstring or iconv or intl $utf8 = \SimplePie\Misc::change_encoding($html, $httpCharsetNormalized, 'UTF-8'); if (is_string($utf8)) { $html = self::stripHtmlMetaCharset($utf8); $httpCharsetNormalized = 'UTF-8'; } } if ($httpCharsetNormalized === 'UTF-8') { // Save encoding information as Unicode BOM return "\xEF\xBB\xBF" . $html; } // Give up return $html; } /** * Set an HTML base URL to the HTML content if there is none. * @param string $html the raw downloaded HTML content * @param string $href the HTML base URL * @return string an HTML string */ private static function enforceHtmlBase(string $html, string $href): string { $doc = new DOMDocument(); $doc->loadHTML($html, LIBXML_NONET | LIBXML_NOERROR | LIBXML_NOWARNING); if ($doc->documentElement === null) { return ''; } $xpath = new DOMXPath($doc); $bases = $xpath->evaluate('//base'); if (!($bases instanceof DOMNodeList) || $bases->length === 0) { $base = $doc->createElement('base'); if ($base === false) { return $html; } $base->setAttribute('href', $href); $head = null; $heads = $xpath->evaluate('//head'); if ($heads instanceof DOMNodeList && $heads->length > 0) { $head = $heads->item(0); } if ($head instanceof DOMElement) { $head->insertBefore($base, $head->firstChild); } else { $doc->documentElement->insertBefore($base, $doc->documentElement->firstChild); } } // Save the start of HTML because libxml2 saveHTML() risks scrambling it $htmlPos = stripos($html, ' 512 ? '' : substr($html, 0, $htmlPos); $encoding = strtolower($doc->encoding ?? ''); $bom = str_starts_with($html, "\xEF\xBB\xBF") ? "\xEF\xBB\xBF" : ''; if ($encoding === 'utf-8' || ($encoding === '' && $bom !== '')) { // For a whole document, libxml2 saveHTML() only knows the charset from . // With or a byte order mark, it writes non-ASCII characters as HTML entities, // also inside