|
@@ -306,12 +306,17 @@ final class FreshRSS_http_Util {
|
|
|
/**
|
|
/**
|
|
|
* Returns a value for CURLOPT_RESOLVE as an array, null if no allowed IPs were found, false if the domain failed to resolve.
|
|
* Returns a value for CURLOPT_RESOLVE as an array, null if no allowed IPs were found, false if the domain failed to resolve.
|
|
|
*
|
|
*
|
|
|
- * @return array<string>|null|false
|
|
|
|
|
|
|
+ * Can also be used for checking if the CURLOPT_PROXY value is allowed, by providing a proxy URL with the `for_proxy` parameter set to `true`.
|
|
|
|
|
+ * In that case, a string value will be returned with the hostname resolved to an IP if allowed.
|
|
|
|
|
+ *
|
|
|
|
|
+ * @return array<string>|string|null|false
|
|
|
*/
|
|
*/
|
|
|
- public static function getCurlResolveInfo(string $url): array|null|false {
|
|
|
|
|
|
|
+ public static function getCurlResolveInfo(string $url, bool $for_proxy = false): array|string|null|false {
|
|
|
|
|
+ // Parse the original URL first so that credentials keep their original case (only the host is case-insensitive).
|
|
|
|
|
+ $parsedOriginal = parse_url($url);
|
|
|
$url = strtolower($url);
|
|
$url = strtolower($url);
|
|
|
$parsed = parse_url($url);
|
|
$parsed = parse_url($url);
|
|
|
- if ($parsed === false) {
|
|
|
|
|
|
|
+ if ($parsed === false || $parsedOriginal === false) {
|
|
|
return false;
|
|
return false;
|
|
|
}
|
|
}
|
|
|
$host = $parsed['host'] ?? null;
|
|
$host = $parsed['host'] ?? null;
|
|
@@ -319,6 +324,12 @@ final class FreshRSS_http_Util {
|
|
|
if ($host === null || $scheme === null) {
|
|
if ($host === null || $scheme === null) {
|
|
|
return false;
|
|
return false;
|
|
|
}
|
|
}
|
|
|
|
|
+ $credentials = '';
|
|
|
|
|
+ $user = $parsedOriginal['user'] ?? null;
|
|
|
|
|
+ $pass = $parsedOriginal['pass'] ?? null;
|
|
|
|
|
+ if (is_string($user) && is_string($pass)) {
|
|
|
|
|
+ $credentials = "$user:$pass@";
|
|
|
|
|
+ }
|
|
|
if (str_starts_with($host, '[') && str_ends_with($host, ']')) {
|
|
if (str_starts_with($host, '[') && str_ends_with($host, ']')) {
|
|
|
if (strlen($host) === 2) {
|
|
if (strlen($host) === 2) {
|
|
|
return false;
|
|
return false;
|
|
@@ -334,15 +345,25 @@ final class FreshRSS_http_Util {
|
|
|
$internal_host_allowlist = FreshRSS_Context::systemConf()->internal_host_allowlist;
|
|
$internal_host_allowlist = FreshRSS_Context::systemConf()->internal_host_allowlist;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- if (in_array('*', $internal_host_allowlist, true)) {
|
|
|
|
|
- return []; // Disables SSRF checks entirely (unsafe)
|
|
|
|
|
- }
|
|
|
|
|
-
|
|
|
|
|
$port = parse_url($url)['port'] ?? match ($scheme) {
|
|
$port = parse_url($url)['port'] ?? match ($scheme) {
|
|
|
'http' => 80,
|
|
'http' => 80,
|
|
|
'https' => 443,
|
|
'https' => 443,
|
|
|
|
|
+ 'socks4' => 1080,
|
|
|
|
|
+ 'socks4a' => 1080,
|
|
|
|
|
+ 'socks5' => 1080,
|
|
|
|
|
+ 'socks5h' => 1080,
|
|
|
default => 0,
|
|
default => 0,
|
|
|
};
|
|
};
|
|
|
|
|
+ if (in_array('*', $internal_host_allowlist, true)) {
|
|
|
|
|
+ if ($for_proxy) {
|
|
|
|
|
+ if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !== false) {
|
|
|
|
|
+ return $credentials . "[$host]:$port";
|
|
|
|
|
+ }
|
|
|
|
|
+ return $credentials . "$host:$port";
|
|
|
|
|
+ }
|
|
|
|
|
+ return []; // Disables SSRF checks entirely (unsafe)
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
$resolve_str = "$host:$port:";
|
|
$resolve_str = "$host:$port:";
|
|
|
$ips_ok = [];
|
|
$ips_ok = [];
|
|
|
$ips = [];
|
|
$ips = [];
|
|
@@ -401,10 +422,20 @@ final class FreshRSS_http_Util {
|
|
|
|
|
|
|
|
if (count($ips_ok) > 0) {
|
|
if (count($ips_ok) > 0) {
|
|
|
if (count($records) > 0 || isset(self::$resolve_ok[$host])) {
|
|
if (count($records) > 0 || isset(self::$resolve_ok[$host])) {
|
|
|
|
|
+ if ($for_proxy) {
|
|
|
|
|
+ // $ips_ok[0] is already bracketed when it is an IPv6 address
|
|
|
|
|
+ return $credentials . "$ips_ok[0]:$port";
|
|
|
|
|
+ }
|
|
|
$resolve_str .= implode(',', $ips_ok);
|
|
$resolve_str .= implode(',', $ips_ok);
|
|
|
return [$resolve_str];
|
|
return [$resolve_str];
|
|
|
}
|
|
}
|
|
|
if (filter_var($host, FILTER_VALIDATE_IP) !== false) {
|
|
if (filter_var($host, FILTER_VALIDATE_IP) !== false) {
|
|
|
|
|
+ if ($for_proxy) {
|
|
|
|
|
+ if (filter_var($host, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !== false) {
|
|
|
|
|
+ return $credentials . "[$host]:$port";
|
|
|
|
|
+ }
|
|
|
|
|
+ return $credentials . "$host:$port";
|
|
|
|
|
+ }
|
|
|
// No resolve overrides since the URL only contained an IP, not a domain
|
|
// No resolve overrides since the URL only contained an IP, not a domain
|
|
|
return [];
|
|
return [];
|
|
|
}
|
|
}
|
|
@@ -453,10 +484,13 @@ final class FreshRSS_http_Util {
|
|
|
|
|
|
|
|
$accept = '';
|
|
$accept = '';
|
|
|
$proxy = is_string(FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] ?? null) ? FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] : '';
|
|
$proxy = is_string(FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] ?? null) ? FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] : '';
|
|
|
|
|
+ $proxy_type = is_int(FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXYTYPE] ?? null) ?
|
|
|
|
|
+ FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXYTYPE] : 0;
|
|
|
$options = []; // User-defined cURL options
|
|
$options = []; // User-defined cURL options
|
|
|
if (is_array($attributes['curl_params'] ?? null)) {
|
|
if (is_array($attributes['curl_params'] ?? null)) {
|
|
|
$options = self::sanitizeCurlParams($attributes['curl_params']);
|
|
$options = self::sanitizeCurlParams($attributes['curl_params']);
|
|
|
$proxy = is_string($options[CURLOPT_PROXY] ?? null) ? $options[CURLOPT_PROXY] : $proxy;
|
|
$proxy = is_string($options[CURLOPT_PROXY] ?? null) ? $options[CURLOPT_PROXY] : $proxy;
|
|
|
|
|
+ $proxy_type = is_int($options[CURLOPT_PROXYTYPE] ?? null) ? $options[CURLOPT_PROXYTYPE] : $proxy_type;
|
|
|
if (is_array($options[CURLOPT_HTTPHEADER] ?? null)) {
|
|
if (is_array($options[CURLOPT_HTTPHEADER] ?? null)) {
|
|
|
// Add Accept header if it is not set
|
|
// Add Accept header if it is not set
|
|
|
if (preg_grep('/^Accept\\s*:/i', $options[CURLOPT_HTTPHEADER]) === false) {
|
|
if (preg_grep('/^Accept\\s*:/i', $options[CURLOPT_HTTPHEADER]) === false) {
|
|
@@ -465,6 +499,7 @@ final class FreshRSS_http_Util {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
$proxy = is_string($curl_options[CURLOPT_PROXY] ?? null) ? $curl_options[CURLOPT_PROXY] : $proxy;
|
|
$proxy = is_string($curl_options[CURLOPT_PROXY] ?? null) ? $curl_options[CURLOPT_PROXY] : $proxy;
|
|
|
|
|
+ $proxy_type = is_int($curl_options[CURLOPT_PROXYTYPE] ?? null) ? $curl_options[CURLOPT_PROXYTYPE] : $proxy_type;
|
|
|
|
|
|
|
|
if (($retryAfter = FreshRSS_http_Util::getRetryAfter($url, $proxy)) > 0) {
|
|
if (($retryAfter = FreshRSS_http_Util::getRetryAfter($url, $proxy)) > 0) {
|
|
|
Minz_Log::warning('For that domain, will first retry after ' . date('c', $retryAfter) . '. ' . \SimplePie\Misc::url_remove_credentials($url));
|
|
Minz_Log::warning('For that domain, will first retry after ' . date('c', $retryAfter) . '. ' . \SimplePie\Misc::url_remove_credentials($url));
|
|
@@ -516,6 +551,38 @@ final class FreshRSS_http_Util {
|
|
|
if (!empty($resolve)) {
|
|
if (!empty($resolve)) {
|
|
|
$curl_options[CURLOPT_RESOLVE] = $resolve; // Prevent DNS rebinding
|
|
$curl_options[CURLOPT_RESOLVE] = $resolve; // Prevent DNS rebinding
|
|
|
}
|
|
}
|
|
|
|
|
+ } else {
|
|
|
|
|
+ defined('CURLPROXY_HTTPS') or define('CURLPROXY_HTTPS', 2); // Compatibility cURL 7.51
|
|
|
|
|
+ $proxy_scheme = match ($proxy_type) {
|
|
|
|
|
+ CURLPROXY_HTTP => 'http',
|
|
|
|
|
+ CURLPROXY_HTTPS => 'https',
|
|
|
|
|
+ CURLPROXY_SOCKS4 => 'socks4',
|
|
|
|
|
+ CURLPROXY_SOCKS4A => 'socks4a',
|
|
|
|
|
+ CURLPROXY_SOCKS5 => 'socks5',
|
|
|
|
|
+ CURLPROXY_SOCKS5_HOSTNAME => 'socks5h',
|
|
|
|
|
+ default => null,
|
|
|
|
|
+ };
|
|
|
|
|
+ if ($proxy_scheme === null) {
|
|
|
|
|
+ // Unsupported proxy type
|
|
|
|
|
+ return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => ''];
|
|
|
|
|
+ }
|
|
|
|
|
+ $proxy_url = "$proxy_scheme://$proxy"; // CURLOPT_PROXY ($proxy) is formatted as user:pass@hostname:port, with the part before @ being optional
|
|
|
|
|
+ $resolve = self::getCurlResolveInfo($proxy_url, for_proxy: true);
|
|
|
|
|
+ if ($resolve === null) {
|
|
|
|
|
+ Minz_Log::warning('Failed to fetch this URL, because the proxy’s IP is not in the allowlist [' .
|
|
|
|
|
+ \SimplePie\Misc::url_remove_credentials($url) . '] [' .
|
|
|
|
|
+ \SimplePie\Misc::url_remove_credentials($proxy_url) . ']');
|
|
|
|
|
+ return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => ''];
|
|
|
|
|
+ } elseif ($resolve === false) {
|
|
|
|
|
+ return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => ''];
|
|
|
|
|
+ }
|
|
|
|
|
+ // Translate from a hostname:port value to ip:port, in order to avoid DNS rebinding
|
|
|
|
|
+ $curl_options[CURLOPT_PROXY] = $resolve;
|
|
|
|
|
+ if (defined('CURLOPT_PROXY_SSL_VERIFYHOST')) {
|
|
|
|
|
+ // Skip verifying the hostname (a bit unsafe, but needed since
|
|
|
|
|
+ // there is no CURLOPT_RESOLVE equivalent for proxy hostnames)
|
|
|
|
|
+ $curl_options[CURLOPT_PROXY_SSL_VERIFYHOST] = 0;
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
// TODO: Implement HTTP 1.1 conditional GET If-Modified-Since
|
|
// TODO: Implement HTTP 1.1 conditional GET If-Modified-Since
|
|
|
$ch = curl_init();
|
|
$ch = curl_init();
|