4
0

httpUtil.php 32 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855
  1. <?php
  2. declare(strict_types=1);
  3. final class FreshRSS_http_Util {
  4. private const RETRY_AFTER_PATH = DATA_PATH . '/Retry-After/';
  5. private const NAT64_PREFIX_RFC6052 = '64:ff9b::/96';
  6. private const NAT64_PREFIX_RFC8215 = '64:ff9b:1::/48';
  7. /** @var array<string, string[]> $resolve_ok */
  8. private static array $resolve_ok = [];
  9. /** @var array<string, bool> $retry_after_domain_wide */
  10. private static array $retry_after_domain_wide = [];
  11. private static function getRetryAfterFile(string $url, string $proxy): string {
  12. $domain = parse_url($url, PHP_URL_HOST);
  13. if (!is_string($domain) || $domain === '') {
  14. return '';
  15. }
  16. // Once per host, as serverIsPublic() may resolve it
  17. $domainWide = self::$retry_after_domain_wide[$domain] ??= Minz_Request::serverIsPublic($url, allowDnsResolve: $proxy === '');
  18. $port = parse_url($url, PHP_URL_PORT);
  19. if (is_int($port)) {
  20. $domain .= ':' . $port;
  21. }
  22. return self::RETRY_AFTER_PATH . urlencode($domain) .
  23. ($domainWide ? '' : '_' . hash('sha256', $url)) .
  24. (empty($proxy) ? '' : '_' . urlencode($proxy)) . '.txt';
  25. }
  26. /**
  27. * Clean up old Retry-After files
  28. */
  29. private static function cleanRetryAfters(): void {
  30. if (!is_dir(self::RETRY_AFTER_PATH)) {
  31. return;
  32. }
  33. $files = glob(self::RETRY_AFTER_PATH . '*.txt', GLOB_NOSORT);
  34. if ($files === false) {
  35. return;
  36. }
  37. foreach ($files as $file) {
  38. if (@filemtime($file) < time()) {
  39. @unlink($file);
  40. }
  41. }
  42. }
  43. /**
  44. * Check whether the URL needs to wait for a Retry-After period.
  45. * @return int The timestamp of when the Retry-After expires, or 0 if not set.
  46. */
  47. public static function getRetryAfter(string $url, string $proxy): int {
  48. if (rand(0, 30) === 1) { // Remove old files once in a while
  49. self::cleanRetryAfters();
  50. }
  51. $txt = self::getRetryAfterFile($url, $proxy);
  52. if ($txt === '') {
  53. return 0;
  54. }
  55. $retryAfter = @filemtime($txt) ?: 0;
  56. if ($retryAfter <= 0) {
  57. return 0;
  58. }
  59. if ($retryAfter < time()) {
  60. @unlink($txt);
  61. return 0;
  62. }
  63. return $retryAfter;
  64. }
  65. /**
  66. * Store the HTTP Retry-After header value of an HTTP `429 Too Many Requests` or `503 Service Unavailable` response.
  67. */
  68. public static function setRetryAfter(string $url, string $proxy, string $retryAfter): int {
  69. $txt = self::getRetryAfterFile($url, $proxy);
  70. if ($txt === '') {
  71. return 0;
  72. }
  73. $limits = FreshRSS_Context::systemConf()->limits;
  74. if (ctype_digit($retryAfter)) {
  75. $retryAfter = time() + (int)$retryAfter;
  76. } else {
  77. $retryAfter = \SimplePie\Misc::parse_date($retryAfter) ?:
  78. (time() + max(600, $limits['retry_after_default'] ?? 0));
  79. }
  80. $retryAfter = min($retryAfter, time() + max(3600, $limits['retry_after_max'] ?? 0));
  81. @mkdir(self::RETRY_AFTER_PATH);
  82. if (!touch($txt, $retryAfter)) {
  83. Minz_Log::error('Failed to set Retry-After for ' . $url);
  84. return 0;
  85. }
  86. return $retryAfter;
  87. }
  88. /**
  89. * @param array<mixed> $curl_params
  90. * @return array<mixed>
  91. */
  92. public static function sanitizeCurlParams(array $curl_params): array {
  93. $safe_params = [
  94. CURLOPT_COOKIE,
  95. CURLOPT_COOKIEFILE,
  96. CURLOPT_FOLLOWLOCATION, // We filter this value later, only allowing `false`
  97. CURLOPT_HTTPHEADER,
  98. CURLOPT_MAXREDIRS,
  99. CURLOPT_POST,
  100. CURLOPT_POSTFIELDS,
  101. CURLOPT_PROXY,
  102. CURLOPT_PROXYTYPE,
  103. CURLOPT_USERAGENT,
  104. ];
  105. foreach ($curl_params as $k => $_) {
  106. if (!in_array($k, $safe_params, true)) {
  107. unset($curl_params[$k]);
  108. continue;
  109. }
  110. // Allow only an empty value just to enable the libcurl cookie engine
  111. if ($k === CURLOPT_COOKIEFILE) {
  112. $curl_params[$k] = '';
  113. }
  114. // Remove HTTP authentication headers problematic for security
  115. if ($k === CURLOPT_HTTPHEADER && is_array($curl_params[$k])) {
  116. $curl_params[$k] = array_filter($curl_params[$k],
  117. fn($header) => is_string($header) && !preg_match('/^(Remote[-_\s]*User|X[-_\s]*WebAuth[-_\s]*User)\\s*:/i', $header));
  118. }
  119. // Forbid '/' in CURLOPT_PROXY [user[:pass]@]host[:port] (the scheme comes from CURLOPT_PROXYTYPE),
  120. // to reject e.g. 'http://example.net/://decoy' and 'localhost:9050/path/to.sock'
  121. if ($k === CURLOPT_PROXY && (!is_string($curl_params[$k]) || str_contains($curl_params[$k], '/'))) {
  122. unset($curl_params[$k], $curl_params[CURLOPT_PROXYTYPE]);
  123. }
  124. }
  125. return $curl_params;
  126. }
  127. private static function idn_to_puny(string $url): string {
  128. if (function_exists('idn_to_ascii')) {
  129. $idn = parse_url($url, PHP_URL_HOST);
  130. if (is_string($idn) && $idn != '') {
  131. $puny = idn_to_ascii($idn);
  132. $pos = strpos($url, $idn);
  133. if ($pos !== false && is_string($puny) && preg_match('/^[a-z0-9._-]+$/iD', $puny) === 1) {
  134. $url = substr_replace($url, $puny, $pos, strlen($idn));
  135. }
  136. }
  137. }
  138. return $url;
  139. }
  140. public static function checkUrl(string $url, bool $fixScheme = true): string|false {
  141. $url = trim($url);
  142. if ($url == '') {
  143. return '';
  144. }
  145. if ($fixScheme && preg_match('#^https?://#i', $url) !== 1) {
  146. $url = 'https://' . ltrim($url, '/');
  147. }
  148. $url = self::idn_to_puny($url); // https://bugs.php.net/bug.php?id=53474
  149. $urlRelaxed = str_replace('_', 'z', $url); //PHP discussion #64948 Underscore
  150. if (is_string(filter_var($urlRelaxed, FILTER_VALIDATE_URL))) {
  151. return $url;
  152. } else {
  153. return false;
  154. }
  155. }
  156. /**
  157. * Remove the charset meta information of an HTML document, e.g.:
  158. * `<meta charset="..." />`
  159. * `<meta http-equiv="Content-Type" content="text/html; charset=...">`
  160. */
  161. private static function stripHtmlMetaCharset(string $html): string {
  162. return preg_replace('/<meta\s[^>]*charset\s*=\s*[^>]+>/i', '', $html, 1) ?? '';
  163. }
  164. /**
  165. * Set an XML preamble to enforce the HTML content type charset received by HTTP.
  166. * @param string $html the raw downloaded HTML content
  167. * @param string $contentType an HTTP Content-Type such as 'text/html; charset=utf-8'
  168. * @return string an HTML string with Unicode BOM encoding information
  169. */
  170. private static function enforceHttpEncoding(string $html, string $contentType = ''): string {
  171. $httpCharset = preg_match('/\bcharset=([0-9a-z_-]{2,12})$/i', $contentType, $matches) === 1 ? $matches[1] : '';
  172. if ($httpCharset == '') {
  173. // No charset defined by HTTP
  174. if (preg_match('/<meta\s[^>]*charset\s*=[\s\'"]*UTF-?8\b/i', substr($html, 0, 2048))) {
  175. // Detect UTF-8 even if declared too deep in HTML for DOMDocument
  176. $httpCharset = 'UTF-8';
  177. } else {
  178. // Do nothing
  179. return $html;
  180. }
  181. }
  182. $httpCharsetNormalized = \SimplePie\Misc::encoding($httpCharset);
  183. if (in_array($httpCharsetNormalized, ['windows-1252', 'US-ASCII'], true)) {
  184. // Default charset for HTTP, do nothing
  185. return $html;
  186. }
  187. if (substr($html, 0, 3) === "\xEF\xBB\xBF" || // UTF-8 BOM
  188. substr($html, 0, 2) === "\xFF\xFE" || // UTF-16 Little Endian BOM
  189. substr($html, 0, 2) === "\xFE\xFF" || // UTF-16 Big Endian BOM
  190. substr($html, 0, 4) === "\xFF\xFE\x00\x00" || // UTF-32 Little Endian BOM
  191. substr($html, 0, 4) === "\x00\x00\xFE\xFF") { // UTF-32 Big Endian BOM
  192. // Existing byte order mark, do nothing
  193. return $html;
  194. }
  195. if (preg_match('/^<[?]xml[^>]+encoding\b/', substr($html, 0, 64))) {
  196. // Existing XML declaration, do nothing
  197. return $html;
  198. }
  199. if ($httpCharsetNormalized !== 'UTF-8') {
  200. // Try to change encoding to UTF-8 using mbstring or iconv or intl
  201. $utf8 = \SimplePie\Misc::change_encoding($html, $httpCharsetNormalized, 'UTF-8');
  202. if (is_string($utf8)) {
  203. $html = self::stripHtmlMetaCharset($utf8);
  204. $httpCharsetNormalized = 'UTF-8';
  205. }
  206. }
  207. if ($httpCharsetNormalized === 'UTF-8') {
  208. // Save encoding information as Unicode BOM
  209. return "\xEF\xBB\xBF" . $html;
  210. }
  211. // Give up
  212. return $html;
  213. }
  214. /**
  215. * Set an HTML base URL to the HTML content if there is none.
  216. * @param string $html the raw downloaded HTML content
  217. * @param string $href the HTML base URL
  218. * @return string an HTML string
  219. */
  220. private static function enforceHtmlBase(string $html, string $href): string {
  221. $doc = Dom\HTMLDocument::createFromString($html, LIBXML_NOERROR);
  222. $documentElement = $doc->documentElement;
  223. if ($documentElement === null) {
  224. return '';
  225. }
  226. if ($doc->querySelector('base') === null) {
  227. $base = $doc->createElement('base');
  228. $base->setAttribute('href', $href);
  229. $head = $doc->querySelector('head');
  230. if ($head instanceof Dom\Element) {
  231. $head->insertBefore($base, $head->firstChild);
  232. } else {
  233. $documentElement->insertBefore($base, $documentElement->firstChild);
  234. }
  235. }
  236. $saved = $doc->saveHTML();
  237. if ($saved === '') {
  238. return $html;
  239. }
  240. // Re-add the Unicode byte order mark, for downstream charset detection
  241. return (str_starts_with($html, "\xEF\xBB\xBF") ? "\xEF\xBB\xBF" : '') . $saved;
  242. }
  243. public static function compareURLOrigins(string $url1, string $url2): bool {
  244. $url1 = parse_url(strtolower($url1));
  245. $url2 = parse_url(strtolower($url2));
  246. if ($url1 === false || $url2 === false) {
  247. return false;
  248. }
  249. foreach ([&$url1, &$url2] as &$url) {
  250. $url['port'] ??= match ($url['scheme'] ?? '') {
  251. 'http' => 80,
  252. 'https' => 443,
  253. default => 0,
  254. };
  255. }
  256. return ($url1['scheme'] ?? '') === ($url2['scheme'] ?? '') &&
  257. ($url1['host'] ?? '') === ($url2['host'] ?? '') &&
  258. ($url1['port'] ?? '') === ($url2['port'] ?? '');
  259. }
  260. /**
  261. * Return 0 if values on either side are equal ignoring the HTTP vs HTTPS differences, or 1/-1 if they differ.
  262. */
  263. public static function compareUrlIgnoringHttps(string $url1, string $url2): int {
  264. $normalizeScheme = static fn(string $url): string => preg_replace('#^https?://#i', '//', trim($url)) ?? $url;
  265. return $normalizeScheme($url1) <=> $normalizeScheme($url2);
  266. }
  267. /**
  268. * Returns a value for CURLOPT_RESOLVE as an array, null if no allowed IPs were found, false if the domain failed to resolve.
  269. *
  270. * @return array<string>|null|false
  271. */
  272. public static function getCurlResolveInfo(string $url): array|null|false {
  273. $url = strtolower($url);
  274. $parsed = parse_url($url);
  275. if ($parsed === false) {
  276. return false;
  277. }
  278. $host = $parsed['host'] ?? null;
  279. $scheme = $parsed['scheme'] ?? null;
  280. if ($host === null || $scheme === null) {
  281. return false;
  282. }
  283. if (str_starts_with($host, '[') && str_ends_with($host, ']')) {
  284. if (strlen($host) === 2) {
  285. return false;
  286. }
  287. $host = substr($host, 1, strlen($host) - 2);
  288. }
  289. $internal_host_allowlist = getenv('INTERNAL_HOST_ALLOWLIST');
  290. if (is_string($internal_host_allowlist) && $internal_host_allowlist !== '') {
  291. $internal_host_allowlist = preg_split('/\s+/', $internal_host_allowlist, -1, PREG_SPLIT_NO_EMPTY);
  292. }
  293. if (!is_array($internal_host_allowlist) || empty($internal_host_allowlist)) {
  294. $internal_host_allowlist = FreshRSS_Context::systemConf()->internal_host_allowlist;
  295. }
  296. $port = parse_url($url)['port'] ?? match ($scheme) {
  297. 'http' => 80,
  298. 'https' => 443,
  299. 'socks4' => 1080,
  300. 'socks4a' => 1080,
  301. 'socks5' => 1080,
  302. 'socks5h' => 1080,
  303. default => 0,
  304. };
  305. if (in_array('*', $internal_host_allowlist, true)) {
  306. return []; // Disables SSRF checks entirely (unsafe)
  307. }
  308. $resolve_str = "$host:$port:";
  309. $ips_ok = [];
  310. $ips = [];
  311. $records = [];
  312. if (filter_var($host, FILTER_VALIDATE_IP) !== false) {
  313. $ips[] = $host;
  314. } elseif (isset(self::$resolve_ok[$host])) {
  315. $ips = self::$resolve_ok[$host];
  316. } else {
  317. $records = @dns_get_record($host, DNS_A + DNS_AAAA);
  318. if ($records === false) {
  319. return false;
  320. }
  321. foreach ($records as $record) {
  322. $ip = $record['ip'] ?? $record['ipv6'];
  323. if (is_string($ip)) {
  324. $ips[] = $ip;
  325. }
  326. }
  327. self::$resolve_ok[$host] = $ips;
  328. }
  329. $cidr_allowlist = array_filter($internal_host_allowlist, fn($v, $_) => str_contains($v, '/'), ARRAY_FILTER_USE_BOTH);
  330. foreach ($ips as $ip) {
  331. $allowlist_str = "$ip:$port";
  332. $add_ip = $ip;
  333. if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) !== false) {
  334. $allowlist_str = "[$ip]:$port";
  335. $add_ip = "[$ip]";
  336. }
  337. foreach ($cidr_allowlist as $cidr) {
  338. if (self::checkCIDR($ip, $cidr)) {
  339. $ips_ok[] = $add_ip;
  340. continue 2;
  341. }
  342. }
  343. if (in_array($allowlist_str, $internal_host_allowlist, true) ||
  344. in_array("$host:$port", $internal_host_allowlist, true)) {
  345. $ips_ok[] = $add_ip;
  346. continue;
  347. }
  348. // NAT64 addresses use the RFC 6052 well-known prefix.
  349. // Validate the embedded IPv4 address because the NAT64 IPv6 address itself
  350. // is globally routable even when it maps to a private/reserved IPv4 address.
  351. if (self::checkCIDR($ip, self::NAT64_PREFIX_RFC6052)) {
  352. $packed = @inet_pton($ip);
  353. if ($packed === false || strlen($packed) !== 16) {
  354. continue;
  355. }
  356. $embedded_ipv4 = @inet_ntop(substr($packed, 12, 4));
  357. if ($embedded_ipv4 === false) {
  358. continue;
  359. }
  360. if (filter_var($embedded_ipv4, FILTER_VALIDATE_IP, FILTER_FLAG_GLOBAL_RANGE) === false) {
  361. continue;
  362. }
  363. $ips_ok[] = $add_ip;
  364. continue;
  365. }
  366. // `FILTER_FLAG_GLOBAL_RANGE` does not reject the NAT64 prefixes
  367. if (filter_var($ip, FILTER_VALIDATE_IP, FILTER_FLAG_GLOBAL_RANGE) === false ||
  368. self::checkCIDR($ip, self::NAT64_PREFIX_RFC8215)) {
  369. continue;
  370. }
  371. $ips_ok[] = $add_ip;
  372. }
  373. if (count($ips_ok) > 0) {
  374. if (count($records) > 0 || isset(self::$resolve_ok[$host])) {
  375. $resolve_str .= implode(',', $ips_ok);
  376. return [$resolve_str];
  377. }
  378. if (filter_var($host, FILTER_VALIDATE_IP) !== false) {
  379. // No resolve overrides since the URL only contained an IP, not a domain
  380. return [];
  381. }
  382. }
  383. if (count($ips) === 0) {
  384. return false;
  385. }
  386. return null;
  387. }
  388. /**
  389. * Whether cURL sends the request as a HTTP POST (`CURLOPT_POST` or `CURLOPT_POSTFIELDS`)
  390. * @param array<mixed> $user_options User-defined cURL options derived from `curl_params`
  391. * @param array<mixed> $curl_options Internal cURL options
  392. */
  393. private static function isPost(array $user_options, array $curl_options): bool {
  394. return !empty($user_options[CURLOPT_POST]) || !empty($curl_options[CURLOPT_POST]) ||
  395. isset($user_options[CURLOPT_POSTFIELDS]) || isset($curl_options[CURLOPT_POSTFIELDS]);
  396. }
  397. /**
  398. * @param non-empty-string $url
  399. * @param string|null $cachePath path to cache file, or `null` to disable caching
  400. * @param string $type {html,ico,json,opml,xml}
  401. * @param array<string,mixed> $attributes May contain user-defined cURL options in `$attributes['curl_params']`
  402. * @param array<int,mixed> $curl_options Internal overrides of cURL options
  403. * @return array{body:string,effective_url:string,redirect_count:int,fail:bool,status:int,error:string,method:string}
  404. * `status` is the HTTP response code (e.g. 200, 404), or a custom negative value:
  405. * * `-200` served from local cache;
  406. * * `-429` blocked by active `Retry-After` period;
  407. * * `-500` `curl_init()` failure, or request refused by a security check.
  408. *
  409. * `method` is the HTTP method used for the request producing `status` ('GET' or 'POST').
  410. */
  411. public static function httpGet(string $url, ?string $cachePath = null, string $type = 'html', array $attributes = [], array $curl_options = []): array {
  412. $options = is_array($attributes['curl_params'] ?? null) ?
  413. self::sanitizeCurlParams($attributes['curl_params']) : []; // User-defined cURL options
  414. $c_method = self::isPost($options, $curl_options) ? 'POST' : 'GET';
  415. if (!\SimplePie\Misc::is_remote_uri($url)) {
  416. Minz_Log::warning('Error fetching content: malformed URL “' . $url . '“');
  417. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  418. }
  419. $limits = FreshRSS_Context::systemConf()->limits;
  420. $feed_timeout = empty($attributes['timeout']) || !is_numeric($attributes['timeout']) ? 0 : intval($attributes['timeout']);
  421. if ($cachePath !== null) {
  422. $cacheMtime = @filemtime($cachePath);
  423. if ($cacheMtime !== false && $cacheMtime > time() - intval($limits['cache_duration'])) {
  424. $body = @file_get_contents($cachePath);
  425. if ($body != false) {
  426. if (FreshRSS_Context::systemConf()->simplepie_syslog_enabled) {
  427. syslog(LOG_DEBUG, 'FreshRSS uses cache for ' . \SimplePie\Misc::url_remove_credentials($url));
  428. }
  429. return ['body' => $body, 'effective_url' => $url, 'redirect_count' => 0, 'fail' => false, 'status' => -200,
  430. 'error' => '', 'method' => $c_method];
  431. }
  432. }
  433. }
  434. if (rand(0, 30) === 1) { // Remove old cache once in a while
  435. cleanCache(CLEANCACHE_HOURS);
  436. }
  437. $accept = '';
  438. $proxy = is_string(FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] ?? null) ? FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXY] : '';
  439. $proxy_type = is_int(FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXYTYPE] ?? null) ?
  440. FreshRSS_Context::systemConf()->curl_options[CURLOPT_PROXYTYPE] : 0;
  441. if (!empty($options)) {
  442. $proxy = is_string($options[CURLOPT_PROXY] ?? null) ? $options[CURLOPT_PROXY] : $proxy;
  443. $proxy_type = is_int($options[CURLOPT_PROXYTYPE] ?? null) ? $options[CURLOPT_PROXYTYPE] : $proxy_type;
  444. if (is_array($options[CURLOPT_HTTPHEADER] ?? null)) {
  445. // Add Accept header if it is not set
  446. if (preg_grep('/^Accept\\s*:/i', $options[CURLOPT_HTTPHEADER]) === false) {
  447. $options[CURLOPT_HTTPHEADER][] = 'Accept: ' . $accept;
  448. }
  449. }
  450. }
  451. $proxy_type = is_int($curl_options[CURLOPT_PROXYTYPE] ?? null) ? $curl_options[CURLOPT_PROXYTYPE] : $proxy_type;
  452. // CURLOPT_PROXY is already scheme-less by the time it reaches here:
  453. $proxy = is_string($curl_options[CURLOPT_PROXY] ?? null) ? $curl_options[CURLOPT_PROXY] : $proxy;
  454. if (($retryAfter = FreshRSS_http_Util::getRetryAfter($url, $proxy)) > 0) {
  455. Minz_Log::warning('For that domain, will first retry after ' . date('c', $retryAfter) . '. ' . \SimplePie\Misc::url_remove_credentials($url));
  456. return ['body' => '', 'effective_url' => $url, 'redirect_count' => 0, 'fail' => true, 'status' => -429, 'error' => '', 'method' => $c_method];
  457. }
  458. if (FreshRSS_Context::systemConf()->simplepie_syslog_enabled) {
  459. syslog(LOG_INFO, 'FreshRSS ' . $c_method . ' ' . $type . ' ' . \SimplePie\Misc::url_remove_credentials($url));
  460. }
  461. switch ($type) {
  462. case 'json':
  463. $accept = 'application/json,application/feed+json,application/javascript;q=0.9,text/javascript;q=0.8,*/*;q=0.7';
  464. break;
  465. case 'opml':
  466. $accept = 'text/x-opml,text/xml;q=0.9,application/xml;q=0.9,*/*;q=0.8';
  467. break;
  468. case 'xml':
  469. $accept = 'application/xml,application/xhtml+xml,text/xml;q=0.9,*/*;q=0.8';
  470. break;
  471. case 'ico':
  472. $accept = 'image/x-icon,image/vnd.microsoft.icon,image/ico,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.1';
  473. break;
  474. case 'html':
  475. default:
  476. $accept = 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8';
  477. break;
  478. }
  479. $original_url = $url;
  480. $fail = false;
  481. $redirs = 0;
  482. $max_redirs = $curl_options[CURLOPT_MAXREDIRS] ?? $options[CURLOPT_MAXREDIRS] ?? FreshRSS_Context::systemConf()->curl_options[CURLOPT_MAXREDIRS] ?? null;
  483. if (!is_int($max_redirs)) {
  484. $max_redirs = 4;
  485. }
  486. while (true) {
  487. $c_method = self::isPost($options, $curl_options) ? 'POST' : 'GET'; // Can change after a redirect has changed the method
  488. $url = is_string($url) ? $url : '';
  489. $resolve = [];
  490. if ($proxy === '') {
  491. $resolve = self::getCurlResolveInfo($url);
  492. if ($resolve === null) {
  493. Minz_Log::warning('Fetching this URL is not allowed, because the host’s IP is not in the allowlist [' .
  494. \SimplePie\Misc::url_remove_credentials($url) . ']');
  495. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  496. } elseif ($resolve === false) {
  497. Minz_Log::warning('Error during HTTP fetch: unable to resolve hostname! ' . \SimplePie\Misc::url_remove_credentials($url));
  498. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  499. }
  500. if (!empty($resolve)) {
  501. $curl_options[CURLOPT_RESOLVE] = $resolve; // Prevent DNS rebinding
  502. }
  503. } else {
  504. defined('CURLPROXY_HTTPS') or define('CURLPROXY_HTTPS', 2); // Compatibility cURL 7.51
  505. $proxy_scheme = match ($proxy_type) {
  506. CURLPROXY_HTTP => 'http',
  507. CURLPROXY_HTTPS => 'https',
  508. CURLPROXY_SOCKS4 => 'socks4',
  509. CURLPROXY_SOCKS4A => 'socks4a',
  510. CURLPROXY_SOCKS5 => 'socks5',
  511. CURLPROXY_SOCKS5_HOSTNAME => 'socks5h',
  512. default => null,
  513. };
  514. if ($proxy_scheme === null) {
  515. // Unsupported proxy type
  516. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  517. }
  518. $proxy_url = "$proxy_scheme://$proxy"; // CURLOPT_PROXY ($proxy) is formatted as user:pass@hostname:port, with the part before @ being optional
  519. $resolve = self::getCurlResolveInfo($proxy_url);
  520. if ($resolve === null) {
  521. Minz_Log::warning('Failed to fetch this URL, because the proxy’s IP is not in the allowlist [' .
  522. \SimplePie\Misc::url_remove_credentials($url) . '] [' .
  523. \SimplePie\Misc::url_remove_credentials($proxy_url) . ']');
  524. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  525. } elseif ($resolve === false) {
  526. Minz_Log::warning('Error during HTTP fetch: unable to resolve the proxy’s hostname! ' . \SimplePie\Misc::url_remove_credentials($proxy_url));
  527. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  528. }
  529. if (!empty($resolve)) {
  530. // Only for the proxy domain, socks4a and socks5h DNS queries will be passed through the proxy.
  531. // For the other proxy protocols, note that IPs for internal domains can be leaked,
  532. // since the domain is resolved outside of the proxy.
  533. $curl_options[CURLOPT_RESOLVE] = $resolve;
  534. }
  535. }
  536. // TODO: Implement HTTP 1.1 conditional GET If-Modified-Since
  537. $ch = curl_init();
  538. if ($ch === false || $url === '') {
  539. return ['body' => '', 'effective_url' => '', 'redirect_count' => 0, 'fail' => true, 'status' => -500, 'error' => '', 'method' => $c_method];
  540. }
  541. curl_setopt_array($ch, [
  542. CURLOPT_URL => $url,
  543. CURLOPT_HTTPHEADER => ['Accept: ' . $accept],
  544. CURLOPT_USERAGENT => FRESHRSS_USERAGENT,
  545. CURLOPT_CONNECTTIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
  546. CURLOPT_TIMEOUT => $feed_timeout > 0 ? $feed_timeout : $limits['timeout'],
  547. CURLOPT_RETURNTRANSFER => true,
  548. CURLOPT_ACCEPT_ENCODING => '', //Enable all encodings
  549. //CURLOPT_VERBOSE => 1, // To debug sent HTTP headers
  550. ]);
  551. curl_setopt_array($ch, $options);
  552. curl_setopt_array($ch, FreshRSS_Context::systemConf()->curl_options);
  553. $responseHeaders = '';
  554. curl_setopt($ch, CURLOPT_HEADERFUNCTION, function (\CurlHandle $ch, string $header) use (&$responseHeaders) {
  555. if (trim($header) !== '') { // Skip e.g. separation with trailer headers
  556. $responseHeaders .= $header;
  557. }
  558. return strlen($header);
  559. });
  560. if (isset($attributes['ssl_verify'])) {
  561. curl_setopt($ch, CURLOPT_SSL_VERIFYHOST, empty($attributes['ssl_verify']) ? 0 : 2);
  562. curl_setopt($ch, CURLOPT_SSL_VERIFYPEER, (bool)$attributes['ssl_verify']);
  563. if (empty($attributes['ssl_verify'])) {
  564. curl_setopt($ch, CURLOPT_SSL_CIPHER_LIST, 'DEFAULT@SECLEVEL=1');
  565. }
  566. }
  567. if (defined('CURLOPT_PROTOCOLS_STR') && is_int(CURLOPT_PROTOCOLS_STR)) {
  568. $curl_options[CURLOPT_PROTOCOLS_STR] = 'http,https';
  569. if (defined('CURLOPT_REDIR_PROTOCOLS_STR') && is_int(CURLOPT_REDIR_PROTOCOLS_STR)) {
  570. $curl_options[CURLOPT_REDIR_PROTOCOLS_STR] = 'http,https';
  571. }
  572. } elseif (defined('CURLPROTO_HTTP') && defined('CURLPROTO_HTTPS')) {
  573. // Legacy PHP 8.2-
  574. if (defined('CURLOPT_PROTOCOLS')) {
  575. $curl_options[CURLOPT_PROTOCOLS] = CURLPROTO_HTTP | CURLPROTO_HTTPS;
  576. }
  577. if (defined('CURLOPT_REDIR_PROTOCOLS')) {
  578. $curl_options[CURLOPT_REDIR_PROTOCOLS] = CURLPROTO_HTTP | CURLPROTO_HTTPS;
  579. }
  580. }
  581. curl_setopt_array($ch, $curl_options);
  582. curl_setopt($ch, CURLOPT_FOLLOWLOCATION, false); // We handle HTTP redirections manually for security
  583. $body = curl_exec($ch);
  584. $c_status = curl_getinfo($ch, CURLINFO_HTTP_CODE);
  585. $c_content_type = '' . curl_getinfo($ch, CURLINFO_CONTENT_TYPE);
  586. $c_effective_url = curl_getinfo($ch, CURLINFO_EFFECTIVE_URL);
  587. $c_error = curl_error($ch);
  588. $headers = [];
  589. if ($body !== false) {
  590. $responseHeaders .= "\r\n";
  591. $responseHeaders = \SimplePie\HTTP\Parser::prepareHeaders($responseHeaders);
  592. $parser = new \SimplePie\HTTP\Parser($responseHeaders);
  593. if ($parser->parse()) {
  594. $headers = $parser->headers;
  595. }
  596. }
  597. if (in_array($c_status, [301, 302, 303, 307, 308], true)) {
  598. // Handle the redirect by making another request
  599. $location = \SimplePie\Misc::absolutize_url($headers['location'] ?? $url, $url);
  600. if ($location === false || !\SimplePie\Misc::is_remote_uri($location)) {
  601. Minz_Log::warning('Invalid redirect location: malformed URL “' . ($headers['location'] ?? $url) . '“');
  602. break;
  603. }
  604. if (!self::compareURLOrigins($url, $location)) {
  605. unset($curl_options[CURLOPT_COOKIE]);
  606. unset($curl_options[CURLOPT_USERPWD]);
  607. unset($options[CURLOPT_COOKIE]);
  608. unset($options[CURLOPT_USERPWD]);
  609. if (is_array($options[CURLOPT_HTTPHEADER] ?? null)) {
  610. $options[CURLOPT_HTTPHEADER] = array_filter($options[CURLOPT_HTTPHEADER], fn(mixed $header): bool =>
  611. is_string($header) && !preg_match('/^(Cookie|Authorization)\\s*:/i', $header));
  612. }
  613. if (is_array($curl_options[CURLOPT_HTTPHEADER] ?? null)) {
  614. $curl_options[CURLOPT_HTTPHEADER] = array_filter($curl_options[CURLOPT_HTTPHEADER], fn(mixed $header): bool =>
  615. is_string($header) && !preg_match('/^(Cookie|Authorization)\\s*:/i', $header));
  616. }
  617. }
  618. if ($max_redirs >= 0) {
  619. $redirs++;
  620. }
  621. if ($redirs > $max_redirs) {
  622. Minz_Log::warning('Error fetching content: Too many redirects were hit [' . \SimplePie\Misc::url_remove_credentials($original_url) . ']');
  623. break;
  624. }
  625. if (self::isPost($options, $curl_options) &&
  626. in_array($c_status, [301, 302, 303], true)) { // Not for 307 and 308, which must not change the HTTP method
  627. unset($curl_options[CURLOPT_POST]);
  628. unset($curl_options[CURLOPT_POSTFIELDS]);
  629. unset($options[CURLOPT_POST]);
  630. unset($options[CURLOPT_POSTFIELDS]);
  631. if (is_array($options[CURLOPT_HTTPHEADER] ?? null)) {
  632. $options[CURLOPT_HTTPHEADER] = array_filter($options[CURLOPT_HTTPHEADER], fn(mixed $header): bool =>
  633. is_string($header) && !str_starts_with(strtolower(trim($header)), 'content-type:'));
  634. }
  635. if (is_array($curl_options[CURLOPT_HTTPHEADER] ?? null)) {
  636. $curl_options[CURLOPT_HTTPHEADER] = array_filter($curl_options[CURLOPT_HTTPHEADER], fn(mixed $header): bool =>
  637. is_string($header) && !str_starts_with(strtolower(trim($header)), 'content-type:'));
  638. }
  639. }
  640. $url = $location;
  641. continue;
  642. }
  643. $fail = $c_status < 200 || $c_status >= 300 || $c_error != '' || $body === false;
  644. if ($fail) {
  645. $body = '';
  646. Minz_Log::warning('Error fetching content: HTTP code ' . $c_status . ': ' . $c_error . ' ' . $url);
  647. if (in_array($c_status, [429, 503], true)) {
  648. $retryAfter = FreshRSS_http_Util::setRetryAfter($url, $proxy, $headers['retry-after'] ?? '');
  649. if ($c_status === 429) {
  650. $errorMessage = 'HTTP 429 Too Many Requests! [' . \SimplePie\Misc::url_remove_credentials($url) . ']';
  651. } elseif ($c_status === 503) {
  652. $errorMessage = 'HTTP 503 Service Unavailable! [' . \SimplePie\Misc::url_remove_credentials($url) . ']';
  653. }
  654. if ($retryAfter > 0) {
  655. $errorMessage .= ' We may retry after ' . date('c', $retryAfter);
  656. }
  657. }
  658. } elseif (!is_string($body) || strlen($body) === 0) { // TODO: Implement HTTP 410 Gone
  659. $body = '';
  660. } else {
  661. if (in_array($type, ['html', 'json', 'opml', 'xml'], true)) {
  662. $body = trim($body, " \n\r\t\v"); // Do not trim \x00 to avoid breaking a BOM
  663. }
  664. if (in_array($type, ['html', 'xml', 'opml'], true)) {
  665. $body = self::enforceHttpEncoding($body, $c_content_type);
  666. }
  667. if (in_array($type, ['html'], true)) {
  668. if (stripos($c_content_type, 'text/plain') !== false) {
  669. // Plain text to be displayed as preformatted text. Prefixed with UTF-8 BOM
  670. $body = "\xEF\xBB\xBF" . '<pre class="text-plain">' . htmlspecialchars($body, ENT_NOQUOTES, 'UTF-8') . '</pre>';
  671. } else {
  672. $body = self::enforceHtmlBase($body, $c_effective_url);
  673. }
  674. }
  675. }
  676. break;
  677. }
  678. if ($cachePath !== null && file_put_contents($cachePath, $body) === false) {
  679. Minz_Log::warning("Error saving cache for $url");
  680. }
  681. return ['body' => is_string($body) ? $body : '', 'effective_url' => $c_effective_url, 'redirect_count' => $redirs,
  682. 'fail' => $fail, 'status' => $c_status, 'error' => $c_error, 'method' => $c_method];
  683. }
  684. /**
  685. * Converts an IP (v4 or v6) to a binary representation using inet_pton
  686. *
  687. * @param string $ip the IP to convert
  688. * @return string a binary representation of the specified IP
  689. */
  690. private static function ipToBits(string $ip): string {
  691. $binaryip = '';
  692. foreach (str_split(inet_pton($ip) ?: '') as $char) {
  693. $binaryip .= str_pad(decbin(ord($char)), 8, '0', STR_PAD_LEFT);
  694. }
  695. return $binaryip;
  696. }
  697. /**
  698. * Check if an ip belongs to the provided range (in CIDR format)
  699. *
  700. * @param string $ip the IP that we want to verify (ex: 192.168.16.1)
  701. * @param string $range the range to check against (ex: 192.168.16.0/24)
  702. * @return bool true if the IP is in the range, otherwise false
  703. */
  704. private static function checkCIDR(string $ip, string $range): bool {
  705. $binary_ip = self::ipToBits($ip);
  706. if ($binary_ip === '') {
  707. return false;
  708. }
  709. $split = explode('/', $range);
  710. $subnet = $split[0] ?? '';
  711. if ($subnet == '') {
  712. return false;
  713. }
  714. $binary_subnet = self::ipToBits($subnet);
  715. if ($binary_subnet === '') {
  716. return false;
  717. }
  718. if (strlen($binary_ip) !== strlen($binary_subnet)) {
  719. return false; // Do not mix IPv4 and IPv6
  720. }
  721. $max_mask_bits = str_contains($ip, ':') ? 128 : 32;
  722. // A range without a subnet is a single address, like Apache’s mod_remoteip accepts
  723. $mask_bits_str = $split[1] ?? (string)$max_mask_bits;
  724. if (!ctype_digit($mask_bits_str)) {
  725. return false;
  726. }
  727. $mask_bits = (int)$mask_bits_str;
  728. if ($mask_bits < 0 || $mask_bits > $max_mask_bits) {
  729. return false; // Reject invalid mask bits lengths
  730. }
  731. if ($mask_bits === 0) {
  732. return true;
  733. }
  734. $ip_net_bits = substr($binary_ip, 0, $mask_bits);
  735. $subnet_bits = substr($binary_subnet, 0, $mask_bits);
  736. return $ip_net_bits === $subnet_bits;
  737. }
  738. /**
  739. * Check if the client (e.g. last proxy) is allowed to send unsafe headers.
  740. * This uses the `TRUSTED_PROXY` environment variable or the `trusted_sources` configuration option to get an array of the authorized ranges,
  741. * The connection IP is obtained from the `CONN_REMOTE_ADDR`
  742. * (if available, to be robust even when using Apache mod_remoteip) or `REMOTE_ADDR` environment variables.
  743. * @return bool true if the sender’s IP is in one of the ranges defined in the configuration, else false
  744. */
  745. public static function checkTrustedIP(): bool {
  746. if (!FreshRSS_Context::hasSystemConf()) {
  747. return false;
  748. }
  749. $remoteIp = Minz_Request::connectionRemoteAddress();
  750. if ($remoteIp === '') {
  751. return false;
  752. }
  753. $trusted = getenv('TRUSTED_PROXY');
  754. if ($trusted != 0 && is_string($trusted)) {
  755. $trusted = preg_split('/\s+/', $trusted, -1, PREG_SPLIT_NO_EMPTY);
  756. }
  757. if (!is_array($trusted) || empty($trusted)) {
  758. $trusted = FreshRSS_Context::systemConf()->trusted_sources;
  759. }
  760. foreach ($trusted as $cidr) {
  761. if (self::checkCIDR($remoteIp, $cidr)) {
  762. return true;
  763. }
  764. }
  765. return false;
  766. }
  767. public static function httpAuthUser(bool $onlyTrusted = true): string {
  768. $auths = array_unique(array_filter(
  769. array_intersect_key($_SERVER, ['REMOTE_USER' => '', 'REDIRECT_REMOTE_USER' => '', 'HTTP_REMOTE_USER' => '', 'HTTP_X_WEBAUTH_USER' => '']),
  770. fn($value) => is_string($value) && $value !== ''
  771. ));
  772. if (count($auths) > 1) {
  773. Minz_Log::warning('Multiple HTTP authentication headers!');
  774. return '';
  775. }
  776. if (!empty($_SERVER['REMOTE_USER']) && is_string($_SERVER['REMOTE_USER'])) {
  777. return $_SERVER['REMOTE_USER'];
  778. }
  779. if (!empty($_SERVER['REDIRECT_REMOTE_USER']) && is_string($_SERVER['REDIRECT_REMOTE_USER'])) {
  780. return $_SERVER['REDIRECT_REMOTE_USER'];
  781. }
  782. if (!$onlyTrusted || self::checkTrustedIP()) {
  783. if (!empty($_SERVER['HTTP_REMOTE_USER']) && is_string($_SERVER['HTTP_REMOTE_USER'])) {
  784. return $_SERVER['HTTP_REMOTE_USER'];
  785. }
  786. if (!empty($_SERVER['HTTP_X_WEBAUTH_USER']) && is_string($_SERVER['HTTP_X_WEBAUTH_USER'])) {
  787. return $_SERVER['HTTP_X_WEBAUTH_USER'];
  788. }
  789. }
  790. return '';
  791. }
  792. }