importExportController.php 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921
  1. <?php
  2. declare(strict_types=1);
  3. /**
  4. * Controller to handle every import and export actions.
  5. */
  6. class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
  7. private FreshRSS_EntryDAO $entryDAO;
  8. private FreshRSS_FeedDAO $feedDAO;
  9. private FreshRSS_CategoryDAO $categoryDAO;
  10. /**
  11. * ZIP import safety limits (defence against ZIP bombs / decompression DoS).
  12. * A legitimate export is an OPML plus a handful of JSON files, so these ceilings
  13. * sit far above any realistic export while keeping import memory bounded.
  14. */
  15. private const IMPORT_ZIP_MAX_MEMBERS = 100;
  16. private const IMPORT_ZIP_MEMBER_MAX_SIZE = 64 * 1024 * 1024;
  17. private const IMPORT_ZIP_TOTAL_MAX_SIZE = 128 * 1024 * 1024;
  18. private const IMPORT_ZIP_MAX_RATIO = 100;
  19. /**
  20. * Read a ZIP member by name through a stream, aborting once the decompressed
  21. * content exceeds $maxBytes. Returns null on read error or when the cap is
  22. * exceeded, so a forged/under-reported uncompressed size in the archive
  23. * metadata cannot be used to force unbounded memory allocation.
  24. */
  25. private static function readZipMemberCapped(ZipArchive $zip, string $name, int $maxBytes): ?string {
  26. $stream = $zip->getStream($name);
  27. if ($stream === false) {
  28. return null;
  29. }
  30. $content = '';
  31. try {
  32. while (!feof($stream)) {
  33. $chunk = fread($stream, 1 << 16);
  34. if ($chunk === false) {
  35. return null;
  36. }
  37. $content .= $chunk;
  38. if (strlen($content) > $maxBytes) {
  39. return null;
  40. }
  41. }
  42. } finally {
  43. fclose($stream);
  44. }
  45. return $content;
  46. }
  47. /**
  48. * This action is called before every other action in that class. It is
  49. * the common boilerplate for every action. It is triggered by the
  50. * underlying framework.
  51. */
  52. #[\Override]
  53. public function firstAction(): void {
  54. if (!FreshRSS_Auth::hasAccess()) {
  55. Minz_Error::error(403);
  56. }
  57. $this->entryDAO = FreshRSS_Factory::createEntryDao();
  58. $this->feedDAO = FreshRSS_Factory::createFeedDao();
  59. $this->categoryDAO = FreshRSS_Factory::createCategoryDao();
  60. }
  61. /**
  62. * This action displays the main page for import / export system.
  63. */
  64. public function indexAction(): void {
  65. $this->view->categories = array_filter(
  66. $this->categoryDAO->listCategories(),
  67. static fn(FreshRSS_Category $category): bool => !empty($category->feeds()),
  68. );
  69. $this->view->feedCount = array_sum(array_map(static fn(FreshRSS_Category $category): int => count($category->feeds()), $this->view->categories));
  70. FreshRSS_View::prependTitle(_t('sub.import_export.title') . ' · ');
  71. $this->listSqliteArchives();
  72. }
  73. private static function minimumMemory(int|string $mb): void {
  74. $mb = (int)$mb;
  75. // `ini_get('memory_limit')` returns e.g. `128M`, or `-1` for unlimited
  76. $ini = (int)(ini_parse_quantity(ini_get('memory_limit') ?: '0') / 1048576);
  77. if ($ini >= 0 && $ini < $mb) {
  78. ini_set('memory_limit', $mb . 'M');
  79. }
  80. }
  81. /**
  82. * @throws FreshRSS_Zip_Exception
  83. * @throws FreshRSS_ZipMissing_Exception
  84. * @throws Minz_ConfigurationNamespaceException
  85. * @throws Minz_PDOConnectionException
  86. */
  87. public function importFile(string $name, string $path, ?string $username = null): bool {
  88. self::minimumMemory(256);
  89. $this->entryDAO = FreshRSS_Factory::createEntryDao($username);
  90. $this->feedDAO = FreshRSS_Factory::createFeedDao($username);
  91. $this->categoryDAO = FreshRSS_Factory::createCategoryDao($username);
  92. $type_file = self::guessFileType($name);
  93. $list_files = [
  94. 'opml' => [],
  95. 'json_starred' => [],
  96. 'json_feed' => [],
  97. 'ttrss_starred' => [],
  98. ];
  99. // We try to list all files according to their type
  100. $skipped = false;
  101. if ('zip' === $type_file && extension_loaded('zip')) {
  102. $zip = new ZipArchive();
  103. $result = $zip->open($path);
  104. if (true !== $result) {
  105. // zip_open cannot open file: something is wrong
  106. throw new FreshRSS_Zip_Exception($result);
  107. }
  108. $accepted = 0;
  109. $totalUncompressed = 0;
  110. for ($i = 0; $i < $zip->numFiles; $i++) {
  111. $entryName = $zip->getNameIndex($i);
  112. if ($entryName === false) {
  113. continue;
  114. }
  115. $type_zipfile = self::guessFileType($entryName);
  116. if ('unknown' === $type_zipfile) {
  117. continue;
  118. }
  119. if ($zip->locateName($entryName) !== $i) {
  120. // Duplicate entry name: keep only the first occurrence
  121. continue;
  122. }
  123. // Reject obvious ZIP bombs cheaply from the central-directory metadata...
  124. $stat = $zip->statIndex($i);
  125. $declaredSize = is_array($stat) ? (int)($stat['size'] ?? 0) : 0;
  126. $compSize = is_array($stat) ? (int)($stat['comp_size'] ?? 0) : 0;
  127. if ($declaredSize > self::IMPORT_ZIP_MEMBER_MAX_SIZE
  128. || ($compSize > 0 && $declaredSize / $compSize > self::IMPORT_ZIP_MAX_RATIO)) {
  129. Minz_Log::warning('Import: skipping oversized/over-compressed ZIP member: ' . $entryName);
  130. $skipped = true;
  131. continue;
  132. }
  133. if ($accepted >= self::IMPORT_ZIP_MAX_MEMBERS) {
  134. Minz_Log::warning('Import: ZIP member budget reached, remaining members skipped');
  135. break;
  136. }
  137. // ...then enforce the real decompressed size while streaming, since the
  138. // declared metadata above is attacker-controlled and can under-report.
  139. $content = self::readZipMemberCapped($zip, $entryName, self::IMPORT_ZIP_MEMBER_MAX_SIZE);
  140. if ($content === null) {
  141. Minz_Log::warning('Import: skipping unreadable/over-limit ZIP member: ' . $entryName);
  142. $skipped = true;
  143. continue;
  144. }
  145. $totalUncompressed += strlen($content);
  146. if ($totalUncompressed > self::IMPORT_ZIP_TOTAL_MAX_SIZE) {
  147. Minz_Log::warning('Import: ZIP total size reached, remaining members skipped');
  148. $skipped = true;
  149. break;
  150. }
  151. $list_files[$type_zipfile][] = $content;
  152. $accepted++;
  153. }
  154. $zip->close();
  155. } elseif ('zip' === $type_file) {
  156. // ZIP extension is not loaded
  157. throw new FreshRSS_ZipMissing_Exception();
  158. } elseif ('txt' === $type_file) {
  159. $contents = file_get_contents($path);
  160. if (is_string($contents)) {
  161. $list_files['opml'][] = self::txtToOpml($contents);
  162. }
  163. } elseif ('unknown' !== $type_file) {
  164. $list_files[$type_file][] = file_get_contents($path);
  165. }
  166. // Import file contents.
  167. // OPML first(so categories and feeds are imported)
  168. // Starred articles then so the "favourite" status is already set
  169. // And finally all other files.
  170. $ok = !$skipped;
  171. $importService = new FreshRSS_Import_Service($username);
  172. foreach ($list_files['opml'] as $opml_file) {
  173. if ($opml_file === false) {
  174. continue;
  175. }
  176. $importService->importOpml($opml_file, trusted_source: true);
  177. if (!$importService->lastStatus()) {
  178. $ok = false;
  179. if (FreshRSS_Context::$isCli) {
  180. fwrite(STDERR, 'FreshRSS error during OPML import' . "\n");
  181. } else {
  182. Minz_Log::warning('Error during OPML import');
  183. }
  184. }
  185. }
  186. foreach ($list_files['json_starred'] as $article_file) {
  187. if (!is_string($article_file) || !$this->importJson($article_file, true)) {
  188. $ok = false;
  189. if (FreshRSS_Context::$isCli) {
  190. fwrite(STDERR, 'FreshRSS error during JSON stars import' . "\n");
  191. } else {
  192. Minz_Log::warning('Error during JSON stars import');
  193. }
  194. }
  195. }
  196. foreach ($list_files['json_feed'] as $article_file) {
  197. if (!is_string($article_file) || !$this->importJson($article_file)) {
  198. $ok = false;
  199. if (FreshRSS_Context::$isCli) {
  200. fwrite(STDERR, 'FreshRSS error during JSON feeds import' . "\n");
  201. } else {
  202. Minz_Log::warning('Error during JSON feeds import');
  203. }
  204. }
  205. }
  206. foreach ($list_files['ttrss_starred'] as $article_file) {
  207. $json = is_string($article_file) ? $this->ttrssXmlToJson($article_file) : false;
  208. if ($json === false || !$this->importJson($json, true)) {
  209. $ok = false;
  210. if (FreshRSS_Context::$isCli) {
  211. fwrite(STDERR, 'FreshRSS error during TT-RSS articles import' . "\n");
  212. } else {
  213. Minz_Log::warning('Error during TT-RSS articles import');
  214. }
  215. }
  216. }
  217. return $ok;
  218. }
  219. /**
  220. * This action handles import action.
  221. *
  222. * It must be reached by a POST request.
  223. *
  224. * Parameter is:
  225. * - file (default: nothing!)
  226. * Available file types are: zip, json or xml.
  227. */
  228. public function importAction(): void {
  229. if (!Minz_Request::isPost()) {
  230. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  231. }
  232. $file = $_FILES['file'] ?? null;
  233. $status_file = is_array($file) ? $file['error'] ?? -1 : -1;
  234. if (!is_array($file) || $status_file !== 0 || !is_string($file['name'] ?? null) || !is_string($file['tmp_name'] ?? null)) {
  235. Minz_Log::warning('File cannot be uploaded. Error code: ' . (is_numeric($status_file) ? $status_file : -1));
  236. Minz_Request::bad(_t('feedback.import_export.file_cannot_be_uploaded'), [ 'c' => 'importExport', 'a' => 'index' ]);
  237. return;
  238. }
  239. if (function_exists('set_time_limit')) {
  240. @set_time_limit(300);
  241. }
  242. $error = false;
  243. try {
  244. $error = !$this->importFile($file['name'], $file['tmp_name']);
  245. } catch (FreshRSS_ZipMissing_Exception) {
  246. Minz_Request::bad(
  247. _t('feedback.import_export.no_zip_extension'),
  248. ['c' => 'importExport', 'a' => 'index']
  249. );
  250. } catch (FreshRSS_Zip_Exception $ze) {
  251. Minz_Log::warning('ZIP archive cannot be imported. Error code: ' . $ze->zipErrorCode());
  252. Minz_Request::bad(
  253. _t('feedback.import_export.zip_error'),
  254. ['c' => 'importExport', 'a' => 'index']
  255. );
  256. }
  257. // And finally, we get import status and redirect to the home page
  258. $content_notif = $error === true ? _t('feedback.import_export.feeds_imported_with_errors') : _t('feedback.import_export.feeds_imported');
  259. Minz_Request::good(
  260. $content_notif,
  261. showNotification: FreshRSS_Context::userConf()->good_notification_timeout > 0
  262. );
  263. }
  264. /**
  265. * This method tries to guess the file type based on its name.
  266. *
  267. * It is a *very* basic guess file type function. Only based on filename.
  268. * That could be improved but should be enough for what we have to do.
  269. */
  270. private static function guessFileType(string $filename): string {
  271. if (str_ends_with($filename, '.zip')) {
  272. return 'zip';
  273. } elseif (str_ends_with($filename, '.txt')) {
  274. return 'txt';
  275. } elseif (stripos($filename, 'opml') !== false) {
  276. return 'opml';
  277. } elseif (str_ends_with($filename, '.json')) {
  278. if (str_contains($filename, 'starred')) {
  279. return 'json_starred';
  280. } else {
  281. return 'json_feed';
  282. }
  283. } elseif (str_ends_with($filename, '.xml')) {
  284. if (preg_match('/Tiny|tt-?rss/i', $filename)) {
  285. return 'ttrss_starred';
  286. } else {
  287. return 'opml';
  288. }
  289. }
  290. return 'unknown';
  291. }
  292. /**
  293. * Wraps a newline-separated list of feed URLs into a minimal OPML document
  294. * so it can be imported through the existing OPML pipeline.
  295. */
  296. private static function txtToOpml(string $contents): string {
  297. $utf8BOM = "\xEF\xBB\xBF";
  298. $contents = preg_replace('/^' . $utf8BOM . '/', '', $contents) ?? $contents;
  299. $outlines = '';
  300. foreach (preg_split('/\R/', $contents) ?: [] as $line) {
  301. $url = trim($line);
  302. if ($url === '' || str_starts_with($url, '#') || str_starts_with($url, '<')) {
  303. continue;
  304. }
  305. if (filter_var($url, FILTER_VALIDATE_URL) === false) {
  306. $message = 'TXT import: skipping invalid URL “' . \SimplePie\Misc::url_remove_credentials($url) . '”';
  307. if (FreshRSS_Context::$isCli) {
  308. fwrite(STDERR, $message . "\n");
  309. } else {
  310. Minz_Log::warning($message);
  311. }
  312. continue;
  313. }
  314. $escaped = htmlspecialchars($url, ENT_COMPAT | ENT_XML1, 'UTF-8');
  315. $outlines .= '<outline type="rss" text="' . $escaped . '" xmlUrl="' . $escaped . '" />' . "\n";
  316. }
  317. return '<?xml version="1.0" encoding="UTF-8"?>' . "\n"
  318. . '<opml version="2.0"><body>' . "\n"
  319. . $outlines
  320. . '</body></opml>' . "\n";
  321. }
  322. /**
  323. * @return string|false JSON, or false if the file is invalid XML
  324. */
  325. private function ttrssXmlToJson(string $xml): string|false {
  326. try {
  327. $dom = @Dom\XMLDocument::createFromString($xml, LIBXML_NOERROR);
  328. } catch (Throwable $ex) { // @phpstan-ignore catch.neverThrown (Dom\XMLDocument::createFromString() does throw)
  329. Minz_Log::warning('Error during TT-RSS articles import: ' . $ex->getMessage());
  330. return false;
  331. }
  332. $domElement = $dom->documentElement;
  333. $table = ['items' => []];
  334. if ($domElement !== null) {
  335. foreach ($domElement->childNodes as $article) {
  336. if (!$article instanceof Dom\Element || $article->nodeName !== 'article') {
  337. continue;
  338. }
  339. $item = [];
  340. foreach ($article->childNodes as $child) {
  341. if ($child instanceof Dom\Element) {
  342. $item[$child->nodeName] = $child->textContent;
  343. }
  344. }
  345. $item = array_filter($item, static fn($v) => trim($v) !== '');
  346. $item['updated'] = isset($item['updated']) ? strtotime($item['updated']) : '';
  347. $item['published'] = $item['updated'];
  348. $item['content'] = ['content' => $item['content'] ?? ''];
  349. $item['categories'] = isset($item['tag_cache']) ? [$item['tag_cache']] : [];
  350. if (!empty($item['marked'])) {
  351. $item['categories'][] = 'user/-/state/com.google/starred';
  352. }
  353. if (!empty($item['published'])) {
  354. $item['categories'][] = 'user/-/state/com.google/broadcast';
  355. }
  356. if (is_string($item['label_cache'] ?? null)) {
  357. $labels_cache = json_decode($item['label_cache'], true);
  358. if (is_array($labels_cache)) {
  359. foreach ($labels_cache as $label_cache) {
  360. if (is_array($label_cache) && !empty($label_cache[1]) && is_string($label_cache[1])) {
  361. $item['categories'][] = 'user/-/label/' . trim($label_cache[1]);
  362. }
  363. }
  364. }
  365. }
  366. $item['alternate'] = [['href' => $item['link'] ?? '']];
  367. $item['origin'] = [
  368. 'title' => $item['feed_title'] ?? '',
  369. 'feedUrl' => $item['feed_url'] ?? '',
  370. ];
  371. $item['id'] = $item['guid'] ?? ($item['feed_url'] ?? $item['published']);
  372. $item['guid'] = $item['id'];
  373. $table['items'][] = $item;
  374. }
  375. }
  376. return json_encode($table);
  377. }
  378. /**
  379. * This method import a JSON-based file (Google Reader format).
  380. *
  381. * $article_file the JSON file content.
  382. * true if articles from the file must be starred.
  383. * @return bool false if an error occurred, true otherwise.
  384. * @throws Minz_ConfigurationNamespaceException
  385. * @throws Minz_PDOConnectionException
  386. */
  387. private function importJson(string $article_file, bool $starred = false): bool {
  388. $article_object = json_decode($article_file, true);
  389. if (!is_array($article_object)) {
  390. if (FreshRSS_Context::$isCli) {
  391. fwrite(STDERR, 'FreshRSS error trying to import a non-JSON file' . "\n");
  392. } else {
  393. Minz_Log::warning('Try to import a non-JSON file');
  394. }
  395. return false;
  396. }
  397. $items = $article_object['items'] ?? $article_object;
  398. if (!is_array($items)) {
  399. $items = [];
  400. }
  401. $mark_as_read = FreshRSS_Context::userConf()->mark_when['reception'] ? 1 : 0;
  402. $error = false;
  403. $article_to_feed = [];
  404. $nb_feeds = count($this->feedDAO->listFeeds());
  405. $newFeedGuids = [];
  406. $limits = FreshRSS_Context::systemConf()->limits;
  407. // First, we check feeds of articles are in DB (and add them if needed).
  408. foreach ($items as &$item) {
  409. if (!is_array($item)) {
  410. continue;
  411. }
  412. if (!is_string($item['guid'] ?? null) && is_string($item['id'] ?? null)) {
  413. $item['guid'] = $item['id'];
  414. }
  415. if (!is_string($item['guid'] ?? null)) {
  416. continue;
  417. }
  418. if (!is_array($item['origin'] ?? null)) {
  419. $item['origin'] = [];
  420. }
  421. if (!is_string($item['origin']['title'] ?? null) || trim($item['origin']['title']) === '') {
  422. $item['origin']['title'] = 'Import';
  423. }
  424. if (is_string($item['origin']['feedUrl'] ?? null)) {
  425. $feedUrl = $item['origin']['feedUrl'];
  426. } elseif (is_string($item['origin']['streamId'] ?? null) && str_starts_with($item['origin']['streamId'], 'feed/')) {
  427. $feedUrl = substr($item['origin']['streamId'], 5); //Google Reader
  428. $item['origin']['feedUrl'] = $feedUrl;
  429. } elseif (is_string($item['origin']['htmlUrl'] ?? null)) {
  430. $feedUrl = $item['origin']['htmlUrl'];
  431. } else {
  432. $feedUrl = 'http://import.localhost/import.xml';
  433. $item['origin']['feedUrl'] = $feedUrl;
  434. $item['origin']['disable'] = 'true';
  435. }
  436. $feedUrlOriginal = $feedUrl;
  437. $feedUrl = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($feedUrl) ?: '');
  438. try {
  439. $feed = new FreshRSS_Feed($feedUrl);
  440. } catch (FreshRSS_BadUrl_Exception) {
  441. Minz_Log::warning('Could not add feed with invalid URL "' . \SimplePie\Misc::url_remove_credentials($feedUrlOriginal) . '" during JSON import');
  442. continue;
  443. }
  444. $feed = $this->feedDAO->searchByUrl($feed->url());
  445. if ($feed === null) {
  446. // Feed does not exist in DB,we should to try to add it.
  447. if ((!FreshRSS_Context::$isCli) && ($nb_feeds >= $limits['max_feeds'])) {
  448. // Oops, no more place!
  449. Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds']));
  450. } else {
  451. $origin = array_filter($item['origin'], fn($value, $key): bool => is_string($key) && is_string($value), ARRAY_FILTER_USE_BOTH);
  452. $feed = $this->addFeedJson($origin);
  453. }
  454. if ($feed === null) {
  455. // Still null? It means something went wrong.
  456. $error = true;
  457. } else {
  458. $nb_feeds++;
  459. }
  460. }
  461. if ($feed !== null) {
  462. $article_to_feed[$item['guid']] = $feed->id();
  463. if (!isset($newFeedGuids['f_' . $feed->id()])) {
  464. $newFeedGuids['f_' . $feed->id()] = [];
  465. }
  466. $newFeedGuids['f_' . $feed->id()][] = safe_ascii($item['guid']);
  467. }
  468. }
  469. $tagDAO = FreshRSS_Factory::createTagDao();
  470. $labels = FreshRSS_Context::labels();
  471. $knownLabels = [];
  472. foreach ($labels as $label) {
  473. $knownLabels[$label->name()]['id'] = $label->id();
  474. $knownLabels[$label->name()]['articles'] = [];
  475. }
  476. unset($labels);
  477. // For each feed, check existing GUIDs already in database.
  478. $existingHashForGuids = [];
  479. foreach ($newFeedGuids as $feedId => $newGuids) {
  480. $existingHashForGuids[$feedId] = $this->entryDAO->listHashForFeedGuids((int)substr($feedId, 2), $newGuids);
  481. }
  482. unset($newFeedGuids);
  483. // Then, articles are imported.
  484. $newGuids = [];
  485. $this->entryDAO->beginTransaction();
  486. foreach ($items as &$item) {
  487. if (!is_array($item) || empty($item['guid']) || !is_string($item['guid']) || empty($article_to_feed[$item['guid']])) {
  488. // Related feed does not exist for this entry, do nothing.
  489. continue;
  490. }
  491. $feed_id = $article_to_feed[$item['guid']];
  492. $author = is_string($item['author'] ?? null) ? Minz_Helper::htmlspecialchars_utf8(htmlspecialchars_decode($item['author'], ENT_QUOTES)) : '';
  493. $is_starred = null; // null is used to preserve the current state if that item exists and is already starred
  494. $is_read = null;
  495. $tags = is_array($item['categories'] ?? null) ? $item['categories'] : [];
  496. $labels = [];
  497. for ($i = count($tags) - 1; $i >= 0; $i--) {
  498. $tag = $tags[$i];
  499. if (!is_string($tag)) {
  500. unset($tags[$i]);
  501. continue;
  502. }
  503. $tag = trim($tag);
  504. if (preg_match('%^user/[A-Za-z0-9_-]+/%', $tag)) {
  505. if (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/starred$%', $tag)) {
  506. $is_starred = true;
  507. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/read$%', $tag)) {
  508. $is_read = true;
  509. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/unread$%', $tag)) {
  510. $is_read = false;
  511. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/label/\s*(?P<tag>.+?)\s*$%', $tag, $matches)) {
  512. $labels[] = Minz_Helper::htmlspecialchars_utf8($matches['tag']);
  513. }
  514. unset($tags[$i]);
  515. }
  516. }
  517. $tags = Minz_Helper::htmlspecialchars_utf8(array_values(array_filter($tags, 'is_string')));
  518. if ($starred && !$is_starred) {
  519. //If the article has no label, mark it as starred (old format)
  520. $is_starred = empty($labels);
  521. }
  522. if ($is_read === null) {
  523. $is_read = $mark_as_read;
  524. }
  525. if (is_array($item['alternate']) && is_array($item['alternate'][0] ?? null) && is_string($item['alternate'][0]['href'] ?? null)) {
  526. $url = $item['alternate'][0]['href'];
  527. } elseif (is_string($item['url'] ?? null)) {
  528. $url = $item['url']; //FeedBin
  529. } else {
  530. $url = '';
  531. }
  532. $url = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($url) ?: '');
  533. $title = is_string($item['title'] ?? null) ? $item['title'] : $url;
  534. $title = Minz_Helper::htmlspecialchars_utf8(htmlspecialchars_decode($title, ENT_QUOTES));
  535. if (is_array($item['content'] ?? null) && is_string($item['content']['content'] ?? null)) {
  536. $content = $item['content']['content'];
  537. } elseif (is_array($item['summary']) && is_string($item['summary']['content'] ?? null)) {
  538. $content = $item['summary']['content'];
  539. } elseif (is_string($item['content'] ?? null)) {
  540. $content = $item['content']; //FeedBin
  541. } else {
  542. $content = '';
  543. }
  544. $content = FreshRSS_SimplePieCustom::sanitizeHTML($content, $url);
  545. if (is_int($item['published'] ?? null) || is_string($item['published'] ?? null)) {
  546. $published = (string)$item['published'];
  547. } elseif (is_int($item['timestampUsec'] ?? null) || is_string($item['timestampUsec'] ?? null)) {
  548. $published = substr((string)$item['timestampUsec'], 0, -6);
  549. } elseif (is_int($item['updated'] ?? null) || is_string($item['updated'] ?? null)) {
  550. $published = (string)$item['updated'];
  551. } else {
  552. $published = '0';
  553. }
  554. if (!ctype_digit($published)) {
  555. $published = (string)(strtotime($published) ?: 0);
  556. }
  557. if (strlen($published) > 10) { // Milliseconds, e.g. Feedly
  558. $published = substr($published, 0, -3);
  559. if (!is_numeric($published)) {
  560. $published = '0'; // For PHPStan
  561. }
  562. }
  563. $entry = new FreshRSS_Entry(
  564. $feed_id, $item['guid'], $title, $author,
  565. $content, $url, $published, $is_read, $is_starred
  566. );
  567. $entry->_id(uTimeString());
  568. $entry->_tags($tags);
  569. if (isset($newGuids[$entry->guid()])) {
  570. continue; //Skip subsequent articles with same GUID
  571. }
  572. $newGuids[$entry->guid()] = true;
  573. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeInsert, $entry);
  574. if (!($entry instanceof FreshRSS_Entry)) {
  575. // An extension has returned a null value, there is nothing to insert.
  576. continue;
  577. }
  578. if (isset($existingHashForGuids['f_' . $feed_id][$entry->guid()])) {
  579. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeUpdate, $entry);
  580. if (!($entry instanceof FreshRSS_Entry)) {
  581. // An extension has returned a null value, there is nothing to insert.
  582. continue;
  583. }
  584. $ok = $this->entryDAO->updateEntry($entry->toArray());
  585. } else {
  586. $entry->_lastSeen(time());
  587. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeAdd, $entry);
  588. if (!($entry instanceof FreshRSS_Entry)) {
  589. // An extension has returned a null value, there is nothing to insert.
  590. continue;
  591. }
  592. $ok = $this->entryDAO->addEntry($entry->toArray());
  593. }
  594. foreach ($labels as $labelName) {
  595. if (empty($knownLabels[$labelName]['id'])) {
  596. $labelId = $tagDAO->addTag(['name' => $labelName]);
  597. $knownLabels[$labelName]['id'] = $labelId;
  598. $knownLabels[$labelName]['articles'] = [];
  599. }
  600. $knownLabels[$labelName]['articles'][] = [
  601. //'id' => $entry->id(), //ID changes after commitNewEntries()
  602. 'id_feed' => $entry->feedId(),
  603. 'guid' => $entry->guid(),
  604. ];
  605. }
  606. $error |= ($ok === false);
  607. }
  608. $this->entryDAO->commit();
  609. $this->entryDAO->beginTransaction();
  610. $this->entryDAO->commitNewEntries();
  611. $this->feedDAO->updateCachedValues();
  612. $this->entryDAO->commit();
  613. $this->entryDAO->beginTransaction();
  614. foreach ($knownLabels as $labelName => $knownLabel) {
  615. $labelId = $knownLabel['id'];
  616. if (!$labelId) {
  617. continue;
  618. }
  619. foreach ($knownLabel['articles'] as $article) {
  620. $entryId = $this->entryDAO->searchIdByGuid($article['id_feed'], $article['guid']);
  621. if ($entryId != null) {
  622. $tagDAO->tagEntry($labelId, $entryId);
  623. } else {
  624. Minz_Log::warning('Could not add label "' . $labelName . '" to entry "' . $article['guid'] . '" in feed ' . $article['id_feed']);
  625. }
  626. }
  627. }
  628. $this->entryDAO->commit();
  629. return !$error;
  630. }
  631. /**
  632. * This method import a JSON-based feed (Google Reader format).
  633. *
  634. * @param array<string,string> $origin represents a feed.
  635. * @return FreshRSS_Feed|null if feed is in database at the end of the process, else null.
  636. */
  637. private function addFeedJson(array $origin): ?FreshRSS_Feed {
  638. $return = null;
  639. if (!empty($origin['feedUrl'])) {
  640. $url = $origin['feedUrl'];
  641. } elseif (!empty($origin['htmlUrl'])) {
  642. $url = $origin['htmlUrl'];
  643. } else {
  644. return null;
  645. }
  646. $url = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($url) ?: '');
  647. if (!empty($origin['htmlUrl'])) {
  648. $website = $origin['htmlUrl'];
  649. } elseif (!empty($origin['feedUrl'])) {
  650. $website = $origin['feedUrl'];
  651. } else {
  652. $website = '';
  653. }
  654. $website = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($website) ?: '');
  655. $name = empty($origin['title']) ? $website : $origin['title'];
  656. $name = Minz_Helper::htmlspecialchars_utf8($name);
  657. $cat_id = FreshRSS_CategoryDAO::DEFAULTCATEGORYID;
  658. $cat_name = Minz_Helper::htmlspecialchars_utf8(trim($origin['category'] ?? ''));
  659. if ($cat_name !== '') {
  660. $new_cat = $this->categoryDAO->searchByName($cat_name);
  661. $cat_id = $new_cat?->id() ?: 0;
  662. if ($cat_id === 0) {
  663. $limits = FreshRSS_Context::systemConf()->limits;
  664. if ($limits['max_categories'] > 0 && $this->categoryDAO->count() >= $limits['max_categories']) {
  665. Minz_Log::warning(_t('feedback.sub.category.over_max', $limits['max_categories']));
  666. } else {
  667. $cat_id = $this->categoryDAO->addCategory(['name' => $cat_name]) ?: 0;
  668. }
  669. }
  670. if ($cat_id === 0) {
  671. $cat_id = FreshRSS_CategoryDAO::DEFAULTCATEGORYID;
  672. }
  673. }
  674. try {
  675. // Create a Feed object and add it in database.
  676. $feed = new FreshRSS_Feed($url);
  677. $feed->_categoryId($cat_id);
  678. $feed->_name($name);
  679. $feed->_website($website);
  680. if (!empty($origin['disable'])) {
  681. $feed->_mute(true);
  682. }
  683. // Call the extension hook
  684. $feed = Minz_ExtensionManager::callHook(Minz_HookType::FeedBeforeInsert, $feed);
  685. if ($feed instanceof FreshRSS_Feed) {
  686. // addFeedObject checks if feed is already in DB so nothing else to
  687. // check here.
  688. $id = $this->feedDAO->addFeedObject($feed);
  689. if ($id !== false) {
  690. $feed->_id($id);
  691. $return = $feed;
  692. }
  693. }
  694. } catch (FreshRSS_Feed_Exception $e) {
  695. if (FreshRSS_Context::$isCli) {
  696. fwrite(STDERR, 'FreshRSS error during JSON feed import: ' . $e->getMessage() . "\n");
  697. } else {
  698. Minz_Log::warning($e->getMessage());
  699. }
  700. }
  701. return $return;
  702. }
  703. /**
  704. * This action handles export action.
  705. *
  706. * This action must be reached by a POST request.
  707. *
  708. * Parameters are:
  709. * - export_opml (default: false)
  710. * - export_starred (default: false)
  711. * - export_labelled (default: false)
  712. * - export_feeds (default: []) a list of feed ids
  713. */
  714. public function exportAction(): void {
  715. if (!Minz_Request::isPost()) {
  716. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  717. return;
  718. }
  719. $username = Minz_User::name() ?? '_';
  720. $export_service = new FreshRSS_Export_Service($username);
  721. $export_opml = Minz_Request::paramBoolean('export_opml');
  722. $export_starred = Minz_Request::paramBoolean('export_starred');
  723. $export_labelled = Minz_Request::paramBoolean('export_labelled');
  724. /** @var array<numeric-string> */
  725. $export_feeds = Minz_Request::paramArray('export_feeds');
  726. $max_number_entries = 50;
  727. $exported_files = [];
  728. if ($export_opml) {
  729. [$filename, $path] = $export_service->generateOpml();
  730. $exported_files[$filename] = $path;
  731. }
  732. // Starred and labelled entries are merged in the same `starred` file
  733. // to avoid duplication of content.
  734. if ($export_starred && $export_labelled) {
  735. [$filename, $path] = $export_service->generateStarredEntries('ST');
  736. $exported_files[$filename] = $path;
  737. } elseif ($export_starred) {
  738. [$filename, $path] = $export_service->generateStarredEntries('S');
  739. $exported_files[$filename] = $path;
  740. } elseif ($export_labelled) {
  741. [$filename, $path] = $export_service->generateStarredEntries('T');
  742. $exported_files[$filename] = $path;
  743. }
  744. foreach ($export_feeds as $feed_id) {
  745. $result = $export_service->generateFeedEntries((int)$feed_id, $max_number_entries);
  746. if ($result === null) {
  747. // It means the actual feed_id doesn’t correspond to any existing feed
  748. continue;
  749. }
  750. [$filename, $path] = $result;
  751. $exported_files[$filename] = $path;
  752. }
  753. $nb_files = count($exported_files);
  754. if ($nb_files <= 0) {
  755. // There’s nothing to do, there are no files to export
  756. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  757. return;
  758. }
  759. if ($nb_files === 1) {
  760. // If we only have one file, we just export it as it is
  761. $filename = key($exported_files);
  762. $path = $exported_files[$filename];
  763. } else {
  764. // More files? Let’s compress them in a Zip archive
  765. if (!extension_loaded('zip')) {
  766. // Oops, there is no ZIP extension!
  767. Minz_Request::bad(
  768. _t('feedback.import_export.export_no_zip_extension'),
  769. ['c' => 'importExport', 'a' => 'index']
  770. );
  771. return;
  772. }
  773. [$filename, $path] = $export_service->zip($exported_files);
  774. }
  775. if (!is_string($path)) {
  776. Minz_Request::bad(_t('feedback.import_export.zip_error'), ['c' => 'importExport', 'a' => 'index']);
  777. return;
  778. }
  779. $content_type = self::filenameToContentType($filename);
  780. header('Content-Type: ' . $content_type);
  781. header('Content-disposition: attachment; filename="' . $filename . '"');
  782. $this->view->_layout(null);
  783. $this->view->exportPath = $path;
  784. }
  785. /**
  786. * Return the Content-Type corresponding to a filename.
  787. *
  788. * If the type of the filename is not supported, it returns
  789. * `application/octet-stream` by default.
  790. */
  791. private static function filenameToContentType(string $filename): string {
  792. $filetype = self::guessFileType($filename);
  793. return match ($filetype) {
  794. 'zip' => 'application/zip',
  795. 'opml' => 'application/xml; charset=utf-8',
  796. 'json_starred', 'json_feed' => 'application/json; charset=utf-8',
  797. default => 'application/octet-stream',
  798. };
  799. }
  800. private const REGEX_SQLITE_FILENAME = '/^(?![.-])[0-9a-zA-Z_.@ #&()~\-]{1,128}\.sqlite$/';
  801. private function listSqliteArchives(): void {
  802. $this->view->sqliteArchives = [];
  803. $files = glob(USERS_PATH . '/' . Minz_User::name() . '/*.sqlite', GLOB_NOSORT) ?: [];
  804. foreach ($files as $file) {
  805. $archive = [
  806. 'name' => basename($file),
  807. 'size' => @filesize($file),
  808. 'mtime' => @filemtime($file),
  809. ];
  810. if ($archive['size'] != false && $archive['mtime'] != false && preg_match(self::REGEX_SQLITE_FILENAME, $archive['name'])) {
  811. $this->view->sqliteArchives[] = $archive;
  812. }
  813. }
  814. // Sort by time, newest first:
  815. usort($this->view->sqliteArchives, static fn(array $a, array $b): int => $b['mtime'] <=> $a['mtime']);
  816. }
  817. public function sqliteAction(): void {
  818. if (!Minz_Request::isPost()) {
  819. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  820. }
  821. $sqlite = Minz_Request::paramString('sqlite');
  822. if (!preg_match(self::REGEX_SQLITE_FILENAME, $sqlite)) {
  823. Minz_Error::error(404);
  824. return;
  825. }
  826. $path = USERS_PATH . '/' . Minz_User::name() . '/' . $sqlite;
  827. if (!file_exists($path) || @filesize($path) == false || @filemtime($path) == false) {
  828. Minz_Error::error(404);
  829. return;
  830. }
  831. $this->view->sqlitePath = $path;
  832. $this->view->sqliteName = basename($path);
  833. if ($this->view->sqliteName === 'db.sqlite') {
  834. $username = Minz_User::name() ?? '_';
  835. $date = date('Y-m-d_H-i-s', filemtime($path) ?: time()); // @phpstan-ignore ternary.alwaysTrue (for additional safety)
  836. $this->view->sqliteName = 'freshrss_' . $username . '_' . $date . '_db.sqlite';
  837. }
  838. $this->view->_layout(null);
  839. }
  840. }