importExportController.php 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829
  1. <?php
  2. declare(strict_types=1);
  3. /**
  4. * Controller to handle every import and export actions.
  5. */
  6. class FreshRSS_importExport_Controller extends FreshRSS_ActionController {
  7. private FreshRSS_EntryDAO $entryDAO;
  8. private FreshRSS_FeedDAO $feedDAO;
  9. private FreshRSS_CategoryDAO $categoryDAO;
  10. /**
  11. * This action is called before every other action in that class. It is
  12. * the common boilerplate for every action. It is triggered by the
  13. * underlying framework.
  14. */
  15. #[\Override]
  16. public function firstAction(): void {
  17. if (!FreshRSS_Auth::hasAccess()) {
  18. Minz_Error::error(403);
  19. }
  20. $this->entryDAO = FreshRSS_Factory::createEntryDao();
  21. $this->feedDAO = FreshRSS_Factory::createFeedDao();
  22. $this->categoryDAO = FreshRSS_Factory::createCategoryDao();
  23. }
  24. /**
  25. * This action displays the main page for import / export system.
  26. */
  27. public function indexAction(): void {
  28. $this->view->categories = array_filter(
  29. $this->categoryDAO->listCategories(),
  30. static fn(FreshRSS_Category $category): bool => !empty($category->feeds()),
  31. );
  32. $this->view->feedCount = array_sum(array_map(static fn(FreshRSS_Category $category): int => count($category->feeds()), $this->view->categories));
  33. FreshRSS_View::prependTitle(_t('sub.import_export.title') . ' · ');
  34. $this->listSqliteArchives();
  35. }
  36. private static function megabytes(string $size_str): float|int|string {
  37. return match (substr($size_str, -1)) {
  38. 'M', 'm' => (int)$size_str,
  39. 'K', 'k' => (int)$size_str / 1024,
  40. 'G', 'g' => (int)$size_str * 1024,
  41. default => $size_str,
  42. };
  43. }
  44. private static function minimumMemory(int|string $mb): void {
  45. $mb = (int)$mb;
  46. $ini = self::megabytes(ini_get('memory_limit') ?: '0');
  47. if ($ini < $mb) {
  48. ini_set('memory_limit', $mb . 'M');
  49. }
  50. }
  51. /**
  52. * @throws FreshRSS_Zip_Exception
  53. * @throws FreshRSS_ZipMissing_Exception
  54. * @throws Minz_ConfigurationNamespaceException
  55. * @throws Minz_PDOConnectionException
  56. */
  57. public function importFile(string $name, string $path, ?string $username = null): bool {
  58. self::minimumMemory(256);
  59. $this->entryDAO = FreshRSS_Factory::createEntryDao($username);
  60. $this->feedDAO = FreshRSS_Factory::createFeedDao($username);
  61. $this->categoryDAO = FreshRSS_Factory::createCategoryDao($username);
  62. $type_file = self::guessFileType($name);
  63. $list_files = [
  64. 'opml' => [],
  65. 'json_starred' => [],
  66. 'json_feed' => [],
  67. 'ttrss_starred' => [],
  68. ];
  69. // We try to list all files according to their type
  70. $list = [];
  71. if ('zip' === $type_file && extension_loaded('zip')) {
  72. $zip = new ZipArchive();
  73. $result = $zip->open($path);
  74. if (true !== $result) {
  75. // zip_open cannot open file: something is wrong
  76. throw new FreshRSS_Zip_Exception($result);
  77. }
  78. for ($i = 0; $i < $zip->numFiles; $i++) {
  79. if ($zip->getNameIndex($i) === false) {
  80. continue;
  81. }
  82. $type_zipfile = self::guessFileType($zip->getNameIndex($i));
  83. if ('unknown' !== $type_zipfile) {
  84. $list_files[$type_zipfile][] = $zip->getFromIndex($i);
  85. }
  86. }
  87. $zip->close();
  88. } elseif ('zip' === $type_file) {
  89. // ZIP extension is not loaded
  90. throw new FreshRSS_ZipMissing_Exception();
  91. } elseif ('txt' === $type_file) {
  92. $contents = file_get_contents($path);
  93. if (is_string($contents)) {
  94. $list_files['opml'][] = self::txtToOpml($contents);
  95. }
  96. } elseif ('unknown' !== $type_file) {
  97. $list_files[$type_file][] = file_get_contents($path);
  98. }
  99. // Import file contents.
  100. // OPML first(so categories and feeds are imported)
  101. // Starred articles then so the "favourite" status is already set
  102. // And finally all other files.
  103. $ok = true;
  104. $importService = new FreshRSS_Import_Service($username);
  105. foreach ($list_files['opml'] as $opml_file) {
  106. if ($opml_file === false) {
  107. continue;
  108. }
  109. $importService->importOpml($opml_file);
  110. if (!$importService->lastStatus()) {
  111. $ok = false;
  112. if (FreshRSS_Context::$isCli) {
  113. fwrite(STDERR, 'FreshRSS error during OPML import' . "\n");
  114. } else {
  115. Minz_Log::warning('Error during OPML import');
  116. }
  117. }
  118. }
  119. foreach ($list_files['json_starred'] as $article_file) {
  120. if (!is_string($article_file) || !$this->importJson($article_file, true)) {
  121. $ok = false;
  122. if (FreshRSS_Context::$isCli) {
  123. fwrite(STDERR, 'FreshRSS error during JSON stars import' . "\n");
  124. } else {
  125. Minz_Log::warning('Error during JSON stars import');
  126. }
  127. }
  128. }
  129. foreach ($list_files['json_feed'] as $article_file) {
  130. if (!is_string($article_file) || !$this->importJson($article_file)) {
  131. $ok = false;
  132. if (FreshRSS_Context::$isCli) {
  133. fwrite(STDERR, 'FreshRSS error during JSON feeds import' . "\n");
  134. } else {
  135. Minz_Log::warning('Error during JSON feeds import');
  136. }
  137. }
  138. }
  139. foreach ($list_files['ttrss_starred'] as $article_file) {
  140. $json = is_string($article_file) ? $this->ttrssXmlToJson($article_file) : false;
  141. if ($json === false || !$this->importJson($json, true)) {
  142. $ok = false;
  143. if (FreshRSS_Context::$isCli) {
  144. fwrite(STDERR, 'FreshRSS error during TT-RSS articles import' . "\n");
  145. } else {
  146. Minz_Log::warning('Error during TT-RSS articles import');
  147. }
  148. }
  149. }
  150. return $ok;
  151. }
  152. /**
  153. * This action handles import action.
  154. *
  155. * It must be reached by a POST request.
  156. *
  157. * Parameter is:
  158. * - file (default: nothing!)
  159. * Available file types are: zip, json or xml.
  160. */
  161. public function importAction(): void {
  162. if (!Minz_Request::isPost()) {
  163. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  164. }
  165. $file = $_FILES['file'] ?? null;
  166. $status_file = is_array($file) ? $file['error'] ?? -1 : -1;
  167. if (!is_array($file) || $status_file !== 0 || !is_string($file['name'] ?? null) || !is_string($file['tmp_name'] ?? null)) {
  168. Minz_Log::warning('File cannot be uploaded. Error code: ' . (is_numeric($status_file) ? $status_file : -1));
  169. Minz_Request::bad(_t('feedback.import_export.file_cannot_be_uploaded'), [ 'c' => 'importExport', 'a' => 'index' ]);
  170. return;
  171. }
  172. if (function_exists('set_time_limit')) {
  173. @set_time_limit(300);
  174. }
  175. $error = false;
  176. try {
  177. $error = !$this->importFile($file['name'], $file['tmp_name']);
  178. } catch (FreshRSS_ZipMissing_Exception) {
  179. Minz_Request::bad(
  180. _t('feedback.import_export.no_zip_extension'),
  181. ['c' => 'importExport', 'a' => 'index']
  182. );
  183. } catch (FreshRSS_Zip_Exception $ze) {
  184. Minz_Log::warning('ZIP archive cannot be imported. Error code: ' . $ze->zipErrorCode());
  185. Minz_Request::bad(
  186. _t('feedback.import_export.zip_error'),
  187. ['c' => 'importExport', 'a' => 'index']
  188. );
  189. }
  190. // And finally, we get import status and redirect to the home page
  191. $content_notif = $error === true ? _t('feedback.import_export.feeds_imported_with_errors') : _t('feedback.import_export.feeds_imported');
  192. Minz_Request::good(
  193. $content_notif,
  194. showNotification: FreshRSS_Context::userConf()->good_notification_timeout > 0
  195. );
  196. }
  197. /**
  198. * This method tries to guess the file type based on its name.
  199. *
  200. * It is a *very* basic guess file type function. Only based on filename.
  201. * That could be improved but should be enough for what we have to do.
  202. */
  203. private static function guessFileType(string $filename): string {
  204. if (str_ends_with($filename, '.zip')) {
  205. return 'zip';
  206. } elseif (str_ends_with($filename, '.txt')) {
  207. return 'txt';
  208. } elseif (stripos($filename, 'opml') !== false) {
  209. return 'opml';
  210. } elseif (str_ends_with($filename, '.json')) {
  211. if (str_contains($filename, 'starred')) {
  212. return 'json_starred';
  213. } else {
  214. return 'json_feed';
  215. }
  216. } elseif (str_ends_with($filename, '.xml')) {
  217. if (preg_match('/Tiny|tt-?rss/i', $filename)) {
  218. return 'ttrss_starred';
  219. } else {
  220. return 'opml';
  221. }
  222. }
  223. return 'unknown';
  224. }
  225. /**
  226. * Wraps a newline-separated list of feed URLs into a minimal OPML document
  227. * so it can be imported through the existing OPML pipeline.
  228. */
  229. private static function txtToOpml(string $contents): string {
  230. $utf8BOM = "\xEF\xBB\xBF";
  231. $contents = preg_replace('/^' . $utf8BOM . '/', '', $contents) ?? $contents;
  232. $outlines = '';
  233. foreach (preg_split('/\R/', $contents) ?: [] as $line) {
  234. $url = trim($line);
  235. if ($url === '' || str_starts_with($url, '#') || str_starts_with($url, '<')) {
  236. continue;
  237. }
  238. if (filter_var($url, FILTER_VALIDATE_URL) === false) {
  239. $message = 'TXT import: skipping invalid URL “' . \SimplePie\Misc::url_remove_credentials($url) . '”';
  240. if (FreshRSS_Context::$isCli) {
  241. fwrite(STDERR, $message . "\n");
  242. } else {
  243. Minz_Log::warning($message);
  244. }
  245. continue;
  246. }
  247. $escaped = htmlspecialchars($url, ENT_COMPAT | ENT_XML1, 'UTF-8');
  248. $outlines .= '<outline type="rss" text="' . $escaped . '" xmlUrl="' . $escaped . '" />' . "\n";
  249. }
  250. return '<?xml version="1.0" encoding="UTF-8"?>' . "\n"
  251. . '<opml version="2.0"><body>' . "\n"
  252. . $outlines
  253. . '</body></opml>' . "\n";
  254. }
  255. private function ttrssXmlToJson(string $xml): string|false {
  256. $table = (array)simplexml_load_string($xml, options: LIBXML_NOBLANKS | LIBXML_NOCDATA);
  257. $table['items'] = $table['article'] ?? [];
  258. if (!is_array($table['items'])) {
  259. $table['items'] = [];
  260. }
  261. unset($table['article']);
  262. for ($i = count($table['items']) - 1; $i >= 0; $i--) {
  263. $item = (array)($table['items'][$i]);
  264. $item = array_filter($item, static fn($v) =>
  265. // Filter out empty properties, potentially reported as empty objects
  266. (is_string($v) && trim($v) !== '') || !empty($v));
  267. $item['updated'] = is_string($item['updated'] ?? null) ? strtotime($item['updated']) : '';
  268. $item['published'] = $item['updated'];
  269. $item['content'] = ['content' => $item['content'] ?? ''];
  270. $item['categories'] = is_string($item['tag_cache'] ?? null) ? [$item['tag_cache']] : [];
  271. if (!empty($item['marked'])) {
  272. $item['categories'][] = 'user/-/state/com.google/starred';
  273. }
  274. if (!empty($item['published'])) {
  275. $item['categories'][] = 'user/-/state/com.google/broadcast';
  276. }
  277. if (is_string($item['label_cache'] ?? null)) {
  278. $labels_cache = json_decode($item['label_cache'], true);
  279. if (is_array($labels_cache)) {
  280. foreach ($labels_cache as $label_cache) {
  281. if (is_array($label_cache) && !empty($label_cache[1]) && is_string($label_cache[1])) {
  282. $item['categories'][] = 'user/-/label/' . trim($label_cache[1]);
  283. }
  284. }
  285. }
  286. }
  287. $item['alternate'] = [['href' => $item['link'] ?? '']];
  288. $item['origin'] = [
  289. 'title' => $item['feed_title'] ?? '',
  290. 'feedUrl' => $item['feed_url'] ?? '',
  291. ];
  292. $item['id'] = $item['guid'] ?? ($item['feed_url'] ?? $item['published']);
  293. $item['guid'] = $item['id'];
  294. $table['items'][$i] = $item;
  295. }
  296. return json_encode($table);
  297. }
  298. /**
  299. * This method import a JSON-based file (Google Reader format).
  300. *
  301. * $article_file the JSON file content.
  302. * true if articles from the file must be starred.
  303. * @return bool false if an error occurred, true otherwise.
  304. * @throws Minz_ConfigurationNamespaceException
  305. * @throws Minz_PDOConnectionException
  306. */
  307. private function importJson(string $article_file, bool $starred = false): bool {
  308. $article_object = json_decode($article_file, true);
  309. if (!is_array($article_object)) {
  310. if (FreshRSS_Context::$isCli) {
  311. fwrite(STDERR, 'FreshRSS error trying to import a non-JSON file' . "\n");
  312. } else {
  313. Minz_Log::warning('Try to import a non-JSON file');
  314. }
  315. return false;
  316. }
  317. $items = $article_object['items'] ?? $article_object;
  318. if (!is_array($items)) {
  319. $items = [];
  320. }
  321. $mark_as_read = FreshRSS_Context::userConf()->mark_when['reception'] ? 1 : 0;
  322. $error = false;
  323. $article_to_feed = [];
  324. $nb_feeds = count($this->feedDAO->listFeeds());
  325. $newFeedGuids = [];
  326. $limits = FreshRSS_Context::systemConf()->limits;
  327. // First, we check feeds of articles are in DB (and add them if needed).
  328. foreach ($items as &$item) {
  329. if (!is_array($item)) {
  330. continue;
  331. }
  332. if (!is_string($item['guid'] ?? null) && is_string($item['id'] ?? null)) {
  333. $item['guid'] = $item['id'];
  334. }
  335. if (!is_string($item['guid'] ?? null)) {
  336. continue;
  337. }
  338. if (!is_array($item['origin'] ?? null)) {
  339. $item['origin'] = [];
  340. }
  341. if (!is_string($item['origin']['title'] ?? null) || trim($item['origin']['title']) === '') {
  342. $item['origin']['title'] = 'Import';
  343. }
  344. if (is_string($item['origin']['feedUrl'] ?? null)) {
  345. $feedUrl = $item['origin']['feedUrl'];
  346. } elseif (is_string($item['origin']['streamId'] ?? null) && str_starts_with($item['origin']['streamId'], 'feed/')) {
  347. $feedUrl = substr($item['origin']['streamId'], 5); //Google Reader
  348. $item['origin']['feedUrl'] = $feedUrl;
  349. } elseif (is_string($item['origin']['htmlUrl'] ?? null)) {
  350. $feedUrl = $item['origin']['htmlUrl'];
  351. } else {
  352. $feedUrl = 'http://import.localhost/import.xml';
  353. $item['origin']['feedUrl'] = $feedUrl;
  354. $item['origin']['disable'] = 'true';
  355. }
  356. $feedUrlOriginal = $feedUrl;
  357. $feedUrl = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($feedUrl) ?: '');
  358. try {
  359. $feed = new FreshRSS_Feed($feedUrl);
  360. } catch (FreshRSS_BadUrl_Exception) {
  361. Minz_Log::warning('Could not add feed with invalid URL "' . \SimplePie\Misc::url_remove_credentials($feedUrlOriginal) . '" during JSON import');
  362. continue;
  363. }
  364. $feed = $this->feedDAO->searchByUrl($feed->url());
  365. if ($feed === null) {
  366. // Feed does not exist in DB,we should to try to add it.
  367. if ((!FreshRSS_Context::$isCli) && ($nb_feeds >= $limits['max_feeds'])) {
  368. // Oops, no more place!
  369. Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds']));
  370. } else {
  371. $origin = array_filter($item['origin'], fn($value, $key): bool => is_string($key) && is_string($value), ARRAY_FILTER_USE_BOTH);
  372. $feed = $this->addFeedJson($origin);
  373. }
  374. if ($feed === null) {
  375. // Still null? It means something went wrong.
  376. $error = true;
  377. } else {
  378. $nb_feeds++;
  379. }
  380. }
  381. if ($feed !== null) {
  382. $article_to_feed[$item['guid']] = $feed->id();
  383. if (!isset($newFeedGuids['f_' . $feed->id()])) {
  384. $newFeedGuids['f_' . $feed->id()] = [];
  385. }
  386. $newFeedGuids['f_' . $feed->id()][] = safe_ascii($item['guid']);
  387. }
  388. }
  389. $tagDAO = FreshRSS_Factory::createTagDao();
  390. $labels = FreshRSS_Context::labels();
  391. $knownLabels = [];
  392. foreach ($labels as $label) {
  393. $knownLabels[$label->name()]['id'] = $label->id();
  394. $knownLabels[$label->name()]['articles'] = [];
  395. }
  396. unset($labels);
  397. // For each feed, check existing GUIDs already in database.
  398. $existingHashForGuids = [];
  399. foreach ($newFeedGuids as $feedId => $newGuids) {
  400. $existingHashForGuids[$feedId] = $this->entryDAO->listHashForFeedGuids((int)substr($feedId, 2), $newGuids);
  401. }
  402. unset($newFeedGuids);
  403. // Then, articles are imported.
  404. $newGuids = [];
  405. $this->entryDAO->beginTransaction();
  406. foreach ($items as &$item) {
  407. if (!is_array($item) || empty($item['guid']) || !is_string($item['guid']) || empty($article_to_feed[$item['guid']])) {
  408. // Related feed does not exist for this entry, do nothing.
  409. continue;
  410. }
  411. $feed_id = $article_to_feed[$item['guid']];
  412. $author = is_string($item['author'] ?? null) ? Minz_Helper::htmlspecialchars_utf8($item['author']) : '';
  413. $is_starred = null; // null is used to preserve the current state if that item exists and is already starred
  414. $is_read = null;
  415. $tags = is_array($item['categories'] ?? null) ? $item['categories'] : [];
  416. $labels = [];
  417. for ($i = count($tags) - 1; $i >= 0; $i--) {
  418. $tag = $tags[$i];
  419. if (!is_string($tag)) {
  420. unset($tags[$i]);
  421. continue;
  422. }
  423. $tag = trim($tag);
  424. if (preg_match('%^user/[A-Za-z0-9_-]+/%', $tag)) {
  425. if (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/starred$%', $tag)) {
  426. $is_starred = true;
  427. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/read$%', $tag)) {
  428. $is_read = true;
  429. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/state/com.google/unread$%', $tag)) {
  430. $is_read = false;
  431. } elseif (preg_match('%^user/[A-Za-z0-9_-]+/label/\s*(?P<tag>.+?)\s*$%', $tag, $matches)) {
  432. $labels[] = $matches['tag'];
  433. }
  434. unset($tags[$i]);
  435. }
  436. }
  437. $tags = Minz_Helper::htmlspecialchars_utf8(array_values(array_filter($tags, 'is_string')));
  438. if ($starred && !$is_starred) {
  439. //If the article has no label, mark it as starred (old format)
  440. $is_starred = empty($labels);
  441. }
  442. if ($is_read === null) {
  443. $is_read = $mark_as_read;
  444. }
  445. if (is_array($item['alternate']) && is_array($item['alternate'][0] ?? null) && is_string($item['alternate'][0]['href'] ?? null)) {
  446. $url = $item['alternate'][0]['href'];
  447. } elseif (is_string($item['url'] ?? null)) {
  448. $url = $item['url']; //FeedBin
  449. } else {
  450. $url = '';
  451. }
  452. $url = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($url) ?: '');
  453. $title = is_string($item['title'] ?? null) ? $item['title'] : $url;
  454. $title = Minz_Helper::htmlspecialchars_utf8($title);
  455. if (is_array($item['content'] ?? null) && is_string($item['content']['content'] ?? null)) {
  456. $content = $item['content']['content'];
  457. } elseif (is_array($item['summary']) && is_string($item['summary']['content'] ?? null)) {
  458. $content = $item['summary']['content'];
  459. } elseif (is_string($item['content'] ?? null)) {
  460. $content = $item['content']; //FeedBin
  461. } else {
  462. $content = '';
  463. }
  464. $content = FreshRSS_SimplePieCustom::sanitizeHTML($content, $url);
  465. if (is_int($item['published'] ?? null) || is_string($item['published'] ?? null)) {
  466. $published = (string)$item['published'];
  467. } elseif (is_int($item['timestampUsec'] ?? null) || is_string($item['timestampUsec'] ?? null)) {
  468. $published = substr((string)$item['timestampUsec'], 0, -6);
  469. } elseif (is_int($item['updated'] ?? null) || is_string($item['updated'] ?? null)) {
  470. $published = (string)$item['updated'];
  471. } else {
  472. $published = '0';
  473. }
  474. if (!ctype_digit($published)) {
  475. $published = (string)(strtotime($published) ?: 0);
  476. }
  477. if (strlen($published) > 10) { // Milliseconds, e.g. Feedly
  478. $published = substr($published, 0, -3);
  479. if (!is_numeric($published)) {
  480. $published = '0'; // For PHPStan
  481. }
  482. }
  483. $entry = new FreshRSS_Entry(
  484. $feed_id, $item['guid'], $title, $author,
  485. $content, $url, $published, $is_read, $is_starred
  486. );
  487. $entry->_id(uTimeString());
  488. $entry->_tags($tags);
  489. if (isset($newGuids[$entry->guid()])) {
  490. continue; //Skip subsequent articles with same GUID
  491. }
  492. $newGuids[$entry->guid()] = true;
  493. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeInsert, $entry);
  494. if (!($entry instanceof FreshRSS_Entry)) {
  495. // An extension has returned a null value, there is nothing to insert.
  496. continue;
  497. }
  498. if (isset($existingHashForGuids['f_' . $feed_id][$entry->guid()])) {
  499. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeUpdate, $entry);
  500. if (!($entry instanceof FreshRSS_Entry)) {
  501. // An extension has returned a null value, there is nothing to insert.
  502. continue;
  503. }
  504. $ok = $this->entryDAO->updateEntry($entry->toArray());
  505. } else {
  506. $entry->_lastSeen(time());
  507. $entry = Minz_ExtensionManager::callHook(Minz_HookType::EntryBeforeAdd, $entry);
  508. if (!($entry instanceof FreshRSS_Entry)) {
  509. // An extension has returned a null value, there is nothing to insert.
  510. continue;
  511. }
  512. $ok = $this->entryDAO->addEntry($entry->toArray());
  513. }
  514. foreach ($labels as $labelName) {
  515. if (empty($knownLabels[$labelName]['id'])) {
  516. $labelId = $tagDAO->addTag(['name' => $labelName]);
  517. $knownLabels[$labelName]['id'] = $labelId;
  518. $knownLabels[$labelName]['articles'] = [];
  519. }
  520. $knownLabels[$labelName]['articles'][] = [
  521. //'id' => $entry->id(), //ID changes after commitNewEntries()
  522. 'id_feed' => $entry->feedId(),
  523. 'guid' => $entry->guid(),
  524. ];
  525. }
  526. $error |= ($ok === false);
  527. }
  528. $this->entryDAO->commit();
  529. $this->entryDAO->beginTransaction();
  530. $this->entryDAO->commitNewEntries();
  531. $this->feedDAO->updateCachedValues();
  532. $this->entryDAO->commit();
  533. $this->entryDAO->beginTransaction();
  534. foreach ($knownLabels as $labelName => $knownLabel) {
  535. $labelId = $knownLabel['id'];
  536. if (!$labelId) {
  537. continue;
  538. }
  539. foreach ($knownLabel['articles'] as $article) {
  540. $entryId = $this->entryDAO->searchIdByGuid($article['id_feed'], $article['guid']);
  541. if ($entryId != null) {
  542. $tagDAO->tagEntry($labelId, $entryId);
  543. } else {
  544. Minz_Log::warning('Could not add label "' . $labelName . '" to entry "' . $article['guid'] . '" in feed ' . $article['id_feed']);
  545. }
  546. }
  547. }
  548. $this->entryDAO->commit();
  549. return !$error;
  550. }
  551. /**
  552. * This method import a JSON-based feed (Google Reader format).
  553. *
  554. * @param array<string,string> $origin represents a feed.
  555. * @return FreshRSS_Feed|null if feed is in database at the end of the process, else null.
  556. */
  557. private function addFeedJson(array $origin): ?FreshRSS_Feed {
  558. $return = null;
  559. if (!empty($origin['feedUrl'])) {
  560. $url = $origin['feedUrl'];
  561. } elseif (!empty($origin['htmlUrl'])) {
  562. $url = $origin['htmlUrl'];
  563. } else {
  564. return null;
  565. }
  566. $url = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($url) ?: '');
  567. if (!empty($origin['htmlUrl'])) {
  568. $website = $origin['htmlUrl'];
  569. } elseif (!empty($origin['feedUrl'])) {
  570. $website = $origin['feedUrl'];
  571. } else {
  572. $website = '';
  573. }
  574. $website = Minz_Helper::htmlspecialchars_utf8(FreshRSS_http_Util::checkUrl($website) ?: '');
  575. $name = empty($origin['title']) ? $website : $origin['title'];
  576. $name = Minz_Helper::htmlspecialchars_utf8($name);
  577. $cat_id = FreshRSS_CategoryDAO::DEFAULTCATEGORYID;
  578. $cat_name = Minz_Helper::htmlspecialchars_utf8(trim($origin['category'] ?? ''));
  579. if ($cat_name !== '') {
  580. $new_cat = $this->categoryDAO->searchByName($cat_name);
  581. $cat_id = $new_cat?->id() ?: $this->categoryDAO->addCategory(['name' => $cat_name]) ?: FreshRSS_CategoryDAO::DEFAULTCATEGORYID;
  582. }
  583. try {
  584. // Create a Feed object and add it in database.
  585. $feed = new FreshRSS_Feed($url);
  586. $feed->_categoryId($cat_id);
  587. $feed->_name($name);
  588. $feed->_website($website);
  589. if (!empty($origin['disable'])) {
  590. $feed->_mute(true);
  591. }
  592. // Call the extension hook
  593. $feed = Minz_ExtensionManager::callHook(Minz_HookType::FeedBeforeInsert, $feed);
  594. if ($feed instanceof FreshRSS_Feed) {
  595. // addFeedObject checks if feed is already in DB so nothing else to
  596. // check here.
  597. $id = $this->feedDAO->addFeedObject($feed);
  598. if ($id !== false) {
  599. $feed->_id($id);
  600. $return = $feed;
  601. }
  602. }
  603. } catch (FreshRSS_Feed_Exception $e) {
  604. if (FreshRSS_Context::$isCli) {
  605. fwrite(STDERR, 'FreshRSS error during JSON feed import: ' . $e->getMessage() . "\n");
  606. } else {
  607. Minz_Log::warning($e->getMessage());
  608. }
  609. }
  610. return $return;
  611. }
  612. /**
  613. * This action handles export action.
  614. *
  615. * This action must be reached by a POST request.
  616. *
  617. * Parameters are:
  618. * - export_opml (default: false)
  619. * - export_starred (default: false)
  620. * - export_labelled (default: false)
  621. * - export_feeds (default: []) a list of feed ids
  622. */
  623. public function exportAction(): void {
  624. if (!Minz_Request::isPost()) {
  625. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  626. return;
  627. }
  628. $username = Minz_User::name() ?? '_';
  629. $export_service = new FreshRSS_Export_Service($username);
  630. $export_opml = Minz_Request::paramBoolean('export_opml');
  631. $export_starred = Minz_Request::paramBoolean('export_starred');
  632. $export_labelled = Minz_Request::paramBoolean('export_labelled');
  633. /** @var array<numeric-string> */
  634. $export_feeds = Minz_Request::paramArray('export_feeds');
  635. $max_number_entries = 50;
  636. $exported_files = [];
  637. if ($export_opml) {
  638. [$filename, $content] = $export_service->generateOpml();
  639. $exported_files[$filename] = $content;
  640. }
  641. // Starred and labelled entries are merged in the same `starred` file
  642. // to avoid duplication of content.
  643. if ($export_starred && $export_labelled) {
  644. [$filename, $content] = $export_service->generateStarredEntries('ST');
  645. $exported_files[$filename] = $content;
  646. } elseif ($export_starred) {
  647. [$filename, $content] = $export_service->generateStarredEntries('S');
  648. $exported_files[$filename] = $content;
  649. } elseif ($export_labelled) {
  650. [$filename, $content] = $export_service->generateStarredEntries('T');
  651. $exported_files[$filename] = $content;
  652. }
  653. foreach ($export_feeds as $feed_id) {
  654. $result = $export_service->generateFeedEntries((int)$feed_id, $max_number_entries);
  655. if ($result === null) {
  656. // It means the actual feed_id doesn’t correspond to any existing feed
  657. continue;
  658. }
  659. [$filename, $content] = $result;
  660. $exported_files[$filename] = $content;
  661. }
  662. $nb_files = count($exported_files);
  663. if ($nb_files <= 0) {
  664. // There’s nothing to do, there are no files to export
  665. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  666. return;
  667. }
  668. if ($nb_files === 1) {
  669. // If we only have one file, we just export it as it is
  670. $filename = key($exported_files);
  671. $content = $exported_files[$filename];
  672. } else {
  673. // More files? Let’s compress them in a Zip archive
  674. if (!extension_loaded('zip')) {
  675. // Oops, there is no ZIP extension!
  676. Minz_Request::bad(
  677. _t('feedback.import_export.export_no_zip_extension'),
  678. ['c' => 'importExport', 'a' => 'index']
  679. );
  680. return;
  681. }
  682. [$filename, $content] = $export_service->zip($exported_files);
  683. }
  684. if (!is_string($content)) {
  685. Minz_Request::bad(_t('feedback.import_export.zip_error'), ['c' => 'importExport', 'a' => 'index']);
  686. return;
  687. }
  688. $content_type = self::filenameToContentType($filename);
  689. header('Content-Type: ' . $content_type);
  690. header('Content-disposition: attachment; filename="' . $filename . '"');
  691. $this->view->_layout(null);
  692. $this->view->content = $content;
  693. }
  694. /**
  695. * Return the Content-Type corresponding to a filename.
  696. *
  697. * If the type of the filename is not supported, it returns
  698. * `application/octet-stream` by default.
  699. */
  700. private static function filenameToContentType(string $filename): string {
  701. $filetype = self::guessFileType($filename);
  702. return match ($filetype) {
  703. 'zip' => 'application/zip',
  704. 'opml' => 'application/xml; charset=utf-8',
  705. 'json_starred', 'json_feed' => 'application/json; charset=utf-8',
  706. default => 'application/octet-stream',
  707. };
  708. }
  709. private const REGEX_SQLITE_FILENAME = '/^(?![.-])[0-9a-zA-Z_.@ #&()~\-]{1,128}\.sqlite$/';
  710. private function listSqliteArchives(): void {
  711. $this->view->sqliteArchives = [];
  712. $files = glob(USERS_PATH . '/' . Minz_User::name() . '/*.sqlite', GLOB_NOSORT) ?: [];
  713. foreach ($files as $file) {
  714. $archive = [
  715. 'name' => basename($file),
  716. 'size' => @filesize($file),
  717. 'mtime' => @filemtime($file),
  718. ];
  719. if ($archive['size'] != false && $archive['mtime'] != false && preg_match(self::REGEX_SQLITE_FILENAME, $archive['name'])) {
  720. $this->view->sqliteArchives[] = $archive;
  721. }
  722. }
  723. // Sort by time, newest first:
  724. usort($this->view->sqliteArchives, static fn(array $a, array $b): int => $b['mtime'] <=> $a['mtime']);
  725. }
  726. public function sqliteAction(): void {
  727. if (!Minz_Request::isPost()) {
  728. Minz_Request::forward(['c' => 'importExport', 'a' => 'index'], true);
  729. }
  730. $sqlite = Minz_Request::paramString('sqlite');
  731. if (!preg_match(self::REGEX_SQLITE_FILENAME, $sqlite)) {
  732. Minz_Error::error(404);
  733. return;
  734. }
  735. $path = USERS_PATH . '/' . Minz_User::name() . '/' . $sqlite;
  736. if (!file_exists($path) || @filesize($path) == false || @filemtime($path) == false) {
  737. Minz_Error::error(404);
  738. return;
  739. }
  740. $this->view->sqlitePath = $path;
  741. $this->view->sqliteName = basename($path);
  742. if ($this->view->sqliteName === 'db.sqlite') {
  743. $username = Minz_User::name() ?? '_';
  744. $date = date('Y-m-d_H-i-s', filemtime($path) ?: time()); // @phpstan-ignore ternary.alwaysTrue (for additional safety)
  745. $this->view->sqliteName = 'freshrss_' . $username . '_' . $date . '_db.sqlite';
  746. }
  747. $this->view->_layout(null);
  748. }
  749. }