importExportController.php 28 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900
  1. <?php
  2. /**
  3. * Controller to handle every import and export actions.
  4. */
  5. class FreshRSS_importExport_Controller extends Minz_ActionController {
  6. /**
  7. * This action is called before every other action in that class. It is
  8. * the common boiler plate for every action. It is triggered by the
  9. * underlying framework.
  10. */
  11. public function firstAction() {
  12. if (!FreshRSS_Auth::hasAccess()) {
  13. Minz_Error::error(403);
  14. }
  15. require_once(LIB_PATH . '/lib_opml.php');
  16. $this->catDAO = new FreshRSS_CategoryDAO();
  17. $this->entryDAO = FreshRSS_Factory::createEntryDao();
  18. $this->feedDAO = FreshRSS_Factory::createFeedDao();
  19. }
  20. /**
  21. * This action displays the main page for import / export system.
  22. */
  23. public function indexAction() {
  24. $this->view->feeds = $this->feedDAO->listFeeds();
  25. Minz_View::prependTitle(_t('sub.import_export.title') . ' · ');
  26. }
  27. public function importFile($name, $path, $username = null) {
  28. require_once(LIB_PATH . '/lib_opml.php');
  29. $this->catDAO = new FreshRSS_CategoryDAO($username);
  30. $this->entryDAO = FreshRSS_Factory::createEntryDao($username);
  31. $this->feedDAO = FreshRSS_Factory::createFeedDao($username);
  32. $type_file = self::guessFileType($name);
  33. $list_files = array(
  34. 'opml' => array(),
  35. 'json_starred' => array(),
  36. 'json_feed' => array(),
  37. 'ttrss_starred' => array(),
  38. );
  39. // We try to list all files according to their type
  40. $list = array();
  41. if ($type_file === 'zip' && extension_loaded('zip')) {
  42. $zip = zip_open($path);
  43. if (!is_resource($zip)) {
  44. // zip_open cannot open file: something is wrong
  45. throw new FreshRSS_Zip_Exception($zip);
  46. }
  47. while (($zipfile = zip_read($zip)) !== false) {
  48. if (!is_resource($zipfile)) {
  49. // zip_entry() can also return an error code!
  50. throw new FreshRSS_Zip_Exception($zipfile);
  51. } else {
  52. $type_zipfile = self::guessFileType(zip_entry_name($zipfile));
  53. if ($type_file !== 'unknown') {
  54. $list_files[$type_zipfile][] = zip_entry_read(
  55. $zipfile,
  56. zip_entry_filesize($zipfile)
  57. );
  58. }
  59. }
  60. }
  61. zip_close($zip);
  62. } elseif ($type_file === 'zip') {
  63. // ZIP extension is not loaded
  64. throw new FreshRSS_ZipMissing_Exception();
  65. } elseif ($type_file !== 'unknown') {
  66. $list_files[$type_file][] = file_get_contents($path);
  67. }
  68. // Import file contents.
  69. // OPML first(so categories and feeds are imported)
  70. // Starred articles then so the "favourite" status is already set
  71. // And finally all other files.
  72. $ok = true;
  73. foreach ($list_files['opml'] as $opml_file) {
  74. if (!$this->importOpml($opml_file)) {
  75. $ok = false;
  76. if (FreshRSS_Context::$isCli) {
  77. fwrite(STDERR, 'FreshRSS error during OPML import' . "\n");
  78. } else {
  79. Minz_Log::warning('Error during OPML import');
  80. }
  81. }
  82. }
  83. foreach ($list_files['json_starred'] as $article_file) {
  84. if (!$this->importJson($article_file, true)) {
  85. $ok = false;
  86. if (FreshRSS_Context::$isCli) {
  87. fwrite(STDERR, 'FreshRSS error during JSON stars import' . "\n");
  88. } else {
  89. Minz_Log::warning('Error during JSON stars import');
  90. }
  91. }
  92. }
  93. foreach ($list_files['json_feed'] as $article_file) {
  94. if (!$this->importJson($article_file)) {
  95. $ok = false;
  96. if (FreshRSS_Context::$isCli) {
  97. fwrite(STDERR, 'FreshRSS error during JSON feeds import' . "\n");
  98. } else {
  99. Minz_Log::warning('Error during JSON feeds import');
  100. }
  101. }
  102. }
  103. foreach ($list_files['ttrss_starred'] as $article_file) {
  104. $json = $this->ttrssXmlToJson($article_file);
  105. if (!$this->importJson($json, true)) {
  106. $ok = false;
  107. if (FreshRSS_Context::$isCli) {
  108. fwrite(STDERR, 'FreshRSS error during TT-RSS articles import' . "\n");
  109. } else {
  110. Minz_Log::warning('Error during TT-RSS articles import');
  111. }
  112. }
  113. }
  114. return $ok;
  115. }
  116. /**
  117. * This action handles import action.
  118. *
  119. * It must be reached by a POST request.
  120. *
  121. * Parameter is:
  122. * - file (default: nothing!)
  123. * Available file types are: zip, json or xml.
  124. */
  125. public function importAction() {
  126. if (!Minz_Request::isPost()) {
  127. Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true);
  128. }
  129. $file = $_FILES['file'];
  130. $status_file = $file['error'];
  131. if ($status_file !== 0) {
  132. Minz_Log::warning('File cannot be uploaded. Error code: ' . $status_file);
  133. Minz_Request::bad(_t('feedback.import_export.file_cannot_be_uploaded'),
  134. array('c' => 'importExport', 'a' => 'index'));
  135. }
  136. @set_time_limit(300);
  137. $error = false;
  138. try {
  139. $error = !$this->importFile($file['name'], $file['tmp_name']);
  140. } catch (FreshRSS_ZipMissing_Exception $zme) {
  141. Minz_Request::bad(_t('feedback.import_export.no_zip_extension'),
  142. array('c' => 'importExport', 'a' => 'index'));
  143. } catch (FreshRSS_Zip_Exception $ze) {
  144. Minz_Log::warning('ZIP archive cannot be imported. Error code: ' . $ze->zipErrorCode());
  145. Minz_Request::bad(_t('feedback.import_export.zip_error'),
  146. array('c' => 'importExport', 'a' => 'index'));
  147. }
  148. // And finally, we get import status and redirect to the home page
  149. Minz_Session::_param('actualize_feeds', true);
  150. $content_notif = $error === true ? _t('feedback.import_export.feeds_imported_with_errors') : _t('feedback.import_export.feeds_imported');
  151. Minz_Request::good($content_notif);
  152. }
  153. /**
  154. * This method tries to guess the file type based on its name.
  155. *
  156. * Itis a *very* basic guess file type function. Only based on filename.
  157. * That's could be improved but should be enough for what we have to do.
  158. */
  159. private static function guessFileType($filename) {
  160. if (substr_compare($filename, '.zip', -4) === 0) {
  161. return 'zip';
  162. } elseif (substr_compare($filename, '.opml', -5) === 0) {
  163. return 'opml';
  164. } elseif (substr_compare($filename, '.json', -5) === 0) {
  165. if (strpos($filename, 'starred') !== false) {
  166. return 'json_starred';
  167. } else {
  168. return 'json_feed';
  169. }
  170. } elseif (substr_compare($filename, '.xml', -4) === 0) {
  171. if (preg_match('/Tiny|tt-?rss/i', $filename)) {
  172. return 'ttrss_starred';
  173. } else {
  174. return 'opml';
  175. }
  176. }
  177. return 'unknown';
  178. }
  179. /**
  180. * This method parses and imports an OPML file.
  181. *
  182. * @param string $opml_file the OPML file content.
  183. * @return boolean false if an error occured, true otherwise.
  184. */
  185. private function importOpml($opml_file) {
  186. $opml_array = array();
  187. try {
  188. $opml_array = libopml_parse_string($opml_file, false);
  189. } catch (LibOPML_Exception $e) {
  190. if (FreshRSS_Context::$isCli) {
  191. fwrite(STDERR, 'FreshRSS error during OPML parsing: ' . $e->getMessage() . "\n");
  192. } else {
  193. Minz_Log::warning($e->getMessage());
  194. }
  195. return false;
  196. }
  197. $this->catDAO->checkDefault();
  198. return $this->addOpmlElements($opml_array['body']);
  199. }
  200. /**
  201. * This method imports an OPML file based on its body.
  202. *
  203. * @param array $opml_elements an OPML element (body or outline).
  204. * @param string $parent_cat the name of the parent category.
  205. * @return boolean false if an error occured, true otherwise.
  206. */
  207. private function addOpmlElements($opml_elements, $parent_cat = null) {
  208. $ok = true;
  209. $nb_feeds = count($this->feedDAO->listFeeds());
  210. $nb_cats = count($this->catDAO->listCategories(false));
  211. $limits = FreshRSS_Context::$system_conf->limits;
  212. foreach ($opml_elements as $elt) {
  213. if (isset($elt['xmlUrl'])) {
  214. // If xmlUrl exists, it means it is a feed
  215. if (FreshRSS_Context::$isCli && $nb_feeds >= $limits['max_feeds']) {
  216. Minz_Log::warning(_t('feedback.sub.feed.over_max',
  217. $limits['max_feeds']));
  218. $ok = false;
  219. continue;
  220. }
  221. if ($this->addFeedOpml($elt, $parent_cat)) {
  222. $nb_feeds++;
  223. } else {
  224. $ok = false;
  225. }
  226. } else {
  227. // No xmlUrl? It should be a category!
  228. $limit_reached = ($nb_cats >= $limits['max_categories']);
  229. if (!FreshRSS_Context::$isCli && $limit_reached) {
  230. Minz_Log::warning(_t('feedback.sub.category.over_max',
  231. $limits['max_categories']));
  232. $ok = false;
  233. continue;
  234. }
  235. if ($this->addCategoryOpml($elt, $parent_cat, $limit_reached)) {
  236. $nb_cats++;
  237. } else {
  238. $ok = false;
  239. }
  240. }
  241. }
  242. return $ok;
  243. }
  244. /**
  245. * This method imports an OPML feed element.
  246. *
  247. * @param array $feed_elt an OPML element (must be a feed element).
  248. * @param string $parent_cat the name of the parent category.
  249. * @return boolean false if an error occured, true otherwise.
  250. */
  251. private function addFeedOpml($feed_elt, $parent_cat) {
  252. if ($parent_cat == null) {
  253. // This feed has no parent category so we get the default one
  254. $this->catDAO->checkDefault();
  255. $default_cat = $this->catDAO->getDefault();
  256. $parent_cat = $default_cat->name();
  257. }
  258. $cat = $this->catDAO->searchByName($parent_cat);
  259. if ($cat == null) {
  260. // If there is not $cat, it means parent category does not exist in
  261. // database.
  262. // If it happens, take the default category.
  263. $this->catDAO->checkDefault();
  264. $cat = $this->catDAO->getDefault();
  265. }
  266. // We get different useful information
  267. $url = Minz_Helper::htmlspecialchars_utf8($feed_elt['xmlUrl']);
  268. $name = Minz_Helper::htmlspecialchars_utf8($feed_elt['text']);
  269. $website = '';
  270. if (isset($feed_elt['htmlUrl'])) {
  271. $website = Minz_Helper::htmlspecialchars_utf8($feed_elt['htmlUrl']);
  272. }
  273. $description = '';
  274. if (isset($feed_elt['description'])) {
  275. $description = Minz_Helper::htmlspecialchars_utf8($feed_elt['description']);
  276. }
  277. $error = false;
  278. try {
  279. // Create a Feed object and add it in DB
  280. $feed = new FreshRSS_Feed($url);
  281. $feed->_category($cat->id());
  282. $feed->_name($name);
  283. $feed->_website($website);
  284. $feed->_description($description);
  285. // Call the extension hook
  286. $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed);
  287. if ($feed != null) {
  288. // addFeedObject checks if feed is already in DB so nothing else to
  289. // check here
  290. $id = $this->feedDAO->addFeedObject($feed);
  291. $error = ($id === false);
  292. } else {
  293. $error = true;
  294. }
  295. } catch (FreshRSS_Feed_Exception $e) {
  296. if (FreshRSS_Context::$isCli) {
  297. fwrite(STDERR, 'FreshRSS error during OPML feed import: ' . $e->getMessage() . "\n");
  298. } else {
  299. Minz_Log::warning($e->getMessage());
  300. }
  301. $error = true;
  302. }
  303. if ($error) {
  304. if (FreshRSS_Context::$isCli) {
  305. fwrite(STDERR, 'FreshRSS error during OPML feed import from URL: ' . $url . ' in category ' . $cat->id() . "\n");
  306. } else {
  307. Minz_Log::warning('Error during OPML feed import from URL: ' . $url . ' in category ' . $cat->id());
  308. }
  309. }
  310. return !$error;
  311. }
  312. /**
  313. * This method imports an OPML category element.
  314. *
  315. * @param array $cat_elt an OPML element (must be a category element).
  316. * @param string $parent_cat the name of the parent category.
  317. * @param boolean $cat_limit_reached indicates if category limit has been reached.
  318. * if yes, category is not added (but we try for feeds!)
  319. * @return boolean false if an error occured, true otherwise.
  320. */
  321. private function addCategoryOpml($cat_elt, $parent_cat, $cat_limit_reached) {
  322. // Create a new Category object
  323. $catName = Minz_Helper::htmlspecialchars_utf8($cat_elt['text']);
  324. $cat = new FreshRSS_Category($catName);
  325. $error = true;
  326. if (FreshRSS_Context::$isCli || !$cat_limit_reached) {
  327. $id = $this->catDAO->addCategoryObject($cat);
  328. $error = ($id === false);
  329. }
  330. if ($error) {
  331. if (FreshRSS_Context::$isCli) {
  332. fwrite(STDERR, 'FreshRSS error during OPML category import from URL: ' . $catName . "\n");
  333. } else {
  334. Minz_Log::warning('Error during OPML category import from URL: ' . $catName);
  335. }
  336. }
  337. if (isset($cat_elt['@outlines'])) {
  338. // Our cat_elt contains more categories or more feeds, so we
  339. // add them recursively.
  340. // Note: FreshRSS does not support yet category arborescence
  341. $error &= !$this->addOpmlElements($cat_elt['@outlines'], $catName);
  342. }
  343. return !$error;
  344. }
  345. private function ttrssXmlToJson($xml) {
  346. $table = (array)simplexml_load_string($xml, null, LIBXML_NOCDATA);
  347. $table['items'] = isset($table['article']) ? $table['article'] : array();
  348. unset($table['article']);
  349. for ($i = count($table['items']) - 1; $i >= 0; $i--) {
  350. $item = (array)($table['items'][$i]);
  351. $item['updated'] = isset($item['updated']) ? strtotime($item['updated']) : '';
  352. $item['published'] = $item['updated'];
  353. $item['content'] = array('content' => isset($item['content']) ? $item['content'] : '');
  354. $item['categories'] = isset($item['tag_cache']) ? array($item['tag_cache']) : array();
  355. if (!empty($item['marked'])) {
  356. $item['categories'][] = 'user/-/state/com.google/starred';
  357. }
  358. if (!empty($item['published'])) {
  359. $item['categories'][] = 'user/-/state/com.google/broadcast';
  360. }
  361. if (!empty($item['label_cache'])) {
  362. $labels_cache = json_decode($item['label_cache'], true);
  363. if (is_array($labels_cache)) {
  364. foreach ($labels_cache as $label_cache) {
  365. if (!empty($label_cache[1])) {
  366. $item['categories'][] = 'user/-/label/' . trim($label_cache[1]);
  367. }
  368. }
  369. }
  370. }
  371. $item['alternate'][0]['href'] = isset($item['link']) ? $item['link'] : '';
  372. $item['origin'] = array(
  373. 'title' => isset($item['feed_title']) ? $item['feed_title'] : '',
  374. 'feedUrl' => isset($item['feed_url']) ? $item['feed_url'] : '',
  375. );
  376. $item['id'] = isset($item['guid']) ? $item['guid'] : (isset($item['feed_url']) ? $item['feed_url'] : $item['published']);
  377. $table['items'][$i] = $item;
  378. }
  379. return json_encode($table);
  380. }
  381. /**
  382. * This method import a JSON-based file (Google Reader format).
  383. *
  384. * @param string $article_file the JSON file content.
  385. * @param boolean $starred true if articles from the file must be starred.
  386. * @return boolean false if an error occured, true otherwise.
  387. */
  388. private function importJson($article_file, $starred = false) {
  389. $article_object = json_decode($article_file, true);
  390. if ($article_object == null) {
  391. if (FreshRSS_Context::$isCli) {
  392. fwrite(STDERR, 'FreshRSS error trying to import a non-JSON file' . "\n");
  393. } else {
  394. Minz_Log::warning('Try to import a non-JSON file');
  395. }
  396. return false;
  397. }
  398. $items = isset($article_object['items']) ? $article_object['items'] : $article_object;
  399. $mark_as_read = FreshRSS_Context::$user_conf->mark_when['reception'] ? 1 : 0;
  400. $error = false;
  401. $article_to_feed = array();
  402. $nb_feeds = count($this->feedDAO->listFeeds());
  403. $newFeedGuids = array();
  404. $limits = FreshRSS_Context::$system_conf->limits;
  405. // First, we check feeds of articles are in DB (and add them if needed).
  406. foreach ($items as $item) {
  407. if (!isset($item['origin'])) {
  408. $item['origin'] = array('title' => 'Import');
  409. }
  410. if (!empty($item['origin']['feedUrl'])) {
  411. $feedUrl = $item['origin']['feedUrl'];
  412. } elseif (!empty($item['origin']['streamId']) && strpos($item['origin']['streamId'], 'feed/') === 0) {
  413. $feedUrl = substr($item['origin']['streamId'], 5); //Google Reader
  414. $item['origin']['feedUrl'] = $feedUrl;
  415. } elseif (!empty($item['origin']['htmlUrl'])) {
  416. $feedUrl = $item['origin']['htmlUrl'];
  417. } else {
  418. $feedUrl = 'http://import.localhost/import.xml';
  419. $item['origin']['feedUrl'] = $feedUrl;
  420. $item['origin']['disable'] = true;
  421. }
  422. $feed = new FreshRSS_Feed($feedUrl);
  423. $feed = $this->feedDAO->searchByUrl($feed->url());
  424. if ($feed == null) {
  425. // Feed does not exist in DB,we should to try to add it.
  426. if ((!FreshRSS_Context::$isCli) && ($nb_feeds >= $limits['max_feeds'])) {
  427. // Oops, no more place!
  428. Minz_Log::warning(_t('feedback.sub.feed.over_max', $limits['max_feeds']));
  429. } else {
  430. $feed = $this->addFeedJson($item['origin']);
  431. }
  432. if ($feed == null) {
  433. // Still null? It means something went wrong.
  434. $error = true;
  435. } else {
  436. $nb_feeds++;
  437. }
  438. }
  439. if ($feed != null) {
  440. $article_to_feed[$item['id']] = $feed->id();
  441. if (!isset($newFeedGuids['f_' . $feed->id()])) {
  442. $newFeedGuids['f_' . $feed->id()] = array();
  443. }
  444. $newFeedGuids['f_' . $feed->id()][] = safe_ascii($item['id']);
  445. }
  446. }
  447. $tagDAO = FreshRSS_Factory::createTagDao();
  448. $labels = $tagDAO->listTags();
  449. $knownLabels = array();
  450. foreach ($labels as $label) {
  451. $knownLabels[$label->name()]['id'] = $label->id();
  452. $knownLabels[$label->name()]['articles'] = array();
  453. }
  454. unset($labels);
  455. // For each feed, check existing GUIDs already in database.
  456. $existingHashForGuids = array();
  457. foreach ($newFeedGuids as $feedId => $newGuids) {
  458. $existingHashForGuids[$feedId] = $this->entryDAO->listHashForFeedGuids(substr($feedId, 2), $newGuids);
  459. }
  460. unset($newFeedGuids);
  461. // Then, articles are imported.
  462. $newGuids = array();
  463. $this->entryDAO->beginTransaction();
  464. foreach ($items as $item) {
  465. if (empty($article_to_feed[$item['id']])) {
  466. // Related feed does not exist for this entry, do nothing.
  467. continue;
  468. }
  469. $feed_id = $article_to_feed[$item['id']];
  470. $author = isset($item['author']) ? $item['author'] : '';
  471. $is_starred = false;
  472. $is_read = null;
  473. $tags = empty($item['categories']) ? array() : $item['categories'];
  474. $labels = array();
  475. for ($i = count($tags) - 1; $i >= 0; $i --) {
  476. $tag = trim($tags[$i]);
  477. if (strpos($tag, 'user/-/') !== false) {
  478. if ($tag === 'user/-/state/com.google/starred') {
  479. $is_starred = true;
  480. } elseif ($tag === 'user/-/state/com.google/read') {
  481. $is_read = true;
  482. } elseif ($tag === 'user/-/state/com.google/unread') {
  483. $is_read = false;
  484. } elseif (strpos($tag, 'user/-/label/') === 0) {
  485. $tag = trim(substr($tag, 13));
  486. if ($tag != '') {
  487. $labels[] = $tag;
  488. }
  489. }
  490. unset($tags[$i]);
  491. }
  492. }
  493. if ($starred && !$is_starred) {
  494. //If the article has no label, mark it as starred (old format)
  495. $is_starred = empty($labels);
  496. }
  497. if ($is_read === null) {
  498. $is_read = $mark_as_read;
  499. }
  500. if (isset($item['alternate'][0]['href'])) {
  501. $url = $item['alternate'][0]['href'];
  502. } elseif (isset($item['url'])) {
  503. $url = $item['url']; //FeedBin
  504. } else {
  505. $url = '';
  506. }
  507. if (!empty($item['content']['content'])) {
  508. $content = $item['content']['content'];
  509. } elseif (!empty($item['summary']['content'])) {
  510. $content = $item['summary']['content'];
  511. } elseif (!empty($item['content'])) {
  512. $content = $item['content']; //FeedBin
  513. } else {
  514. $content = '';
  515. }
  516. $content = sanitizeHTML($content, $url);
  517. if (!empty($item['published'])) {
  518. $published = $item['published'];
  519. } elseif (!empty($item['timestampUsec'])) {
  520. $published = substr($item['timestampUsec'], 0, -6);
  521. } elseif (!empty($item['updated'])) {
  522. $published = $item['updated'];
  523. } else {
  524. $published = 0;
  525. }
  526. if (!ctype_digit('' . $published)) {
  527. $published = strtotime($published);
  528. }
  529. $entry = new FreshRSS_Entry(
  530. $feed_id, $item['id'], $item['title'], $author,
  531. $content, $url, $published, $is_read, $is_starred
  532. );
  533. $entry->_id(uTimeString());
  534. $entry->_tags($tags);
  535. if (isset($newGuids[$entry->guid()])) {
  536. continue; //Skip subsequent articles with same GUID
  537. }
  538. $newGuids[$entry->guid()] = true;
  539. $entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry);
  540. if ($entry == null) {
  541. // An extension has returned a null value, there is nothing to insert.
  542. continue;
  543. }
  544. $values = $entry->toArray();
  545. $ok = false;
  546. if (isset($existingHashForGuids['f_' . $feed_id][$entry->guid()])) {
  547. $ok = $this->entryDAO->updateEntry($values);
  548. } else {
  549. $ok = $this->entryDAO->addEntry($values);
  550. }
  551. foreach ($labels as $labelName) {
  552. if (empty($knownLabels[$labelName]['id'])) {
  553. $labelId = $tagDAO->addTag(array('name' => $labelName));
  554. $knownLabels[$labelName]['id'] = $labelId;
  555. $knownLabels[$labelName]['articles'] = array();
  556. }
  557. $knownLabels[$labelName]['articles'][] = array(
  558. //'id' => $entry->id(), //ID changes after commitNewEntries()
  559. 'id_feed' => $entry->feed(),
  560. 'guid' => $entry->guid(),
  561. );
  562. }
  563. $error |= ($ok === false);
  564. }
  565. $this->entryDAO->commit();
  566. $this->entryDAO->beginTransaction();
  567. $this->entryDAO->commitNewEntries();
  568. $this->feedDAO->updateCachedValues();
  569. $this->entryDAO->commit();
  570. $this->entryDAO->beginTransaction();
  571. foreach ($knownLabels as $labelName => $knownLabel) {
  572. $labelId = $knownLabel['id'];
  573. foreach ($knownLabel['articles'] as $article) {
  574. $entryId = $this->entryDAO->searchIdByGuid($article['id_feed'], $article['guid']);
  575. if ($entryId != null) {
  576. $tagDAO->tagEntry($labelId, $entryId);
  577. } else {
  578. Minz_Log::warning('Could not add label "' . $labelName . '" to entry "' . $article['guid'] . '" in feed ' . $article['id_feed']);
  579. }
  580. }
  581. }
  582. $this->entryDAO->commit();
  583. return !$error;
  584. }
  585. /**
  586. * This method import a JSON-based feed (Google Reader format).
  587. *
  588. * @param array $origin represents a feed.
  589. * @return FreshRSS_Feed if feed is in database at the end of the process,
  590. * else null.
  591. */
  592. private function addFeedJson($origin) {
  593. $return = null;
  594. if (!empty($origin['feedUrl'])) {
  595. $url = $origin['feedUrl'];
  596. } elseif (!empty($origin['htmlUrl'])) {
  597. $url = $origin['htmlUrl'];
  598. } else {
  599. return null;
  600. }
  601. if (!empty($origin['htmlUrl'])) {
  602. $website = $origin['htmlUrl'];
  603. } elseif (!empty($origin['feedUrl'])) {
  604. $website = $origin['feedUrl'];
  605. }
  606. $name = empty($origin['title']) ? '' : $origin['title'];
  607. try {
  608. // Create a Feed object and add it in database.
  609. $feed = new FreshRSS_Feed($url);
  610. $feed->_category(FreshRSS_CategoryDAO::DEFAULTCATEGORYID);
  611. $feed->_name($name);
  612. $feed->_website($website);
  613. if (!empty($origin['disable'])) {
  614. $feed->_ttl(-1 * FreshRSS_Context::$user_conf->ttl_default);
  615. }
  616. // Call the extension hook
  617. $feed = Minz_ExtensionManager::callHook('feed_before_insert', $feed);
  618. if ($feed != null) {
  619. // addFeedObject checks if feed is already in DB so nothing else to
  620. // check here.
  621. $id = $this->feedDAO->addFeedObject($feed);
  622. if ($id !== false) {
  623. $feed->_id($id);
  624. $return = $feed;
  625. }
  626. }
  627. } catch (FreshRSS_Feed_Exception $e) {
  628. if (FreshRSS_Context::$isCli) {
  629. fwrite(STDERR, 'FreshRSS error during JSON feed import: ' . $e->getMessage() . "\n");
  630. } else {
  631. Minz_Log::warning($e->getMessage());
  632. }
  633. }
  634. return $return;
  635. }
  636. public function exportFile($export_opml = true, $export_starred = false, $export_labelled = false, $export_feeds = array(), $maxFeedEntries = 50, $username = null) {
  637. require_once(LIB_PATH . '/lib_opml.php');
  638. $this->catDAO = new FreshRSS_CategoryDAO($username);
  639. $this->entryDAO = FreshRSS_Factory::createEntryDao($username);
  640. $this->feedDAO = FreshRSS_Factory::createFeedDao($username);
  641. $this->entryDAO->disableBuffering();
  642. if ($export_feeds === true) {
  643. //All feeds
  644. $export_feeds = $this->feedDAO->listFeedsIds();
  645. }
  646. if (!is_array($export_feeds)) {
  647. $export_feeds = array();
  648. }
  649. $day = date('Y-m-d');
  650. $export_files = array();
  651. if ($export_opml) {
  652. $export_files["feeds_${day}.opml.xml"] = $this->generateOpml();
  653. }
  654. if ($export_starred || $export_labelled) {
  655. $export_files["starred_${day}.json"] = $this->generateEntries(
  656. ($export_starred ? 'S' : '') .
  657. ($export_labelled ? 'T' : '')
  658. );
  659. }
  660. foreach ($export_feeds as $feed_id) {
  661. $feed = $this->feedDAO->searchById($feed_id);
  662. if ($feed) {
  663. $filename = "feed_${day}_" . $feed->category() . '_'
  664. . $feed->id() . '.json';
  665. $export_files[$filename] = $this->generateEntries('f', $feed, $maxFeedEntries);
  666. }
  667. }
  668. $nb_files = count($export_files);
  669. if ($nb_files > 1) {
  670. // If there are more than 1 file to export, we need a ZIP archive.
  671. try {
  672. $this->sendZip($export_files);
  673. } catch (Exception $e) {
  674. throw new FreshRSS_ZipMissing_Exception($e);
  675. }
  676. } elseif ($nb_files === 1) {
  677. // Only one file? Guess its type and export it.
  678. $filename = key($export_files);
  679. $type = self::guessFileType($filename);
  680. $this->sendFile('freshrss_' . Minz_Session::param('currentUser', '_') . '_' . $filename, $export_files[$filename], $type);
  681. }
  682. return $nb_files;
  683. }
  684. /**
  685. * This action handles export action.
  686. *
  687. * This action must be reached by a POST request.
  688. *
  689. * Parameters are:
  690. * - export_opml (default: false)
  691. * - export_starred (default: false)
  692. * - export_feeds (default: array()) a list of feed ids
  693. */
  694. public function exportAction() {
  695. if (!Minz_Request::isPost()) {
  696. Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true);
  697. }
  698. $this->view->_useLayout(false);
  699. $nb_files = 0;
  700. try {
  701. $nb_files = $this->exportFile(
  702. Minz_Request::param('export_opml', false),
  703. Minz_Request::param('export_starred', false),
  704. Minz_Request::param('export_labelled', false),
  705. Minz_Request::param('export_feeds', array())
  706. );
  707. } catch (FreshRSS_ZipMissing_Exception $zme) {
  708. # Oops, there is no ZIP extension!
  709. Minz_Request::bad(_t('feedback.import_export.export_no_zip_extension'),
  710. array('c' => 'importExport', 'a' => 'index'));
  711. }
  712. if ($nb_files < 1) {
  713. // Nothing to do...
  714. Minz_Request::forward(array('c' => 'importExport', 'a' => 'index'), true);
  715. }
  716. }
  717. /**
  718. * This method returns the OPML file based on user subscriptions.
  719. *
  720. * @return string the OPML file content.
  721. */
  722. private function generateOpml() {
  723. $list = array();
  724. foreach ($this->catDAO->listCategories() as $key => $cat) {
  725. $list[$key]['name'] = $cat->name();
  726. $list[$key]['feeds'] = $this->feedDAO->listByCategory($cat->id());
  727. }
  728. $this->view->categories = $list;
  729. return $this->view->helperToString('export/opml');
  730. }
  731. /**
  732. * This method returns a JSON file content.
  733. *
  734. * @param string $type must be one of:
  735. * 'S' (starred/favourite), 'f' (feed), 'T' (taggued/labelled), 'ST' (starred or labelled)
  736. * @param FreshRSS_Feed $feed feed of which we want to get entries.
  737. * @return string the JSON file content.
  738. */
  739. private function generateEntries($type, $feed = null, $maxFeedEntries = 50) {
  740. $this->view->categories = $this->catDAO->listCategories();
  741. $tagDAO = FreshRSS_Factory::createTagDao();
  742. if ($type === 's' || $type === 'S' || $type === 'T' || $type === 'ST') {
  743. $this->view->list_title = _t('sub.import_export.starred_list');
  744. $this->view->type = 'starred';
  745. $this->view->entriesId = $this->entryDAO->listIdsWhere($type, '', FreshRSS_Entry::STATE_ALL, 'ASC', -1);
  746. $this->view->entryIdsTagNames = $tagDAO->getEntryIdsTagNames($this->view->entriesId);
  747. //The following is a streamable query, i.e. must be last
  748. $this->view->entriesRaw = $this->entryDAO->listWhereRaw($type, '', FreshRSS_Entry::STATE_ALL, 'ASC', -1);
  749. } elseif ($type === 'f' && $feed != null) {
  750. $this->view->list_title = _t('sub.import_export.feed_list', $feed->name());
  751. $this->view->type = 'feed/' . $feed->id();
  752. $this->view->entriesId = $this->entryDAO->listIdsWhere($type, $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC', $maxFeedEntries);
  753. $this->view->entryIdsTagNames = $tagDAO->getEntryIdsTagNames($this->view->entriesId);
  754. //The following is a streamable query, i.e. must be last
  755. $this->view->entriesRaw = $this->entryDAO->listWhereRaw($type, $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC', $maxFeedEntries);
  756. $this->view->feed = $feed;
  757. }
  758. return $this->view->helperToString('export/articles');
  759. }
  760. /**
  761. * This method zips a list of files and returns it by HTTP.
  762. *
  763. * @param array $files list of files where key is filename and value the content.
  764. * @throws Exception if Zip extension is not loaded.
  765. */
  766. private function sendZip($files) {
  767. if (!extension_loaded('zip')) {
  768. throw new Exception();
  769. }
  770. // From https://stackoverflow.com/questions/1061710/php-zip-files-on-the-fly
  771. $zip_file = @tempnam('/tmp', 'zip');
  772. $zip = new ZipArchive();
  773. $zip->open($zip_file, ZipArchive::OVERWRITE);
  774. foreach ($files as $filename => $content) {
  775. $zip->addFromString($filename, $content);
  776. }
  777. // Close and send to user
  778. $zip->close();
  779. header('Content-Type: application/zip');
  780. header('Content-Length: ' . filesize($zip_file));
  781. $day = date('Y-m-d');
  782. header('Content-Disposition: attachment; filename="freshrss_' . Minz_Session::param('currentUser', '_') . '_' . $day . '_export.zip"');
  783. readfile($zip_file);
  784. unlink($zip_file);
  785. }
  786. /**
  787. * This method returns a single file (OPML or JSON) by HTTP.
  788. *
  789. * @param string $filename
  790. * @param string $content
  791. * @param string $type the file type (opml, json_feed or json_starred).
  792. * If equals to unknown, nothing happens.
  793. */
  794. private function sendFile($filename, $content, $type) {
  795. if ($type === 'unknown') {
  796. return;
  797. }
  798. $content_type = '';
  799. if ($type === 'opml') {
  800. $content_type = 'application/xml';
  801. } elseif ($type === 'json_feed' || $type === 'json_starred') {
  802. $content_type = 'application/json';
  803. }
  804. header('Content-Type: ' . $content_type . '; charset=utf-8');
  805. header('Content-disposition: attachment; filename=' . $filename);
  806. print($content);
  807. }
  808. }