Merge pull request #1373 from Alkarex/StreamingExport

Stream JSON export

CHANGELOG.md

@@ -12,6 +12,8 @@
 	* PostgreSQL: fix bug when updating cached values [#1360](https://github.com/FreshRSS/FreshRSS/issues/1360)
 	* Fix bug in confirmation before marking as read [#1348](https://github.com/FreshRSS/FreshRSS/issues/1348)
 	* Fix small bugs in installer [#1363](https://github.com/FreshRSS/FreshRSS/pull/1363)
+* Misc.
+	* More robust export function in the case of large datasets [#1372](https://github.com/FreshRSS/FreshRSS/issues/1372)
 
 
 ## 2016-11-02 FreshRSS 1.6.1
@@ -67,7 +69,7 @@
 	* Download icon 💾 for podcasts [#1236](https://github.com/FreshRSS/FreshRSS/issues/1236)
 * SimplePie
 	* Fix auto-discovery of RSS feeds in Web pages served as `text/xml` [#1264](https://github.com/FreshRSS/FreshRSS/issues/1264)
-* Mics.
+* Misc.
 	* Removed *resource-priorities* attributes (`defer`, `lazyload`), deprecated by W3C [#1222](https://github.com/FreshRSS/FreshRSS/pull/1222)
 
 

app/Controllers/importExportController.php

@@ -531,6 +531,8 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 		$this->entryDAO = FreshRSS_Factory::createEntryDao($username);
 		$this->feedDAO = FreshRSS_Factory::createFeedDao($username);
 
+		$this->entryDAO->disableBuffering();
+
 		if ($export_feeds === true) {
 			//All feeds
 			$export_feeds = $this->feedDAO->listFeedsIds();
@@ -641,13 +643,13 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 			$this->view->list_title = _t('sub.import_export.starred_list');
 			$this->view->type = 'starred';
 			$unread_fav = $this->entryDAO->countUnreadReadFavorites();
-			$this->view->entries = $this->entryDAO->listWhere(
+			$this->view->entriesRaw = $this->entryDAO->listWhereRaw(
 				's', '', FreshRSS_Entry::STATE_ALL, 'ASC', $unread_fav['all']
 			);
 		} elseif ($type === 'feed' && $feed != null) {
 			$this->view->list_title = _t('sub.import_export.feed_list', $feed->name());
 			$this->view->type = 'feed/' . $feed->id();
-			$this->view->entries = $this->entryDAO->listWhere(
+			$this->view->entriesRaw = $this->entryDAO->listWhereRaw(
 				'f', $feed->id(), FreshRSS_Entry::STATE_ALL, 'ASC',
 				$maxFeedEntries
 			);

app/Models/EntryDAO.php

@@ -518,7 +518,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm->execute($values);
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntry($res);
+		$entries = self::daoToEntries($res);
 		return isset($entries[0]) ? $entries[0] : null;
 	}
 
@@ -533,7 +533,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm->execute($values);
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntry($res);
+		$entries = self::daoToEntries($res);
 		return isset($entries[0]) ? $entries[0] : null;
 	}
 
@@ -666,7 +666,7 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 			. ($limit > 0 ? ' LIMIT ' . $limit : ''));	//TODO: See http://explainextended.com/2009/10/23/mysql-order-by-limit-performance-late-row-lookups/
 	}
 
-	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
+	public function listWhereRaw($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
 		list($values, $sql) = $this->sqlListWhere($type, $id, $state, $order, $limit, $firstId, $filter, $date_min);
 
 		$sql = 'SELECT e0.id, e0.guid, e0.title, e0.author, '
@@ -680,8 +680,12 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 
 		$stm = $this->bd->prepare($sql);
 		$stm->execute($values);
+		return $stm;
+	}
 
-		return self::daoToEntry($stm->fetchAll(PDO::FETCH_ASSOC));
+	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {
+		$stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filter, $date_min);
+		return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
 	}
 
 	public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filter = '', $date_min = 0) {	//For API
@@ -810,15 +814,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $res[0];
 	}
 
-	public static function daoToEntry($listDAO) {
-		$list = array();
-
-		if (!is_array($listDAO)) {
-			$listDAO = array($listDAO);
-		}
-
-		foreach ($listDAO as $key => $dao) {
-			$entry = new FreshRSS_Entry(
+	public static function daoToEntry($dao) {
+		$entry = new FreshRSS_Entry(
 				$dao['id_feed'],
 				$dao['guid'],
 				$dao['title'],
@@ -830,10 +827,21 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 				$dao['is_favorite'],
 				$dao['tags']
 			);
-			if (isset($dao['id'])) {
-				$entry->_id($dao['id']);
-			}
-			$list[] = $entry;
+		if (isset($dao['id'])) {
+			$entry->_id($dao['id']);
+		}
+		return $entry;
+	}
+
+	private static function daoToEntries($listDAO) {
+		$list = array();
+
+		if (!is_array($listDAO)) {
+			$listDAO = array($listDAO);
+		}
+
+		foreach ($listDAO as $key => $dao) {
+			$list[] = self::daoToEntry($dao);
 		}
 
 		unset($listDAO);

app/views/helpers/export/articles.phtml

@@ -1,47 +1,66 @@
 <?php
-    $username = Minz_Session::param('currentUser', '_');
+$username = Minz_Session::param('currentUser', '_');
 
-    $articles = array(
-        'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type,
-        'title' => $this->list_title,
-        'author' => $username,
-        'items' => array()
-    );
+$options = 0;
+if (version_compare(PHP_VERSION, '5.4.0') >= 0) {
+	$options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
+}
 
-    foreach ($this->entries as $entry) {
-        if (!isset($this->feed)) {
-            $feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed ());
-        } else {
-            $feed = $this->feed;
-        }
+$articles = array(
+	'id' => 'user/' . str_replace('/', '', $username) . '/state/org.freshrss/' . $this->type,
+	'title' => $this->list_title,
+	'author' => $username,
+	'items' => array(),
+);
 
-        $articles['items'][] = array(
-            'id' => $entry->guid(),
-            'categories' => array_values($entry->tags()),
-            'title' => $entry->title(),
-            'author' => $entry->author(),
-            'published' => $entry->date(true),
-            'updated' => $entry->date(true),
-            'alternate' => array(array(
-                'href' => $entry->link(),
-                'type' => 'text/html'
-            )),
-            'content' => array(
-                'content' => $entry->content()
-            ),
-            'origin' => array(
-                'streamId' => $feed->id(),
-                'title' => $feed->name(),
-                'htmlUrl' => $feed->website(),
-                'feedUrl' => $feed->url()
-            )
-        );
-    }
+echo rtrim(json_encode($articles, $options), " ]}\n\r\t"), "\n";
+$first = true;
 
-    $options = 0;
-    if (version_compare(PHP_VERSION, '5.4.0') >= 0) {
-        $options = JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
-    }
+foreach ($this->entriesRaw as $entryRaw) {
+	if (empty($entryRaw)) {
+		continue;
+	}
+	$entry = FreshRSS_EntryDAO::daoToEntry($entryRaw);
+	if (!isset($this->feed)) {
+		$feed = FreshRSS_CategoryDAO::findFeed($this->categories, $entry->feed());
+		if ($feed == null) {
+			$feed = $entry->feed(true);
+		}
+	} else {
+		$feed = $this->feed;
+	}
 
-    echo json_encode($articles, $options);
-?>
+	$article = array(
+		'id' => $entry->guid(),
+		'categories' => array_values($entry->tags()),
+		'title' => $entry->title(),
+		'author' => $entry->author(),
+		'published' => $entry->date(true),
+		'updated' => $entry->date(true),
+		'alternate' => array(array(
+			'href' => $entry->link(),
+			'type' => 'text/html',
+		)),
+		'content' => array(
+			'content' => $entry->content(),
+		),
+		'origin' => array(
+			'streamId' => $feed == null ? '' : $feed->id(),
+			'title' => $feed == null ? '' : $feed->name(),
+			'htmlUrl' => $feed == null ? '' : $feed->website(),
+			'feedUrl' => $feed == null ? '' : $feed->url(),
+		)
+	);
+
+	$line = json_encode($article, $options);
+	if ($line != '') {
+		if ($first) {
+			$first = false;
+		} else {
+			echo ",\n";
+		}
+		echo $line;
+	}
+}
+
+echo "\n]}\n";

lib/Minz/ModelPdo.php

@@ -116,6 +116,12 @@ class Minz_ModelPdo {
 		self::$sharedBd = null;
 		self::$sharedPrefix = '';
 	}
+
+	public function disableBuffering() {
+		if ((self::$sharedDbType === 'mysql') && defined('PDO::MYSQL_ATTR_USE_BUFFERED_QUERY')) {
+			$this->bd->setAttribute(PDO::MYSQL_ATTR_USE_BUFFERED_QUERY, false);
+		}
+	}
 }
 
 class MinzPDO extends PDO {