Better performance with yield (#2588)

* Better performance with yield

Largely decrease the time to first byte, and reduced memory consumtion.
Before, we used to make several copies in memory of the whole list of
articles before sending them to the client. Now streamed as they are
processed.

* Travis

app/Controllers/indexController.php

@@ -32,6 +32,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			Minz_Error::error(404);
 		}
 
+		$this->_csp([
+			'default-src' => "'self'",
+			'frame-src' => '*',
+			'img-src' => '* data:',
+			'media-src' => '*',
+		]);
+
 		$this->view->categories = FreshRSS_Context::$categories;
 
 		$this->view->rss_title = FreshRSS_Context::$name . ' | ' . Minz_View::title();
@@ -54,42 +61,35 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			}
 		};
 
-		$this->view->callbackBeforePagination = function ($view) {
+		$this->view->callbackBeforeEntries = function ($view) {
 			try {
 				FreshRSS_Context::$number++;	//+1 for pagination
-				$entries = FreshRSS_index_Controller::listEntriesByContext();
+				$view->entries = FreshRSS_index_Controller::listEntriesByContext();
 				FreshRSS_Context::$number--;
-
-				$nb_entries = count($entries);
-				if ($nb_entries > FreshRSS_Context::$number) {
-					// We have more elements for pagination
-					$last_entry = array_pop($entries);
-					FreshRSS_Context::$next_id = $last_entry->id();
-				}
-
-				$first_entry = $nb_entries > 0 ? $entries[0] : null;
-				FreshRSS_Context::$id_max = $first_entry === null ? (time() - 1) . '000000' : $first_entry->id();
-				if (FreshRSS_Context::$order === 'ASC') {
-					// In this case we do not know but we guess id_max
-					$id_max = (time() - 1) . '000000';
-					if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
-						FreshRSS_Context::$id_max = $id_max;
-					}
-				}
-
-				$view->entries = $entries;
+				ob_start();
 			} catch (FreshRSS_EntriesGetter_Exception $e) {
 				Minz_Log::notice($e->getMessage());
 				Minz_Error::error(404);
 			}
 		};
 
-		$this->_csp([
-			'default-src' => "'self'",
-			'frame-src' => '*',
-			'img-src' => '* data:',
-			'media-src' => '*',
-		]);
+		$this->view->callbackBeforePagination = function ($view, $nbEntries, $firstEntry, $lastEntry) {
+			if ($nbEntries >= FreshRSS_Context::$number) {
+				//We have enough entries: we discard the last one to use it for the next pagination
+				ob_clean();
+				FreshRSS_Context::$next_id = $lastEntry->id();
+			}
+			ob_end_flush();
+
+			FreshRSS_Context::$id_max = $firstEntry === null ? (time() - 1) . '000000' : $firstEntry->id();
+			if (FreshRSS_Context::$order === 'ASC') {
+				// In this case we do not know but we guess id_max
+				$id_max = (time() - 1) . '000000';
+				if (strcmp($id_max, FreshRSS_Context::$id_max) > 0) {
+					FreshRSS_Context::$id_max = $id_max;
+				}
+			}
+		};
 	}
 
 	/**
@@ -247,23 +247,13 @@ class FreshRSS_index_Controller extends Minz_ActionController {
 			$limit = FreshRSS_Context::$user_conf->max_posts_per_rss;
 		}
 
-		$entries = $entryDAO->listWhere(
-			$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-			$limit, FreshRSS_Context::$first_id,
-			FreshRSS_Context::$search, $date_min
-		);
-
-		if (FreshRSS_Context::$sinceHours && (count($entries) < FreshRSS_Context::$user_conf->min_posts_per_rss)) {
-			$date_min = 0;
-			$limit = FreshRSS_Context::$user_conf->min_posts_per_rss;
-			$entries = $entryDAO->listWhere(
-				$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
-				$limit, FreshRSS_Context::$first_id,
-				FreshRSS_Context::$search, $date_min
-			);
+		foreach ($entryDAO->listWhere(
+					$type, $id, FreshRSS_Context::$state, FreshRSS_Context::$order,
+					$limit, FreshRSS_Context::$first_id,
+					FreshRSS_Context::$search, $date_min)
+				as $entry) {
+			yield $entry;
 		}
-
-		return $entries;
 	}
 
 	/**

app/Models/EntryDAO.php

@@ -634,8 +634,7 @@ SQL;
 		$stm->bindParam(':guid', $guid);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchById($id) {
@@ -647,8 +646,7 @@ SQL;
 		$stm->bindParam(':id', $id, PDO::PARAM_INT);
 		$stm->execute();
 		$res = $stm->fetchAll(PDO::FETCH_ASSOC);
-		$entries = self::daoToEntries($res);
-		return isset($entries[0]) ? $entries[0] : null;
+		return isset($res[0]) ? self::daoToEntry($res[0]) : null;
 	}
 
 	public function searchIdByGuid($id_feed, $guid) {
@@ -885,15 +883,17 @@ SQL;
 	public function listWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null, $date_min = 0) {
 		$stm = $this->listWhereRaw($type, $id, $state, $order, $limit, $firstId, $filters, $date_min);
 		if ($stm) {
-			return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+			while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+				yield self::daoToEntry($row);
+			}
 		} else {
-			return false;
+			yield false;
 		}
 	}
 
 	public function listByIds($ids, $order = 'DESC') {
 		if (count($ids) < 1) {
-			return array();
+			yield false;
 		}
 
 		$sql = 'SELECT id, guid, title, author, '
@@ -905,7 +905,9 @@ SQL;
 
 		$stm = $this->pdo->prepare($sql);
 		$stm->execute($ids);
-		return self::daoToEntries($stm->fetchAll(PDO::FETCH_ASSOC));
+		while ($row = $stm->fetch(PDO::FETCH_ASSOC)) {
+			yield self::daoToEntry($row);
+		}
 	}
 
 	public function listIdsWhere($type = 'a', $id = '', $state = FreshRSS_Entry::STATE_ALL, $order = 'DESC', $limit = 1, $firstId = '', $filters = null) {	//For API
@@ -1058,20 +1060,4 @@ SQL;
 		}
 		return $entry;
 	}
-
-	private static function daoToEntries($listDAO) {
-		$list = array();
-
-		if (!is_array($listDAO)) {
-			$listDAO = array($listDAO);
-		}
-
-		foreach ($listDAO as $key => $dao) {
-			$list[] = self::daoToEntry($dao);
-		}
-
-		unset($listDAO);
-
-		return $list;
-	}
 }

app/layout/nav_menu.phtml

@@ -1,10 +1,6 @@
 <?php
 	$actual_view = Minz_Request::actionName();
-
 	flush();
-	if (isset($this->callbackBeforePagination)) {
-		call_user_func($this->callbackBeforePagination, $this);
-	}
 ?>
 
 <div class="nav_menu">

app/views/index/normal.phtml

@@ -3,22 +3,32 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$display_today = true;
-	$display_yesterday = true;
-	$display_others = true;
-	$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
 
-	$today = @strtotime('today');
+$display_today = true;
+$display_yesterday = true;
+$display_others = true;
+$hidePosts = !FreshRSS_Context::$user_conf->display_posts;
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
+
+$today = @strtotime('today');
 ?>
 
-<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>"><?php
-	?><div id="new-article">
+<div id="stream" class="normal<?= $hidePosts ? ' hide_posts' : '' ?>">
+	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
 	</div><?php
-	foreach ($this->entries as $item) {
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$this->entry = Minz_ExtensionManager::callHook('entry_before_display', $item);
 		if ($this->entry == null) {
 			continue;
@@ -87,16 +97,19 @@ if (!empty($this->entries)) {
 
 		?></div>
 	</div><?php
-	}
-
-	$this->renderHelper('pagination');
-?></div>
+	endforeach;
 
-<?php if (FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn normal">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>
+
+<?php if ($nbEntries > 0 && FreshRSS_Context::$user_conf->show_nav_buttons) $this->partial('nav_entries'); ?>

app/views/index/reader.phtml

@@ -2,18 +2,28 @@
 $this->partial('aside_feed');
 $this->partial('nav_menu');
 
-if (!empty($this->entries)) {
-	$lazyload = FreshRSS_Context::$user_conf->lazyload;
-	$content_width = FreshRSS_Context::$user_conf->content_width;
+call_user_func($this->callbackBeforeEntries, $this);
+
+$lazyload = FreshRSS_Context::$user_conf->lazyload;
+$content_width = FreshRSS_Context::$user_conf->content_width;
 ?>
 
 <div id="stream" class="reader">
 	<div id="new-article">
 		<a href="<?= Minz_Url::display(Minz_Request::currentRequest()) ?>"><?= _t('gen.js.new_article'); /* TODO: move string in JS*/ ?></a>
-	</div>
-	<?php foreach ($this->entries as $item) {
+	</div><?php
+	$firstEntry = null;
+	$lastEntry = null;
+	$nbEntries = 0;
+	foreach ($this->entries as $item):
+		if ($nbEntries === 0) {
+			$firstEntry = $item;
+		}
+		$lastEntry = $item;
+		$nbEntries++;
+		ob_flush();
 		$item = Minz_ExtensionManager::callHook('entry_before_display', $item);
-		if (is_null($item)) {
+		if ($item == null) {
 			continue;
 		}
 	?><div class="flux<?= !$item->isRead() ? ' not_read' : '' ?><?= $item->isFavorite() ? ' favorite' : '' ?>" id="flux_<?= $item->id() ?>">
@@ -61,15 +71,18 @@ if (!empty($this->entries)) {
 				<?= $item->content() ?>
 			</div>
 		</div>
-	</div>
-	<?php } ?>
+	</div><?php
+	endforeach;
 
-	<?php $this->renderHelper('pagination'); ?>
-</div>
-
-<?php } else { ?>
+	if ($nbEntries > 0):
+		call_user_func($this->callbackBeforePagination, $this, $nbEntries, $firstEntry, $lastEntry);
+		$this->renderHelper('pagination');
+?></div><?php
+	else:
+		ob_end_clean();	//Discard the articles headers, as we have no articles
+?>
 <div id="stream" class="prompt alert alert-warn reader">
 	<h2><?= _t('index.feed.empty') ?></h2>
 	<a href="<?= _url('subscription', 'index') ?>"><?= _t('index.feed.add') ?></a><br /><br />
 </div>
-<?php } ?>
+<?php endif; ?>

cli/README.md

@@ -47,7 +47,7 @@ cd /usr/share/FreshRSS
 ./cli/reconfigure.php
 # Same parameters as for do-install.php. Used to update an existing installation.
 
-./cli/create-user.php --user username [ --password 'password' --api_password 'api_password' --language en --email user@example.net --token 'longRandomString' --no_default_feeds --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600 --since_hours_posts_per_rss 168 --min_posts_per_rss 2 --max_posts_per_rss 400 ]
+./cli/create-user.php --user username [ --password 'password' --api_password 'api_password' --language en --email user@example.net --token 'longRandomString' --no_default_feeds --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600 --since_hours_posts_per_rss 168 --max_posts_per_rss 400 ]
 # --language can be: 'en' (default), 'fr', or one of the [supported languages](../app/i18n/)
 
 ./cli/update-user.php --user username [ ... ]

cli/_update-or-create-user.php

@@ -11,7 +11,6 @@ $params = array(
 		'feed_min_articles_default:',
 		'feed_ttl_default:',
 		'since_hours_posts_per_rss:',
-		'min_posts_per_rss:',
 		'max_posts_per_rss:',
 	);
 
@@ -27,7 +26,7 @@ if (!validateOptions($argv, $params) || empty($options['user'])) {
 		" --language en --email user@example.net --token 'longRandomString'" .
 		($isUpdate ? '' : '--no_default_feeds') .
 		" --purge_after_months 3 --feed_min_articles_default 50 --feed_ttl_default 3600" .
-		" --since_hours_posts_per_rss 168 --min_posts_per_rss 2 --max_posts_per_rss 400 )");
+		" --since_hours_posts_per_rss 168 --max_posts_per_rss 400 )");
 }
 
 function strParam($name) {
@@ -48,7 +47,6 @@ $values = array(
 		'keep_history_default' => intParam('feed_min_articles_default'),	//TODO: Update with new mechanism
 		'ttl_default' => intParam('feed_ttl_default'),
 		'since_hours_posts_per_rss' => intParam('since_hours_posts_per_rss'),
-		'min_posts_per_rss' => intParam('min_posts_per_rss'),
 		'max_posts_per_rss' => intParam('max_posts_per_rss'),
 	);
 

config-user.default.php

@@ -24,7 +24,6 @@ return array (
 	'feverKey' => '',
 	'posts_per_page' => 20,
 	'since_hours_posts_per_rss' => 168,
-	'min_posts_per_rss' => 2,
 	'max_posts_per_rss' => 400,
 	'view_mode' => 'normal',
 	'default_view' => 'adaptive',

p/api/greader.php

@@ -638,6 +638,7 @@ function streamContents($path, $include_target, $start_time, $stop_time, $count,
 
 	$entryDAO = FreshRSS_Factory::createEntryDao();
 	$entries = $entryDAO->listWhere($type, $include_target, $state, $order === 'o' ? 'ASC' : 'DESC', $count, $continuation, $searches);
+	$entries = iterator_to_array($entries);	//TODO: Improve
 
 	$items = entriesToArray($entries);
 
@@ -730,6 +731,7 @@ function streamContentsItems($e_ids, $order) {
 
 	$entryDAO = FreshRSS_Factory::createEntryDao();
 	$entries = $entryDAO->listByIds($e_ids, $order === 'o' ? 'ASC' : 'DESC');
+	$entries = iterator_to_array($entries);	//TODO: Improve
 
 	$items = entriesToArray($entries);