Merge pull request #1470 from Alkarex/defered-insertion

Implementation of defered insertion

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## 2017-xx-xx FreshRSS 1.7.0-dev
+* Features:
+	* Deferred insertion of new articles, for better chronological order [#530](https://github.com/FreshRSS/FreshRSS/issues/530)
 * Compatibility:
 	* Add support for PHP 7.1 [#1471](https://github.com/FreshRSS/FreshRSS/issues/1471)
 	* PostgreSQL is not experimental anymore [#1476](https://github.com/FreshRSS/FreshRSS/pull/1476)

app/Controllers/feedController.php

@@ -226,7 +226,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		}
 	}
 
-	public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false) {
+	public static function actualizeFeed($feed_id, $feed_url, $force, $simplePiePush = null, $isNewFeed = false, $noCommit = false) {
 		@set_time_limit(300);
 
 		$feedDAO = FreshRSS_Factory::createFeedDao();
@@ -308,6 +308,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				// -2 means we take the default value from configuration
 				$feed_history = FreshRSS_Context::$user_conf->keep_history_default;
 			}
+			$needFeedCacheRefresh = false;
 
 			// We want chronological order and SimplePie uses reverse order.
 			$entries = array_reverse($feed->entries());
@@ -333,6 +334,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 							//Minz_Log::debug('Entry with GUID `' . $entry->guid() . '` updated in feed ' . $feed->id() .
 								//', old hash ' . $existingHash . ', new hash ' . $entry->hash());
 							//TODO: Make an updated/is_read policy by feed, in addition to the global one.
+							$needFeedCacheRefresh = FreshRSS_Context::$user_conf->mark_updated_article_unread;
 							$entry->_isRead(FreshRSS_Context::$user_conf->mark_updated_article_unread ? false : null);	//Change is_read according to policy.
 							if (!$entryDAO->inTransaction()) {
 								$entryDAO->beginTransaction();
@@ -388,12 +390,16 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				                                $date_min,
 				                                max($feed_history, count($entries) + 10));
 				if ($nb > 0) {
+					$needFeedCacheRefresh = true;
 					Minz_Log::debug($nb . ' old entries cleaned in feed [' .
 					                $feed->url() . ']');
 				}
 			}
 
-			$feedDAO->updateLastUpdate($feed->id(), false, $entryDAO->inTransaction(), $mtime);
+			$feedDAO->updateLastUpdate($feed->id(), false, $mtime);
+			if ($needFeedCacheRefresh) {
+				$feedDAO->updateCachedValue($feed->id());
+			}
 			if ($entryDAO->inTransaction()) {
 				$entryDAO->commit();
 			}
@@ -434,6 +440,16 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				break;
 			}
 		}
+		if (!$noCommit) {
+			if (!$entryDAO->inTransaction()) {
+				$entryDAO->beginTransaction();
+			}
+			$entryDAO->commitNewEntries();
+			$feedDAO->updateCachedValues();
+			if ($entryDAO->inTransaction()) {
+				$entryDAO->commit();
+			}
+		}
 		return array($updated_feeds, reset($feeds));
 	}
 
@@ -444,6 +460,7 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 	 *   - id (default: false): Feed ID
 	 *   - url (default: false): Feed URL
 	 *   - force (default: false)
+	 *   - noCommit (default: 0): Set to 1 to prevent committing the new articles to the main database
 	 * If id and url are not specified, all the feeds are actualized. But if force is
 	 * false, process stops at 10 feeds to avoid time execution problem.
 	 */
@@ -452,8 +469,19 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 		$id = Minz_Request::param('id');
 		$url = Minz_Request::param('url');
 		$force = Minz_Request::param('force');
-
-		list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force);
+		$noCommit = Minz_Request::fetchPOST('noCommit', 0) == 1;
+
+		if ($id == -1 && !$noCommit) {	//Special request only to commit & refresh DB cache
+			$updated_feeds = 0;
+			$entryDAO = FreshRSS_Factory::createEntryDao();
+			$feedDAO = FreshRSS_Factory::createFeedDao();
+			$entryDAO->beginTransaction();
+			$entryDAO->commitNewEntries();
+			$feedDAO->updateCachedValues();
+			$entryDAO->commit();
+		} else {
+			list($updated_feeds, $feed) = self::actualizeFeed($id, $url, $force, null, false, $noCommit);
+		}
 
 		if (Minz_Request::param('ajax')) {
 			// Most of the time, ajax request is for only one feed. But since

app/Controllers/importExportController.php

@@ -475,6 +475,11 @@ class FreshRSS_importExport_Controller extends Minz_ActionController {
 		}
 		$this->entryDAO->commit();
 
+		$this->entryDAO->beginTransaction();
+		$this->entryDAO->commitNewEntries();
+		$this->feedDAO->updateCachedValues();
+		$this->entryDAO->commit();
+
 		return !$error;
 	}
 

app/Models/EntryDAO.php

@@ -88,6 +88,38 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return false;
 	}
 
+	protected function createEntryTempTable() {
+		$ok = false;
+		$hadTransaction = $this->bd->inTransaction();
+		if ($hadTransaction) {
+			$this->bd->commit();
+		}
+		try {
+			$db = FreshRSS_Context::$system_conf->db;
+			require_once(APP_PATH . '/SQL/install.sql.' . $db['type'] . '.php');
+			Minz_Log::warning('SQL CREATE TABLE entrytmp...');
+			if (defined('SQL_CREATE_TABLE_ENTRYTMP')) {
+				$sql = sprintf(SQL_CREATE_TABLE_ENTRYTMP, $this->prefix);
+				$stm = $this->bd->prepare($sql);
+				$ok = $stm && $stm->execute();
+			} else {
+				global $SQL_CREATE_TABLE_ENTRYTMP;
+				$ok = !empty($SQL_CREATE_TABLE_ENTRYTMP);
+				foreach ($SQL_CREATE_TABLE_ENTRYTMP as $instruction) {
+					$sql = sprintf($instruction, $this->prefix);
+					$stm = $this->bd->prepare($sql);
+					$ok &= $stm && $stm->execute();
+				}
+			}
+		} catch (Exception $e) {
+			Minz_Log::error('FreshRSS_EntryDAO::createEntryTempTable error: ' . $e->getMessage());
+		}
+		if ($hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		return $ok;
+	}
+
 	protected function autoUpdateDb($errorInfo) {
 		if (isset($errorInfo[0])) {
 			if ($errorInfo[0] === '42S22') {	//ER_BAD_FIELD_ERROR
@@ -97,6 +129,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 						return $this->addColumn($column);
 					}
 				}
+			} elseif ($errorInfo[0] === '42S02' && stripos($errorInfo[2], 'entrytmp') !== false) {	//ER_BAD_TABLE_ERROR
+				return $this->createEntryTempTable();	//v1.7
 			}
 		}
 		if (isset($errorInfo[1])) {
@@ -110,8 +144,8 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 	private $addEntryPrepared = null;
 
 	public function addEntry($valuesTmp) {
-		if ($this->addEntryPrepared === null) {
-			$sql = 'INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, '
+		if ($this->addEntryPrepared == null) {
+			$sql = 'INSERT INTO `' . $this->prefix . 'entrytmp` (id, guid, title, author, '
 			     . ($this->isCompressed() ? 'content_bin' : 'content')
 			     . ', link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) '
 			     . 'VALUES(:id, :guid, :title, :author, '
@@ -121,41 +155,43 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 			     . ', :is_read, :is_favorite, :id_feed, :tags)';
 			$this->addEntryPrepared = $this->bd->prepare($sql);
 		}
-		$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
-		$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
-		$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
-		$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
-		$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
-		$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
-		$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
-		$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
-		$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
-		$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
-		$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
-		$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
-		$valuesTmp['lastSeen'] = time();
-		$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
-		$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
-		$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
-		$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
-		$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
-		$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
-		$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
-
-		if ($this->hasNativeHex()) {
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
-		} else {
-			$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
-			$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+		if ($this->addEntryPrepared) {
+			$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
+			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
+			$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
+			$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
+			$valuesTmp['title'] = substr($valuesTmp['title'], 0, 255);
+			$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
+			$valuesTmp['author'] = substr($valuesTmp['author'], 0, 255);
+			$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
+			$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
+			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
+			$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
+			$this->addEntryPrepared->bindParam(':date', $valuesTmp['date'], PDO::PARAM_INT);
+			$valuesTmp['lastSeen'] = time();
+			$this->addEntryPrepared->bindParam(':last_seen', $valuesTmp['lastSeen'], PDO::PARAM_INT);
+			$valuesTmp['is_read'] = $valuesTmp['is_read'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_read', $valuesTmp['is_read'], PDO::PARAM_INT);
+			$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
+			$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
+			$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
+			$valuesTmp['tags'] = substr($valuesTmp['tags'], 0, 1023);
+			$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
+
+			if ($this->hasNativeHex()) {
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hash']);
+			} else {
+				$valuesTmp['hashBin'] = pack('H*', $valuesTmp['hash']);	//hex2bin() is PHP5.4+
+				$this->addEntryPrepared->bindParam(':hash', $valuesTmp['hashBin']);
+			}
 		}
-
 		if ($this->addEntryPrepared && $this->addEntryPrepared->execute()) {
 			return true;
 		} else {
 			$info = $this->addEntryPrepared == null ? array(0 => '', 1 => '', 2 => 'syntax error') : $this->addEntryPrepared->errorInfo();
 			if ($this->autoUpdateDb($info)) {
+				$this->addEntryPrepared = null;
 				return $this->addEntry($valuesTmp);
 			} elseif ((int)((int)$info[0] / 1000) !== 23) {	//Filter out "SQLSTATE Class code 23: Constraint Violation" because of expected duplicate entries
 				Minz_Log::error('SQL error addEntry: ' . $info[0] . ': ' . $info[1] . ' ' . $info[2]
@@ -165,6 +201,22 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		}
 	}
 
+	public function commitNewEntries() {
+		$sql = 'SET @rank=(SELECT MAX(id) - COUNT(*) FROM `' . $this->prefix . 'entrytmp`); ' .	//MySQL-specific
+			'INSERT IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags) ' .
+				'SELECT @rank:=@rank+1 AS id, guid, title, author, content_bin, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date; ' .
+			'DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= @rank;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	private $updateEntryPrepared = null;
 
 	public function updateEntry($valuesTmp) {

app/Models/EntryDAOPGSQL.php

@@ -11,6 +11,11 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
+		if (isset($errorInfo[0])) { 
+			if ($errorInfo[0] === '42P01' && stripos($errorInfo[2], 'entrytmp') !== false) {	//undefined_table
+				return $this->createEntryTempTable();
+			}
+		}
 		return false;
 	}
 
@@ -18,6 +23,27 @@ class FreshRSS_EntryDAOPGSQL extends FreshRSS_EntryDAOSQLite {
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = 'DO $$
+DECLARE
+maxrank bigint := (SELECT MAX(id) FROM `' . $this->prefix . 'entrytmp`);
+rank bigint := (SELECT maxrank - COUNT(*) FROM `' . $this->prefix . 'entrytmp`);
+BEGIN
+	INSERT INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+		(SELECT rank + row_number() OVER(ORDER BY date) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date);
+	DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= maxrank;
+END $$;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	public function size($all = true) {
 		$db = FreshRSS_Context::$system_conf->db;
 		$sql = 'SELECT pg_size_pretty(pg_database_size(?))';

app/Models/EntryDAOSQLite.php

@@ -7,21 +7,42 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO {
 	}
 
 	protected function autoUpdateDb($errorInfo) {
-		if (empty($errorInfo[0]) || $errorInfo[0] == '42S22') {	//ER_BAD_FIELD_ERROR
-			//autoAddColumn
-			if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
-				$showCreate = $tableInfo->fetchColumn();
-				Minz_Log::debug('FreshRSS_EntryDAOSQLite::autoUpdateDb: ' . $showCreate);
-				foreach (array('lastSeen', 'hash') as $column) {
-					if (stripos($showCreate, $column) === false) {
-						return $this->addColumn($column);
-					}
+		Minz_Log::error('FreshRSS_EntryDAO::autoUpdateDb error: ' . print_r($errorInfo, true));
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entrytmp'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			if (stripos($showCreate, 'entrytmp') === false) {
+				return $this->createEntryTempTable();
+			}
+		}
+		if ($tableInfo = $this->bd->query("SELECT sql FROM sqlite_master where name='entry'")) {
+			$showCreate = $tableInfo->fetchColumn();
+			foreach (array('lastSeen', 'hash') as $column) {
+				if (stripos($showCreate, $column) === false) {
+					return $this->addColumn($column);
 				}
 			}
 		}
 		return false;
 	}
 
+	public function commitNewEntries() {
+		$sql = '
+CREATE TEMP TABLE `tmp` AS SELECT id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `' . $this->prefix . 'entrytmp` ORDER BY date;
+INSERT OR IGNORE INTO `' . $this->prefix . 'entry` (id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags)
+	SELECT rowid + (SELECT MAX(id) - COUNT(*) FROM `tmp`) AS id, guid, title, author, content, link, date, `lastSeen`, hash, is_read, is_favorite, id_feed, tags FROM `tmp` ORDER BY date;
+DELETE FROM `' . $this->prefix . 'entrytmp` WHERE id <= (SELECT MAX(id) FROM `tmp`);
+DROP TABLE `tmp`;';
+		$hadTransaction = $this->bd->inTransaction();
+		if (!$hadTransaction) {
+			$this->bd->beginTransaction();
+		}
+		$result = $this->bd->exec($sql) !== false;
+		if (!$hadTransaction) {
+			$this->bd->commit();
+		}
+		return $result;
+	}
+
 	protected function sqlConcat($s1, $s2) {
 		return $s1 . '||' . $s2;
 	}

app/Models/Factory.php

@@ -3,14 +3,7 @@
 class FreshRSS_Factory {
 
 	public static function createFeedDao($username = null) {
-		$conf = Minz_Configuration::get('system');
-		switch ($conf->db['type']) {
-			case 'sqlite':
-			case 'pgsql':
-				return new FreshRSS_FeedDAOSQLite($username);
-			default:
-				return new FreshRSS_FeedDAO($username);
-		}
+		return new FreshRSS_FeedDAO($username);
 	}
 
 	public static function createEntryDao($username = null) {

app/Models/FeedDAO.php

@@ -92,29 +92,15 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		}
 	}
 
-	public function updateLastUpdate($id, $inError = false, $updateCache = true, $mtime = 0) {
-		if ($updateCache) {
-			$sql = 'UPDATE `' . $this->prefix . 'feed` '	//2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE
-			     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
-			     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0),'
-			     . '`lastUpdate`=?, error=? '
-			     . 'WHERE id=?';
-		} else {
-			$sql = 'UPDATE `' . $this->prefix . 'feed` '
-			     . 'SET `lastUpdate`=?, error=? '
-			     . 'WHERE id=?';
-		}
-
-		if ($mtime <= 0) {
-			$mtime = time();
-		}
-
+	public function updateLastUpdate($id, $inError = false, $mtime = 0) {	//See also updateCachedValue()
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '
+		     . 'SET `lastUpdate`=?, error=? '
+		     . 'WHERE id=?';
 		$values = array(
-			$mtime,
+			$mtime <= 0 ? time() : $mtime,
 			$inError ? 1 : 0,
 			$id,
 		);
-
 		$stm = $this->bd->prepare($sql);
 
 		if ($stm && $stm->execute($values)) {
@@ -294,18 +280,28 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $res[0]['count'];
 	}
 
-	public function updateCachedValues() {	//For one single feed, call updateLastUpdate($id)
-		$sql = 'UPDATE `' . $this->prefix . 'feed` f '
-		     . 'INNER JOIN ('
-		     .	'SELECT e.id_feed, '
-		     .	'COUNT(CASE WHEN e.is_read = 0 THEN 1 END) AS nbUnreads, '
-		     .	'COUNT(e.id) AS nbEntries '
-		     .	'FROM `' . $this->prefix . 'entry` e '
-		     .	'GROUP BY e.id_feed'
-		     . ') x ON x.id_feed=f.id '
-		     . 'SET f.`cache_nbEntries`=x.nbEntries, f.`cache_nbUnreads`=x.nbUnreads';
+	public function updateCachedValue($id) {	//For multiple feeds, call updateCachedValues()
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '	//2 sub-requests with FOREIGN KEY(e.id_feed), INDEX(e.is_read) faster than 1 request with GROUP BY or CASE
+		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
+		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0) '
+		     . 'WHERE id=?';
+		$values = array($id);
 		$stm = $this->bd->prepare($sql);
 
+		if ($stm && $stm->execute($values)) {
+			return $stm->rowCount();
+		} else {
+			$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
+			Minz_Log::error('SQL error updateCachedValue: ' . $info[2]);
+			return false;
+		}
+	}
+
+	public function updateCachedValues() {	//For one single feed, call updateCachedValue($id)
+		$sql = 'UPDATE `' . $this->prefix . 'feed` '
+		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
+		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)';
+		$stm = $this->bd->prepare($sql);
 		if ($stm && $stm->execute()) {
 			return $stm->rowCount();
 		} else {
@@ -343,7 +339,7 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $affected;
 	}
 
-	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateLastUpdate($id) or updateCachedValues() just after
+	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateCachedValue($id) or updateCachedValues() just after
 		$sql = 'DELETE FROM `' . $this->prefix . 'entry` '
 		     . 'WHERE id_feed=:id_feed AND id<=:id_max '
 		     . 'AND is_favorite=0 '	//Do not remove favourites

app/Models/FeedDAOSQLite.php

@@ -1,19 +0,0 @@
-<?php
-
-class FreshRSS_FeedDAOSQLite extends FreshRSS_FeedDAO {
-
-	public function updateCachedValues() {	//For one single feed, call updateLastUpdate($id)
-		$sql = 'UPDATE `' . $this->prefix . 'feed` '
-		     . 'SET `cache_nbEntries`=(SELECT COUNT(e1.id) FROM `' . $this->prefix . 'entry` e1 WHERE e1.id_feed=`' . $this->prefix . 'feed`.id),'
-		     . '`cache_nbUnreads`=(SELECT COUNT(e2.id) FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed=`' . $this->prefix . 'feed`.id AND e2.is_read=0)';
-		$stm = $this->bd->prepare($sql);
-		if ($stm && $stm->execute()) {
-			return $stm->rowCount();
-		} else {
-			$info = $stm == null ? array(2 => 'syntax error') : $stm->errorInfo();
-			Minz_Log::error('SQL error updateCachedValues: ' . $info[2]);
-			return false;
-		}
-	}
-
-}

app/Models/UserDAO.php

@@ -14,21 +14,23 @@ class FreshRSS_UserDAO extends Minz_ModelPdo {
 			$ok = false;
 			$bd_prefix_user = $db['prefix'] . $username . '_';
 			if (defined('SQL_CREATE_TABLES')) {	//E.g. MySQL
-				$sql = sprintf(SQL_CREATE_TABLES, $bd_prefix_user, _t('gen.short.default_category'));
+				$sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP, $bd_prefix_user, _t('gen.short.default_category'));
 				$stm = $userPDO->bd->prepare($sql);
 				$ok = $stm && $stm->execute();
 			} else {	//E.g. SQLite
 				global $SQL_CREATE_TABLES;
+				global $SQL_CREATE_TABLE_ENTRYTMP;
 				if (is_array($SQL_CREATE_TABLES)) {
-					$ok = true;
-					foreach ($SQL_CREATE_TABLES as $instruction) {
+					$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP);
+					$ok = !empty($instructions);
+					foreach ($instructions as $instruction) {
 						$sql = sprintf($instruction, $bd_prefix_user, _t('gen.short.default_category'));
 						$stm = $userPDO->bd->prepare($sql);
 						$ok &= ($stm && $stm->execute());
 					}
 				}
 			}
-			if ($insertDefaultFeeds) {
+			if ($ok && $insertDefaultFeeds) {
 				if (defined('SQL_INSERT_FEEDS')) {	//E.g. MySQL
 					$sql = sprintf(SQL_INSERT_FEEDS, $bd_prefix_user);
 					$stm = $userPDO->bd->prepare($sql);

app/SQL/install.sql.mysql.php

@@ -55,12 +55,38 @@ CREATE TABLE IF NOT EXISTS `%1$sentry` (
 	INDEX (`is_favorite`),	-- v0.7
 	INDEX (`is_read`),	-- v0.7
 	INDEX `entry_lastSeen_index` (`lastSeen`)	-- v1.1.1
+	-- INDEX `entry_feed_read_index` (`id_feed`,`is_read`)	-- v1.7 Located futher down
 ) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
 ENGINE = INNODB;
 
 INSERT IGNORE INTO `%1$scategory` (id, name) VALUES(1, "%2$s");
 ');
 
+define('SQL_CREATE_TABLE_ENTRYTMP', '
+CREATE TABLE IF NOT EXISTS `%1$sentrytmp` (	-- v1.7
+	`id` bigint NOT NULL,
+	`guid` varchar(760) CHARACTER SET latin1 NOT NULL,
+	`title` varchar(255) NOT NULL,
+	`author` varchar(255),
+	`content_bin` blob,
+	`link` varchar(1023) CHARACTER SET latin1 NOT NULL,
+	`date` int(11),
+	`lastSeen` INT(11) DEFAULT 0,
+	`hash` BINARY(16),
+	`is_read` boolean NOT NULL DEFAULT 0,
+	`is_favorite` boolean NOT NULL DEFAULT 0,
+	`id_feed` SMALLINT,
+	`tags` varchar(1023),
+	PRIMARY KEY (`id`),
+	FOREIGN KEY (`id_feed`) REFERENCES `%1$sfeed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE KEY (`id_feed`,`guid`),
+	INDEX (`date`)
+) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci
+ENGINE = INNODB;
+
+CREATE INDEX `entry_feed_read_index` ON `%1$sentry`(`id_feed`,`is_read`);	-- v1.7 Located here to be auto-added
+');
+
 define('SQL_INSERT_FEEDS', '
 INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);
 INSERT IGNORE INTO `%1$sfeed` (url, category, name, website, description, ttl) VALUES("https://github.com/FreshRSS/FreshRSS/releases.atom", 1, "FreshRSS @ GitHub", "https://github.com/FreshRSS/FreshRSS/", "FreshRSS releases @ GitHub", 86400);

app/SQL/install.sql.pgsql.php

@@ -54,6 +54,30 @@ $SQL_CREATE_TABLES = array(
 'INSERT INTO "%1$scategory" (name) SELECT \'%2$s\' WHERE NOT EXISTS (SELECT id FROM "%1$scategory" WHERE id = 1);',
 );
 
+global $SQL_CREATE_TABLE_ENTRYTMP;
+$SQL_CREATE_TABLE_ENTRYTMP = array(
+'CREATE TABLE IF NOT EXISTS "%1$sentrytmp" (	-- v1.7
+	"id" BIGINT NOT NULL PRIMARY KEY,
+	"guid" VARCHAR(760) NOT NULL,
+	"title" VARCHAR(255) NOT NULL,
+	"author" VARCHAR(255),
+	"content" TEXT,
+	"link" VARCHAR(1023) NOT NULL,
+	"date" INT,
+	"lastSeen" INT DEFAULT 0,
+	"hash" BYTEA,
+	"is_read" SMALLINT NOT NULL DEFAULT 0,
+	"is_favorite" SMALLINT NOT NULL DEFAULT 0,
+	"id_feed" SMALLINT,
+	"tags" VARCHAR(1023),
+	FOREIGN KEY ("id_feed") REFERENCES "%1$sfeed" ("id") ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE ("id_feed","guid")
+);',
+'CREATE INDEX %1$sentrytmp_date_index ON "%1$sentrytmp" ("date");',
+
+'CREATE INDEX %1$sentry_feed_read_index ON "%1$sentry" ("id_feed","is_read");',	//v1.7
+);
+
 global $SQL_INSERT_FEEDS;
 $SQL_INSERT_FEEDS = array(
 'INSERT INTO "%1$sfeed" (url, category, name, website, description, ttl) SELECT \'http://freshrss.org/feeds/all.atom.xml\', 1, \'FreshRSS.org\', \'http://freshrss.org/\', \'FreshRSS, a free, self-hostable aggregator…\', 86400 WHERE NOT EXISTS (SELECT id FROM "%1$sfeed" WHERE url = \'http://freshrss.org/feeds/all.atom.xml\');',

app/SQL/install.sql.sqlite.php

@@ -26,7 +26,6 @@ $SQL_CREATE_TABLES = array(
 	FOREIGN KEY (`category`) REFERENCES `category`(`id`) ON DELETE SET NULL ON UPDATE CASCADE,
 	UNIQUE (`url`)
 );',
-
 'CREATE INDEX IF NOT EXISTS feed_name_index ON `feed`(`name`);',
 'CREATE INDEX IF NOT EXISTS feed_priority_index ON `feed`(`priority`);',
 'CREATE INDEX IF NOT EXISTS feed_keep_history_index ON `feed`(`keep_history`);',
@@ -49,7 +48,6 @@ $SQL_CREATE_TABLES = array(
 	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE (`id_feed`,`guid`)
 );',
-
 'CREATE INDEX IF NOT EXISTS entry_is_favorite_index ON `entry`(`is_favorite`);',
 'CREATE INDEX IF NOT EXISTS entry_is_read_index ON `entry`(`is_read`);',
 'CREATE INDEX IF NOT EXISTS entry_lastSeen_index ON `entry`(`lastSeen`);',	//v1.1.1
@@ -57,6 +55,31 @@ $SQL_CREATE_TABLES = array(
 'INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "%2$s");',
 );
 
+global $SQL_CREATE_TABLE_ENTRYTMP;
+$SQL_CREATE_TABLE_ENTRYTMP = array(
+'CREATE TABLE IF NOT EXISTS `entrytmp` (	-- v1.7
+	`id` bigint NOT NULL,
+	`guid` varchar(760) NOT NULL,
+	`title` varchar(255) NOT NULL,
+	`author` varchar(255),
+	`content` text,
+	`link` varchar(1023) NOT NULL,
+	`date` int(11),
+	`lastSeen` INT(11) DEFAULT 0,
+	`hash` BINARY(16),
+	`is_read` boolean NOT NULL DEFAULT 0,
+	`is_favorite` boolean NOT NULL DEFAULT 0,
+	`id_feed` SMALLINT,
+	`tags` varchar(1023),
+	PRIMARY KEY (`id`),
+	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+	UNIQUE (`id_feed`,`guid`)
+);',
+'CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);',
+
+'CREATE INDEX IF NOT EXISTS `entry_feed_read_index` ON `entry`(`id_feed`,`is_read`);',	//v1.7
+);
+
 global $SQL_INSERT_FEEDS;
 $SQL_INSERT_FEEDS = array(
 'INSERT OR IGNORE INTO `feed` (url, category, name, website, description, ttl) VALUES("http://freshrss.org/feeds/all.atom.xml", 1, "FreshRSS.org", "http://freshrss.org/", "FreshRSS, a free, self-hostable aggregator…", 86400);',

app/install.php

@@ -342,35 +342,19 @@ function checkDbUser(&$dbOptions) {
 	$driver_options = $dbOptions['options'];
 	try {
 		$c = new PDO($str, $dbOptions['user'], $dbOptions['password'], $driver_options);
-
 		if (defined('SQL_CREATE_TABLES')) {
-			$sql = sprintf(SQL_CREATE_TABLES, $dbOptions['prefix_user'], _t('gen.short.default_category'));
-			$stm = $c->prepare($sql);
-			$ok = $stm->execute();
-		} else {
-			global $SQL_CREATE_TABLES;
-			if (is_array($SQL_CREATE_TABLES)) {
-				$ok = true;
-				foreach ($SQL_CREATE_TABLES as $instruction) {
-					$sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category'));
-					$stm = $c->prepare($sql);
-					$ok &= $stm->execute();
-				}
-			}
-		}
-
-		if (defined('SQL_INSERT_FEEDS')) {
-			$sql = sprintf(SQL_INSERT_FEEDS, $dbOptions['prefix_user']);
+			$sql = sprintf(SQL_CREATE_TABLES . SQL_CREATE_TABLE_ENTRYTMP . SQL_INSERT_FEEDS,
+				$dbOptions['prefix_user'], _t('gen.short.default_category'));
 			$stm = $c->prepare($sql);
-			$ok &= $stm->execute();
+			$ok = $stm && $stm->execute();
 		} else {
-			global $SQL_INSERT_FEEDS;
-			if (is_array($SQL_INSERT_FEEDS)) {
-				foreach ($SQL_INSERT_FEEDS as $instruction) {
-					$sql = sprintf($instruction, $dbOptions['prefix_user']);
-					$stm = $c->prepare($sql);
-					$ok &= $stm->execute();
-				}
+			global $SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS;
+			$instructions = array_merge($SQL_CREATE_TABLES, $SQL_CREATE_TABLE_ENTRYTMP, $SQL_INSERT_FEEDS);
+			$ok = !empty($instructions);
+			foreach ($instructions as $instruction) {
+				$sql = sprintf($instruction, $dbOptions['prefix_user'], _t('gen.short.default_category'));
+				$stm = $c->prepare($sql);
+				$ok &= $stm && $stm->execute();
 			}
 		}
 	} catch (PDOException $e) {

p/scripts/main.js

@@ -803,12 +803,12 @@ function updateFeed(feeds, feeds_count) {
 	if (!feed) {
 		return;
 	}
-
 	$.ajax({
 		type: 'POST',
 		url: feed.url,
 		data : {
 			_csrf: context.csrf,
+			noCommit: feeds.length > 0 ? 1 : 0,
 		},
 	}).always(function (data) {
 		feed_processed++;
@@ -830,7 +830,6 @@ function init_actualize() {
 		if (ajax_loading) {
 			return false;
 		}
-
 		ajax_loading = true;
 
 		$.getJSON('./?c=javascript&a=actualize').done(function (data) {
@@ -841,7 +840,16 @@ function init_actualize() {
 			}
 			if (data.feeds.length === 0) {
 				openNotification(data.feedback_no_refresh, "good");
-				ajax_loading = false;
+				$.ajax({	//Empty request to force refresh server database cache
+					type: 'POST',
+					url: './?c=feed&a=actualize&id=-1',
+					data : {
+						_csrf: context.csrf,
+						noCommit: 0,
+					},
+				}).always(function (data) {
+					ajax_loading = false;
+				});
 				return;
 			}
 			//Progress bar