SQL: more robust purge

https://github.com/FreshRSS/FreshRSS/issues/798
https://github.com/FreshRSS/FreshRSS/issues/493

app/Controllers/feedController.php

@@ -329,7 +329,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				// For this feed, check existing GUIDs already in database.
 				$existingHashForGuids = $entryDAO->listHashForFeedGuids($feed->id(), $newGuids);
 				unset($newGuids);
-				$use_declared_date = empty($existingHashForGuids);
 
 				$oldGuids = array();
 				// Add entries in database if possible.
@@ -353,14 +352,14 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 						// This entry should not be added considering configuration and date.
 						$oldGuids[] = $entry->guid();
 					} else {
-						$id = uTimeString();
-						if ($use_declared_date || $entry_date < $date_min) {
-							// Use declared date at first import.
+						if ($entry_date < $date_min) {
 							$id = min(time(), $entry_date) . uSecString();
+							$entry->_isRead(true);	//Old article that was not in database. Probably an error, so mark as read
+						} else {
+							$id = uTimeString();
+							$entry->_isRead($is_read);
 						}
-
 						$entry->_id($id);
-						$entry->_isRead($is_read);
 
 						$entry = Minz_ExtensionManager::callHook('entry_before_insert', $entry);
 						if ($entry === null) {
@@ -376,7 +375,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				}
 				$entryDAO->updateLastSeen($feed->id(), $oldGuids);
 			}
-			//TODO: updateLastSeen old GUIDS once in a while, in the case of caching (i.e. the whole feed content has not changed)
 
 			if ($feed_history >= 0 && rand(0, 30) === 1) {
 				// TODO: move this function in web cron when available (see entry::purge)
@@ -384,7 +382,6 @@ class FreshRSS_feed_Controller extends Minz_ActionController {
 				if (!$entryDAO->hasTransaction()) {
 					$entryDAO->beginTransaction();
 				}
-				//TODO: more robust system based on entry.lastSeen to avoid cleaning entries that are still published in the RSS feed.
 
 				$nb = $feedDAO->cleanOldEntries($feed->id(),
 				                                $date_min,

app/Models/Feed.php

@@ -245,7 +245,7 @@ class FreshRSS_Feed extends Minz_Model {
 					$this->_url($clean_url);
 				}
 
-				if (($mtime === true) ||($mtime > $this->lastUpdate)) {
+				if (($mtime === true) || ($mtime > $this->lastUpdate)) {
 					Minz_Log::notice('FreshRSS no cache ' . $mtime . ' > ' . $this->lastUpdate . ' for ' . $clean_url);
 					$this->loadEntries($feed);	// et on charge les articles du flux
 				} else {
@@ -255,7 +255,6 @@ class FreshRSS_Feed extends Minz_Model {
 
 				$feed->__destruct();	//http://simplepie.org/wiki/faq/i_m_getting_memory_leaks
 				unset($feed);
-				//TODO: Return a different information in case of cache/no-cache, and give access to the GUIDs in case of cache
 			}
 		}
 	}

app/Models/FeedDAO.php

@@ -322,10 +322,12 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return $affected;
 	}
 
-	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateLastUpdate($id) just after
+	public function cleanOldEntries($id, $date_min, $keep = 15) {	//Remember to call updateLastUpdate($id) or updateCachedValues() just after
 		$sql = 'DELETE FROM `' . $this->prefix . 'entry` '
-		     . 'WHERE id_feed = :id_feed AND id <= :id_max AND is_favorite=0 AND id NOT IN '
-		     . '(SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed = :id_feed ORDER BY id DESC LIMIT :keep) keep)';	//Double select MySQL doesn't yet support 'LIMIT & IN/ALL/ANY/SOME subquery'
+		     . 'WHERE id_feed = :id_feed AND id <= :id_max '
+		     . 'AND is_favorite=0 '	//Do not remove favourites
+		     . 'AND lastSeen < (SELECT maxLastSeen FROM (SELECT (MAX(e3.lastSeen) - 99) AS maxLastSeen FROM `' . $this->prefix . 'entry` e3 WHERE e3.id_feed = :id_feed) recent) '	//Do not remove the most newly seen articles, plus a few seconds of tolerance
+		     . 'AND id NOT IN (SELECT id FROM (SELECT e2.id FROM `' . $this->prefix . 'entry` e2 WHERE e2.id_feed = :id_feed ORDER BY id DESC LIMIT :keep) keep)';	//Double select: MySQL doesn't support 'LIMIT & IN/ALL/ANY/SOME subquery'
 		$stm = $this->bd->prepare($sql);
 
 		$id_max = intval($date_min) . '000000';