CLI optimize database (#1663)

CLI optimize database https://github.com/FreshRSS/FreshRSS/issues/1583
And VACUUM in SQLite https://github.com/FreshRSS/FreshRSS/issues/918
Add VACUUM for PostgreSQL (Not tested yet)

CHANGELOG.md

@@ -8,6 +8,10 @@
 	* Remove "SimplePie" name from HTTP User-Agent string [#1656](https://github.com/FreshRSS/FreshRSS/pull/1656)
 * Bug fixing
 	* Work-around for `CURLOPT_FOLLOWLOCATION` `open_basedir` bug in favicons and PubSubHubbub [#1655](https://github.com/FreshRSS/FreshRSS/issues/1655)
+* CLI
+	* New command `./cli/db-optimize.php` for database optimisation [#1583](https://github.com/FreshRSS/FreshRSS/issues/1583)
+* SQL
+	* Perform `VACUUM` on SQLite and PostgreSQL databases when optimisation is requested [#918](https://github.com/FreshRSS/FreshRSS/issues/918)
 * Misc.
 	* Translation validation tool [#1653](https://github.com/FreshRSS/FreshRSS/pull/1653)
 	* Translation manipulation tool [#1658](https://github.com/FreshRSS/FreshRSS/pull/1658)

app/Controllers/configureController.php

@@ -225,10 +225,12 @@ class FreshRSS_configure_Controller extends Minz_ActionController {
 
 		$entryDAO = FreshRSS_Factory::createEntryDao();
 		$this->view->nb_total = $entryDAO->count();
-		$this->view->size_user = $entryDAO->size();
+
+		$databaseDAO = FreshRSS_Factory::createDatabaseDAO();
+		$this->view->size_user = $databaseDAO->size();
 
 		if (FreshRSS_Auth::hasAccess('admin')) {
-			$this->view->size_total = $entryDAO->size(true);
+			$this->view->size_total = $databaseDAO->size(true);
 		}
 	}
 

app/Controllers/entryController.php

@@ -147,8 +147,8 @@ class FreshRSS_entry_Controller extends Minz_ActionController {
 
 		@set_time_limit(300);
 
-		$entryDAO = FreshRSS_Factory::createEntryDao();
-		$entryDAO->optimizeTable();
+		$databaseDAO = FreshRSS_Factory::createDatabaseDAO();
+		$databaseDAO->optimize();
 
 		$feedDAO = FreshRSS_Factory::createFeedDao();
 		$feedDAO->updateCachedValues();

app/Controllers/userController.php

@@ -120,7 +120,9 @@ class FreshRSS_user_Controller extends Minz_ActionController {
 		// Get information about the current user.
 		$entryDAO = FreshRSS_Factory::createEntryDao($this->view->current_user);
 		$this->view->nb_articles = $entryDAO->count();
-		$this->view->size_user = $entryDAO->size();
+
+		$databaseDAO = FreshRSS_Factory::createDatabaseDAO();
+		$this->view->size_user = $databaseDAO->size();
 	}
 
 	public static function createUser($new_user_name, $passwordPlain, $apiPasswordPlain, $userConfig = array(), $insertDefaultFeeds = true) {

app/Models/DatabaseDAO.php

@@ -80,4 +80,45 @@ class FreshRSS_DatabaseDAO extends Minz_ModelPdo {
 
 		return $list;
 	}
+
+	public function size($all = false) {
+		$db = FreshRSS_Context::$system_conf->db;
+		$sql = 'SELECT SUM(data_length + index_length) FROM information_schema.TABLES WHERE table_schema=?';	//MySQL
+		$values = array($db['base']);
+		if (!$all) {
+			$sql .= ' AND table_name LIKE ?';
+			$values[] = $this->prefix . '%';
+		}
+		$stm = $this->bd->prepare($sql);
+		$stm->execute($values);
+		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
+		return $res[0];
+	}
+
+	public function optimize() {
+		$ok = true;
+
+		$sql = 'OPTIMIZE TABLE `' . $this->prefix . 'entry`';	//MySQL
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		$sql = 'OPTIMIZE TABLE `' . $this->prefix . 'feed`';	//MySQL
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		$sql = 'OPTIMIZE TABLE `' . $this->prefix . 'category`';	//MySQL
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		return $ok;
+	}
 }

app/Models/DatabaseDAOPGSQL.php

@@ -40,4 +40,41 @@ class FreshRSS_DatabaseDAOPGSQL extends FreshRSS_DatabaseDAO {
 			'default' => $dao['default'],
 		);
 	}
+
+	public function size($all = true) {
+		$db = FreshRSS_Context::$system_conf->db;
+		$sql = 'SELECT pg_size_pretty(pg_database_size(?))';
+		$values = array($db['base']);
+		$stm = $this->bd->prepare($sql);
+		$stm->execute($values);
+		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
+		return $res[0];
+	}
+
+	public function optimize() {
+		$ok = true;
+
+		$sql = 'VACUUM `' . $this->prefix . 'entry`';
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		$sql = 'VACUUM `' . $this->prefix . 'feed`';
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		$sql = 'VACUUM `' . $this->prefix . 'category`';
+		$stm = $this->bd->prepare($sql);
+		$ok &= $stm != false;
+		if ($stm) {
+			$ok &= $stm->execute();
+		}
+
+		return $ok;
+	}
 }

app/Models/DatabaseDAOSQLite.php

@@ -45,4 +45,17 @@ class FreshRSS_DatabaseDAOSQLite extends FreshRSS_DatabaseDAO {
 			'default' => $dao['dflt_value'],
 		);
 	}
+
+	public function size($all = false) {
+		return @filesize(join_path(DATA_PATH, 'users', $this->current_user, 'db.sqlite'));
+	}
+
+	public function optimize() {
+		$sql = 'VACUUM';
+		$stm = $this->bd->prepare($sql);
+		if ($stm) {
+			return $stm->execute();
+		}
+		return false;
+	}
 }

app/Models/EntryDAO.php

@@ -885,28 +885,6 @@ class FreshRSS_EntryDAO extends Minz_ModelPdo implements FreshRSS_Searchable {
 		return array('all' => $all, 'unread' => $unread, 'read' => $all - $unread);
 	}
 
-	public function optimizeTable() {
-		$sql = 'OPTIMIZE TABLE `' . $this->prefix . 'entry`';	//MySQL
-		$stm = $this->bd->prepare($sql);
-		if ($stm) {
-			return $stm->execute();
-		}
-	}
-
-	public function size($all = false) {
-		$db = FreshRSS_Context::$system_conf->db;
-		$sql = 'SELECT SUM(data_length + index_length) FROM information_schema.TABLES WHERE table_schema=?';	//MySQL
-		$values = array($db['base']);
-		if (!$all) {
-			$sql .= ' AND table_name LIKE ?';
-			$values[] = $this->prefix . '%';
-		}
-		$stm = $this->bd->prepare($sql);
-		$stm->execute($values);
-		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
-		return $res[0];
-	}
-
 	public static function daoToEntry($dao) {
 		$entry = new FreshRSS_Entry(
 				$dao['id_feed'],

app/Models/EntryDAOPGSQL.php

@@ -46,15 +46,4 @@ END $$;';
 		}
 		return $result;
 	}
-
-	public function size($all = true) {
-		$db = FreshRSS_Context::$system_conf->db;
-		$sql = 'SELECT pg_size_pretty(pg_database_size(?))';
-		$values = array($db['base']);
-		$stm = $this->bd->prepare($sql);
-		$stm->execute($values);
-		$res = $stm->fetchAll(PDO::FETCH_COLUMN, 0);
-		return $res[0];
-	}
-
 }

app/Models/EntryDAOSQLite.php

@@ -261,12 +261,4 @@ class FreshRSS_EntryDAOSQLite extends FreshRSS_EntryDAO {
 		}
 		return $affected;
 	}
-
-	public function optimizeTable() {
-		//TODO: Search for an equivalent in SQLite
-	}
-
-	public function size($all = false) {
-		return @filesize(join_path(DATA_PATH, 'users', $this->current_user, 'db.sqlite'));
-	}
 }

cli/README.md

@@ -69,6 +69,9 @@ cd /usr/share/FreshRSS
 # Returns: 1) a * iff the user is admin, 2) the name of the user,
 #  3) the date/time of last user action, 4) the size occupied,
 #  and the number of: 5) categories, 6) feeds, 7) read articles, 8) unread articles, and 9) favourites
+
+./cli/db-optimize.php --user username
+# Optimize database (reduces the size) for a given user (perform `OPTIMIZE TABLE` in MySQL, `VACUUM` in SQLite)
 ```
 
 

cli/db-optimize.php

@@ -0,0 +1,20 @@
+#!/usr/bin/php
+<?php
+require('_cli.php');
+
+$options = getopt('', array(
+		'user:',
+	));
+
+if (empty($options['user'])) {
+	fail('Usage: ' . basename(__FILE__) . " --user username");
+}
+
+$username = cliInitUser($options['user']);
+
+echo 'FreshRSS optimizing database for user “', $username, "”…\n";
+
+$databaseDAO = FreshRSS_Factory::createDatabaseDAO($username);
+$ok = $databaseDAO->optimize();
+
+done($ok);

cli/user-info.php

@@ -19,6 +19,7 @@ foreach ($users as $username) {
 	$catDAO = new FreshRSS_CategoryDAO();
 	$feedDAO = FreshRSS_Factory::createFeedDao($username);
 	$entryDAO = FreshRSS_Factory::createEntryDao($username);
+	$databaseDAO = FreshRSS_Factory::createDatabaseDAO($username);
 
 	$nbEntries = $entryDAO->countUnreadRead();
 	$nbFavorites = $entryDAO->countUnreadReadFavorites();
@@ -27,7 +28,7 @@ foreach ($users as $username) {
 		echo
 			$username, "\t",
 			date('c', FreshRSS_UserDAO::mtime($username)), "\t",
-			format_bytes($entryDAO->size()), "\t",
+			format_bytes($databaseDAO->size()), "\t",
 			$catDAO->count(), " categories\t",
 			count($feedDAO->listFeedsIds()), " feeds\t",
 			$nbEntries['read'], " reads\t",
@@ -38,7 +39,7 @@ foreach ($users as $username) {
 		echo
 			$username, "\t",
 			FreshRSS_UserDAO::mtime($username), "\t",
-			$entryDAO->size(), "\t",
+			$databaseDAO->size(), "\t",
 			$catDAO->count(), "\t",
 			count($feedDAO->listFeedsIds()), "\t",
 			$nbEntries['read'], "\t",