SQL increase length of VARCHAR fields (#5756)

* SQL increase length of VARCHAR fields
Increase length of all fields, keeping the limits for:
* Unique indexes on UTF-8: 191 bytes for MySQL;
* Unique indexes on ASCII: 767 bytes for MySQL;
* Max URL for external tools: 32768 characters;
* Max VARCHAR: 65535 bytes for MySQL;
Follow-up of https://github.com/FreshRSS/FreshRSS/pull/5038
Fix https://github.com/FreshRSS/FreshRSS/issues/4986

* Fix length test

app/Models/Category.php

@@ -143,7 +143,7 @@ class FreshRSS_Category extends Minz_Model {
 	}
 
 	public function _name(string $value): void {
-		$this->name = mb_strcut(trim($value), 0, 255, 'UTF-8');
+		$this->name = mb_strcut(trim($value), 0, FreshRSS_DatabaseDAO::LENGTH_INDEX_UNICODE, 'UTF-8');
 	}
 
 	/** @param array<FreshRSS_Feed>|FreshRSS_Feed $values */

app/Models/EntryDAO.php

@@ -109,18 +109,18 @@ SQL;
 		}
 		if ($this->addEntryPrepared) {
 			$this->addEntryPrepared->bindParam(':id', $valuesTmp['id']);
-			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
+			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 767);
 			$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
 			$this->addEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
-			$valuesTmp['title'] = mb_strcut($valuesTmp['title'], 0, 255, 'UTF-8');
+			$valuesTmp['title'] = mb_strcut($valuesTmp['title'], 0, 65535, 'UTF-8');
 			$valuesTmp['title'] = safe_utf8($valuesTmp['title']);
 			$this->addEntryPrepared->bindParam(':title', $valuesTmp['title']);
-			$valuesTmp['author'] = mb_strcut($valuesTmp['author'], 0, 255, 'UTF-8');
+			$valuesTmp['author'] = mb_strcut($valuesTmp['author'], 0, 65535, 'UTF-8');
 			$valuesTmp['author'] = safe_utf8($valuesTmp['author']);
 			$this->addEntryPrepared->bindParam(':author', $valuesTmp['author']);
 			$valuesTmp['content'] = safe_utf8($valuesTmp['content']);
 			$this->addEntryPrepared->bindParam(':content', $valuesTmp['content']);
-			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
+			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 32768);
 			$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
 			$this->addEntryPrepared->bindParam(':link', $valuesTmp['link']);
 			$valuesTmp['date'] = min($valuesTmp['date'], 2147483647);
@@ -134,7 +134,7 @@ SQL;
 			$valuesTmp['is_favorite'] = $valuesTmp['is_favorite'] ? 1 : 0;
 			$this->addEntryPrepared->bindParam(':is_favorite', $valuesTmp['is_favorite'], PDO::PARAM_INT);
 			$this->addEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
-			$valuesTmp['tags'] = mb_strcut($valuesTmp['tags'], 0, 1023, 'UTF-8');
+			$valuesTmp['tags'] = mb_strcut($valuesTmp['tags'], 0, 65535, 'UTF-8');
 			$valuesTmp['tags'] = safe_utf8($valuesTmp['tags']);
 			$this->addEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
 			if (!isset($valuesTmp['attributes'])) {
@@ -215,18 +215,18 @@ SQL;
 			$this->updateEntryPrepared = $this->pdo->prepare($sql) ?: null;
 		}
 		if ($this->updateEntryPrepared) {
-			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 760);
+			$valuesTmp['guid'] = substr($valuesTmp['guid'], 0, 767);
 			$valuesTmp['guid'] = safe_ascii($valuesTmp['guid']);
 			$this->updateEntryPrepared->bindParam(':guid', $valuesTmp['guid']);
-			$valuesTmp['title'] = mb_strcut($valuesTmp['title'], 0, 255, 'UTF-8');
+			$valuesTmp['title'] = mb_strcut($valuesTmp['title'], 0, 65535, 'UTF-8');
 			$valuesTmp['title'] = safe_utf8($valuesTmp['title']);
 			$this->updateEntryPrepared->bindParam(':title', $valuesTmp['title']);
-			$valuesTmp['author'] = mb_strcut($valuesTmp['author'], 0, 255, 'UTF-8');
+			$valuesTmp['author'] = mb_strcut($valuesTmp['author'], 0, 65535, 'UTF-8');
 			$valuesTmp['author'] = safe_utf8($valuesTmp['author']);
 			$this->updateEntryPrepared->bindParam(':author', $valuesTmp['author']);
 			$valuesTmp['content'] = safe_utf8($valuesTmp['content']);
 			$this->updateEntryPrepared->bindParam(':content', $valuesTmp['content']);
-			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 1023);
+			$valuesTmp['link'] = substr($valuesTmp['link'], 0, 32768);
 			$valuesTmp['link'] = safe_ascii($valuesTmp['link']);
 			$this->updateEntryPrepared->bindParam(':link', $valuesTmp['link']);
 			$valuesTmp['date'] = min($valuesTmp['date'], 2147483647);
@@ -243,7 +243,7 @@ SQL;
 				$this->updateEntryPrepared->bindValue(':is_favorite', $valuesTmp['is_favorite'] ? 1 : 0, PDO::PARAM_INT);
 			}
 			$this->updateEntryPrepared->bindParam(':id_feed', $valuesTmp['id_feed'], PDO::PARAM_INT);
-			$valuesTmp['tags'] = mb_strcut($valuesTmp['tags'], 0, 1023, 'UTF-8');
+			$valuesTmp['tags'] = mb_strcut($valuesTmp['tags'], 0, 65535, 'UTF-8');
 			$valuesTmp['tags'] = safe_utf8($valuesTmp['tags']);
 			$this->updateEntryPrepared->bindParam(':tags', $valuesTmp['tags']);
 			if (!isset($valuesTmp['attributes'])) {

app/Models/FeedDAO.php

@@ -58,10 +58,10 @@ class FreshRSS_FeedDAO extends Minz_ModelPdo {
 			$valuesTmp['category'],
 			mb_strcut(trim($valuesTmp['name']), 0, FreshRSS_DatabaseDAO::LENGTH_INDEX_UNICODE, 'UTF-8'),
 			$valuesTmp['website'],
-			sanitizeHTML($valuesTmp['description'], '', 1023),
+			sanitizeHTML($valuesTmp['description'], ''),
 			$valuesTmp['lastUpdate'],
 			isset($valuesTmp['priority']) ? (int)$valuesTmp['priority'] : FreshRSS_Feed::PRIORITY_MAIN_STREAM,
-			mb_strcut($valuesTmp['pathEntries'], 0, 511, 'UTF-8'),
+			mb_strcut($valuesTmp['pathEntries'], 0, 65535, 'UTF-8'),
 			base64_encode($valuesTmp['httpAuth']),
 			isset($valuesTmp['error']) ? (int)$valuesTmp['error'] : 0,
 			isset($valuesTmp['ttl']) ? (int)$valuesTmp['ttl'] : FreshRSS_Feed::TTL_DEFAULT,

app/Models/TagDAO.php

@@ -20,7 +20,7 @@ WHERE NOT EXISTS (SELECT 1 FROM `_category` WHERE name = TRIM(?))
 SQL;
 		$stm = $this->pdo->prepare($sql);
 
-		$valuesTmp['name'] = mb_strcut(trim($valuesTmp['name']), 0, 63, 'UTF-8');
+		$valuesTmp['name'] = mb_strcut(trim($valuesTmp['name']), 0, FreshRSS_DatabaseDAO::LENGTH_INDEX_UNICODE, 'UTF-8');
 		if (!isset($valuesTmp['attributes'])) {
 			$valuesTmp['attributes'] = [];
 		}
@@ -61,7 +61,7 @@ UPDATE `_tag` SET name=? WHERE id=?
 AND NOT EXISTS (SELECT 1 FROM `_category` WHERE name = ?)
 SQL;
 
-		$name = mb_strcut(trim($name), 0, 63, 'UTF-8');
+		$name = mb_strcut(trim($name), 0, FreshRSS_DatabaseDAO::LENGTH_INDEX_UNICODE, 'UTF-8');
 		$stm = $this->pdo->prepare($sql);
 		if ($stm !== false &&
 			$stm->bindValue(':id', $id, PDO::PARAM_INT) &&

app/SQL/install.sql.mysql.php

@@ -26,8 +26,8 @@ CREATE TABLE IF NOT EXISTS `_feed` (
 	`description` TEXT,
 	`lastUpdate` INT(11) DEFAULT 0,	-- Until year 2038
 	`priority` TINYINT(2) NOT NULL DEFAULT 10,
-	`pathEntries` VARCHAR(511) DEFAULT NULL,
-	`httpAuth` VARCHAR(511) DEFAULT NULL,
+	`pathEntries` VARCHAR(65535) DEFAULT NULL,
+	`httpAuth` VARCHAR(1024) DEFAULT NULL,
 	`error` BOOLEAN DEFAULT 0,
 	`ttl` INT NOT NULL DEFAULT 0,	-- v0.7.3
 	`attributes` TEXT,	-- v1.11.0
@@ -42,18 +42,18 @@ ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS `_entry` (
 	`id` BIGINT NOT NULL,	-- v0.7
-	`guid` VARCHAR(760) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,	-- Maximum for UNIQUE is 767B
-	`title` VARCHAR(255) NOT NULL,
-	`author` VARCHAR(255),
+	`guid` VARCHAR(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,	-- Maximum for UNIQUE is 767B
+	`title` VARCHAR(65535) NOT NULL,
+	`author` VARCHAR(65535),
 	`content_bin` MEDIUMBLOB,	-- v0.7
-	`link` VARCHAR(1023) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+	`link` VARCHAR(32768) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
 	`date` INT(11),	-- Until year 2038
 	`lastSeen` INT(11) DEFAULT 0,	-- v1.1.1, Until year 2038
 	`hash` BINARY(16),	-- v1.1.1
 	`is_read` BOOLEAN NOT NULL DEFAULT 0,
 	`is_favorite` BOOLEAN NOT NULL DEFAULT 0,
 	`id_feed` INT,	-- 1.20.0
-	`tags` VARCHAR(1023),
+	`tags` VARCHAR(65535),
 	`attributes` TEXT,	-- v1.20.0
 	PRIMARY KEY (`id`),
 	FOREIGN KEY (`id_feed`) REFERENCES `_feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -69,18 +69,18 @@ INSERT IGNORE INTO `_category` (id, name) VALUES(1, "Uncategorized");
 
 CREATE TABLE IF NOT EXISTS `_entrytmp` (	-- v1.7
 	`id` BIGINT NOT NULL,
-	`guid` VARCHAR(760) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
-	`title` VARCHAR(255) NOT NULL,
-	`author` VARCHAR(255),
+	`guid` VARCHAR(767) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+	`title` VARCHAR(65535) NOT NULL,
+	`author` VARCHAR(65535),
 	`content_bin` MEDIUMBLOB,
-	`link` VARCHAR(1023) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
+	`link` VARCHAR(32768) CHARACTER SET latin1 COLLATE latin1_bin NOT NULL,
 	`date` INT(11),
 	`lastSeen` INT(11) DEFAULT 0,
 	`hash` BINARY(16),
 	`is_read` BOOLEAN NOT NULL DEFAULT 0,
 	`is_favorite` BOOLEAN NOT NULL DEFAULT 0,
 	`id_feed` INT,	-- 1.20.0
-	`tags` VARCHAR(1023),
+	`tags` VARCHAR(65535),
 	`attributes` TEXT,	-- v1.20.0
 	PRIMARY KEY (`id`),
 	FOREIGN KEY (`id_feed`) REFERENCES `_feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -91,7 +91,7 @@ ENGINE = INNODB;
 
 CREATE TABLE IF NOT EXISTS `_tag` (	-- v1.12
 	`id` INT NOT NULL AUTO_INCREMENT,
-	`name` VARCHAR(63) NOT NULL,
+	`name` VARCHAR(191) NOT NULL,
 	`attributes` TEXT,
 	PRIMARY KEY (`id`),
 	UNIQUE KEY (`name`)

app/SQL/install.sql.pgsql.php

@@ -6,7 +6,7 @@ SQL;
 $GLOBALS['SQL_CREATE_TABLES'] = <<<'SQL'
 CREATE TABLE IF NOT EXISTS `_category` (
 	"id" SERIAL PRIMARY KEY,
-	"name" VARCHAR(255) UNIQUE NOT NULL,
+	"name" VARCHAR(191) UNIQUE NOT NULL,
 	"kind" SMALLINT DEFAULT 0,	-- 1.20.0
 	"lastUpdate" BIGINT DEFAULT 0,	-- 1.20.0
 	"error" SMALLINT DEFAULT 0,	-- 1.20.0
@@ -18,13 +18,13 @@ CREATE TABLE IF NOT EXISTS `_feed` (
 	"url" VARCHAR(32768) NOT NULL,
 	"kind" SMALLINT DEFAULT 0, -- 1.20.0
 	"category" INT DEFAULT 0,	-- 1.20.0
-	"name" VARCHAR(255) NOT NULL,
+	"name" VARCHAR(191) NOT NULL,
 	"website" VARCHAR(32768),
 	"description" TEXT,
 	"lastUpdate" INT DEFAULT 0,
 	"priority" SMALLINT NOT NULL DEFAULT 10,
-	"pathEntries" VARCHAR(511) DEFAULT NULL,
-	"httpAuth" VARCHAR(511) DEFAULT NULL,
+	"pathEntries" VARCHAR(65535) DEFAULT NULL,
+	"httpAuth" VARCHAR(1024) DEFAULT NULL,
 	"error" SMALLINT DEFAULT 0,
 	"ttl" INT NOT NULL DEFAULT 0,
 	"attributes" TEXT,	-- v1.11.0
@@ -37,18 +37,18 @@ CREATE INDEX IF NOT EXISTS `_priority_index` ON `_feed` ("priority");
 
 CREATE TABLE IF NOT EXISTS `_entry` (
 	"id" BIGINT NOT NULL PRIMARY KEY,
-	"guid" VARCHAR(760) NOT NULL,
-	"title" VARCHAR(255) NOT NULL,
-	"author" VARCHAR(255),
+	"guid" VARCHAR(767) NOT NULL,
+	"title" VARCHAR(65535) NOT NULL,
+	"author" VARCHAR(65535),
 	"content" TEXT,
-	"link" VARCHAR(1023) NOT NULL,
+	"link" VARCHAR(32768) NOT NULL,
 	"date" INT,
 	"lastSeen" INT DEFAULT 0,
 	"hash" BYTEA,
 	"is_read" SMALLINT NOT NULL DEFAULT 0,
 	"is_favorite" SMALLINT NOT NULL DEFAULT 0,
 	"id_feed" INT,	-- 1.20.0
-	"tags" VARCHAR(1023),
+	"tags" VARCHAR(65535),
 	"attributes" TEXT,	-- v1.20.0
 	FOREIGN KEY ("id_feed") REFERENCES `_feed` ("id") ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE ("id_feed","guid")
@@ -65,18 +65,18 @@ INSERT INTO `_category` (id, name)
 
 CREATE TABLE IF NOT EXISTS `_entrytmp` (	-- v1.7
 	"id" BIGINT NOT NULL PRIMARY KEY,
-	"guid" VARCHAR(760) NOT NULL,
-	"title" VARCHAR(255) NOT NULL,
-	"author" VARCHAR(255),
+	"guid" VARCHAR(767) NOT NULL,
+	"title" VARCHAR(65535) NOT NULL,
+	"author" VARCHAR(65535),
 	"content" TEXT,
-	"link" VARCHAR(1023) NOT NULL,
+	"link" VARCHAR(32768) NOT NULL,
 	"date" INT,
 	"lastSeen" INT DEFAULT 0,
 	"hash" BYTEA,
 	"is_read" SMALLINT NOT NULL DEFAULT 0,
 	"is_favorite" SMALLINT NOT NULL DEFAULT 0,
 	"id_feed" INT,	-- 1.20.0
-	"tags" VARCHAR(1023),
+	"tags" VARCHAR(65535),
 	"attributes" TEXT,	-- v1.20.0
 	FOREIGN KEY ("id_feed") REFERENCES `_feed` ("id") ON DELETE CASCADE ON UPDATE CASCADE,
 	UNIQUE ("id_feed","guid")
@@ -85,7 +85,7 @@ CREATE INDEX IF NOT EXISTS `_entrytmp_date_index` ON `_entrytmp` ("date");
 
 CREATE TABLE IF NOT EXISTS `_tag` (	-- v1.12
 	"id" SERIAL PRIMARY KEY,
-	"name" VARCHAR(63) UNIQUE NOT NULL,
+	"name" VARCHAR(191) UNIQUE NOT NULL,
 	"attributes" TEXT
 );
 CREATE TABLE IF NOT EXISTS `_entrytag` (

app/SQL/install.sql.sqlite.php

@@ -6,7 +6,7 @@ SQL;
 $GLOBALS['SQL_CREATE_TABLES'] = <<<'SQL'
 CREATE TABLE IF NOT EXISTS `category` (
 	`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`name` VARCHAR(255) NOT NULL,
+	`name` VARCHAR(191) NOT NULL,
 	`kind` SMALLINT DEFAULT 0,	-- 1.20.0
 	`lastUpdate` BIGINT DEFAULT 0,	-- 1.20.0
 	`error` SMALLINT DEFAULT 0,	-- 1.20.0
@@ -19,13 +19,13 @@ CREATE TABLE IF NOT EXISTS `feed` (
 	`url` VARCHAR(32768) NOT NULL,
 	`kind` SMALLINT DEFAULT 0,	-- 1.20.0
 	`category` INTEGER DEFAULT 0,	-- 1.20.0
-	`name` VARCHAR(255) NOT NULL,
+	`name` VARCHAR(191) NOT NULL,
 	`website` VARCHAR(32768),
 	`description` TEXT,
 	`lastUpdate` INT(11) DEFAULT 0,	-- Until year 2038
 	`priority` TINYINT(2) NOT NULL DEFAULT 10,
-	`pathEntries` VARCHAR(511) DEFAULT NULL,
-	`httpAuth` VARCHAR(511) DEFAULT NULL,
+	`pathEntries` VARCHAR(65535) DEFAULT NULL,
+	`httpAuth` VARCHAR(1024) DEFAULT NULL,
 	`error` BOOLEAN DEFAULT 0,
 	`ttl` INT NOT NULL DEFAULT 0,
 	`attributes` TEXT,	-- v1.11.0
@@ -38,18 +38,18 @@ CREATE INDEX IF NOT EXISTS feed_priority_index ON `feed`(`priority`);
 
 CREATE TABLE IF NOT EXISTS `entry` (
 	`id` BIGINT NOT NULL,
-	`guid` VARCHAR(760) NOT NULL,
-	`title` VARCHAR(255) NOT NULL,
-	`author` VARCHAR(255),
+	`guid` VARCHAR(767) NOT NULL,
+	`title` VARCHAR(65535) NOT NULL,
+	`author` VARCHAR(65535),
 	`content` TEXT,
-	`link` VARCHAR(1023) NOT NULL,
+	`link` VARCHAR(32768) NOT NULL,
 	`date` INT(11),	-- Until year 2038
 	`lastSeen` INT(11) DEFAULT 0,	-- v1.1.1, Until year 2038
 	`hash` BINARY(16),	-- v1.1.1
 	`is_read` BOOLEAN NOT NULL DEFAULT 0,
 	`is_favorite` BOOLEAN NOT NULL DEFAULT 0,
 	`id_feed` INTEGER,	-- 1.20.0
-	`tags` VARCHAR(1023),
+	`tags` VARCHAR(65535),
 	`attributes` TEXT,	-- v1.20.0
 	PRIMARY KEY (`id`),
 	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -64,18 +64,18 @@ INSERT OR IGNORE INTO `category` (id, name) VALUES(1, "Uncategorized");
 
 CREATE TABLE IF NOT EXISTS `entrytmp` (	-- v1.7
 	`id` BIGINT NOT NULL,
-	`guid` VARCHAR(760) NOT NULL,
-	`title` VARCHAR(255) NOT NULL,
-	`author` VARCHAR(255),
+	`guid` VARCHAR(767) NOT NULL,
+	`title` VARCHAR(65535) NOT NULL,
+	`author` VARCHAR(65535),
 	`content` TEXT,
-	`link` VARCHAR(1023) NOT NULL,
+	`link` VARCHAR(32768) NOT NULL,
 	`date` INT(11),
 	`lastSeen` INT(11) DEFAULT 0,
 	`hash` BINARY(16),
 	`is_read` BOOLEAN NOT NULL DEFAULT 0,
 	`is_favorite` BOOLEAN NOT NULL DEFAULT 0,
 	`id_feed` INTEGER,	-- 1.20.0
-	`tags` VARCHAR(1023),
+	`tags` VARCHAR(65535),
 	`attributes` TEXT,	-- v1.20.0
 	PRIMARY KEY (`id`),
 	FOREIGN KEY (`id_feed`) REFERENCES `feed`(`id`) ON DELETE CASCADE ON UPDATE CASCADE,
@@ -85,7 +85,7 @@ CREATE INDEX IF NOT EXISTS entrytmp_date_index ON `entrytmp`(`date`);
 
 CREATE TABLE IF NOT EXISTS `tag` (	-- v1.12
 	`id` INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`name` VARCHAR(63) NOT NULL,
+	`name` VARCHAR(191) NOT NULL,
 	`attributes` TEXT,
 	UNIQUE (`name`)
 );

tests/app/Models/CategoryTest.php

@@ -24,8 +24,7 @@ class CategoryTest extends PHPUnit\Framework\TestCase {
 			array('  this string needs trimming on left', 'this string needs trimming on left'),
 			array('this string needs trimming on right  ', 'this string needs trimming on right'),
 			array('  this string needs trimming on both ends  ', 'this string needs trimming on both ends'),
-			array(str_repeat('This string needs to be shortened because its length is way too long. ', 4),
-				str_repeat('This string needs to be shortened because its length is way too long. ', 3) . 'This string needs to be shortened because its'),
+			array(str_repeat('X', 512), str_repeat('X', FreshRSS_DatabaseDAO::LENGTH_INDEX_UNICODE)),	// max length
 		);
 	}