database: entry URLs can exceeds btree index size limit

internal/database/migrations.go

@@ -882,4 +882,10 @@ var migrations = []func(tx *sql.Tx) error{
 		_, err = tx.Exec(sql)
 		return err
 	},
+	func(tx *sql.Tx) (err error) {
+		// Entry URLs can exceeds btree maximum size
+		// Checking entry existence is now using entries_feed_id_status_hash_idx index
+		_, err = tx.Exec(`DROP INDEX entries_feed_url_idx`)
+		return err
+	},
 }

internal/reader/processor/processor.go

@@ -42,8 +42,9 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
 
 		slog.Debug("Processing entry",
 			slog.Int64("user_id", user.ID),
-			slog.Int64("entry_id", entry.ID),
 			slog.String("entry_url", entry.URL),
+			slog.String("entry_hash", entry.Hash),
+			slog.String("entry_title", entry.Title),
 			slog.Int64("feed_id", feed.ID),
 			slog.String("feed_url", feed.FeedURL),
 		)
@@ -52,14 +53,18 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
 		}
 
 		websiteURL := getUrlFromEntry(feed, entry)
-		entryIsNew := !store.EntryURLExists(feed.ID, entry.URL)
+		entryIsNew := store.IsNewEntry(feed.ID, entry.Hash)
 		if feed.Crawler && (entryIsNew || forceRefresh) {
 			slog.Debug("Scraping entry",
 				slog.Int64("user_id", user.ID),
-				slog.Int64("entry_id", entry.ID),
 				slog.String("entry_url", entry.URL),
+				slog.String("entry_hash", entry.Hash),
+				slog.String("entry_title", entry.Title),
 				slog.Int64("feed_id", feed.ID),
 				slog.String("feed_url", feed.FeedURL),
+				slog.Bool("entry_is_new", entryIsNew),
+				slog.Bool("force_refresh", forceRefresh),
+				slog.String("website_url", websiteURL),
 			)
 
 			startTime := time.Now()
@@ -90,7 +95,6 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed, user *model.Us
 			if scraperErr != nil {
 				slog.Warn("Unable to scrape entry",
 					slog.Int64("user_id", user.ID),
-					slog.Int64("entry_id", entry.ID),
 					slog.String("entry_url", entry.URL),
 					slog.Int64("feed_id", feed.ID),
 					slog.String("feed_url", feed.FeedURL),
@@ -134,7 +138,6 @@ func isBlockedEntry(feed *model.Feed, entry *model.Entry) bool {
 
 	if compiledBlocklist.MatchString(entry.URL) || compiledBlocklist.MatchString(entry.Title) || compiledBlocklist.MatchString(entry.Author) || containsBlockedTag {
 		slog.Debug("Blocking entry based on rule",
-			slog.Int64("entry_id", entry.ID),
 			slog.String("entry_url", entry.URL),
 			slog.Int64("feed_id", feed.ID),
 			slog.String("feed_url", feed.FeedURL),
@@ -165,7 +168,6 @@ func isAllowedEntry(feed *model.Feed, entry *model.Entry) bool {
 
 	if compiledKeeplist.MatchString(entry.URL) || compiledKeeplist.MatchString(entry.Title) || compiledKeeplist.MatchString(entry.Author) || containsAllowedTag {
 		slog.Debug("Allow entry based on rule",
-			slog.Int64("entry_id", entry.ID),
 			slog.String("entry_url", entry.URL),
 			slog.Int64("feed_id", feed.ID),
 			slog.String("feed_url", feed.FeedURL),
@@ -230,7 +232,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
 			re := regexp.MustCompile(parts[1])
 			url = re.ReplaceAllString(entry.URL, parts[2])
 			slog.Debug("Rewriting entry URL",
-				slog.Int64("entry_id", entry.ID),
 				slog.String("original_entry_url", entry.URL),
 				slog.String("rewritten_entry_url", url),
 				slog.Int64("feed_id", feed.ID),
@@ -238,7 +239,6 @@ func getUrlFromEntry(feed *model.Feed, entry *model.Entry) string {
 			)
 		} else {
 			slog.Debug("Cannot find search and replace terms for replace rule",
-				slog.Int64("entry_id", entry.ID),
 				slog.String("original_entry_url", entry.URL),
 				slog.String("rewritten_entry_url", url),
 				slog.Int64("feed_id", feed.ID),

internal/storage/entry.go

@@ -225,6 +225,12 @@ func (s *Storage) entryExists(tx *sql.Tx, entry *model.Entry) (bool, error) {
 	return result, nil
 }
 
+func (s *Storage) IsNewEntry(feedID int64, entryHash string) bool {
+	var result bool
+	s.db.QueryRow(`SELECT true FROM entries WHERE feed_id=$1 AND hash=$2`, feedID, entryHash).Scan(&result)
+	return !result
+}
+
 // GetReadTime fetches the read time of an entry based on its hash, and the feed id and user id from the feed.
 // It's intended to be used on entries objects created by parsing a feed as they don't contain much information.
 // The feed param helps to scope the search to a specific user and feed in order to avoid hash clashes.
@@ -575,14 +581,6 @@ func (s *Storage) MarkCategoryAsRead(userID, categoryID int64, before time.Time)
 	return nil
 }
 
-// EntryURLExists returns true if an entry with this URL already exists.
-func (s *Storage) EntryURLExists(feedID int64, entryURL string) bool {
-	var result bool
-	query := `SELECT true FROM entries WHERE feed_id=$1 AND url=$2`
-	s.db.QueryRow(query, feedID, entryURL).Scan(&result)
-	return result
-}
-
 // EntryShareCode returns the share code of the provided entry.
 // It generates a new one if not already defined.
 func (s *Storage) EntryShareCode(userID int64, entryID int64) (shareCode string, err error) {