Add global option POLLING_PARSING_ERROR_LIMIT

config/config_test.go

@@ -761,6 +761,24 @@ func TestDefautSchedulerCountBasedMinInterval(t *testing.T) {
 	}
 }
 
+func TestPollingParsingErrorLimit(t *testing.T) {
+	os.Clearenv()
+	os.Setenv("POLLING_PARSING_ERROR_LIMIT", "100")
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+
+	expected := 100
+	result := opts.PollingParsingErrorLimit()
+
+	if result != expected {
+		t.Fatalf(`Unexpected POLLING_SCHEDULER value, got %v instead of %v`, result, expected)
+	}
+}
+
 func TestOAuth2UserCreationWhenUnset(t *testing.T) {
 	os.Clearenv()
 

config/options.go

@@ -29,6 +29,7 @@ const (
 	defaultPollingScheduler                   = "round_robin"
 	defaultSchedulerEntryFrequencyMinInterval = 5
 	defaultSchedulerEntryFrequencyMaxInterval = 24 * 60
+	defaultPollingParsingErrorLimit           = 3
 	defaultRunMigrations                      = false
 	defaultDatabaseURL                        = "user=postgres password=postgres dbname=miniflux2 sslmode=disable"
 	defaultDatabaseMaxConns                   = 20
@@ -103,6 +104,7 @@ type Options struct {
 	pollingScheduler                   string
 	schedulerEntryFrequencyMinInterval int
 	schedulerEntryFrequencyMaxInterval int
+	pollingParsingErrorLimit           int
 	workerPoolSize                     int
 	createAdmin                        bool
 	adminUsername                      string
@@ -159,6 +161,7 @@ func NewOptions() *Options {
 		pollingScheduler:                   defaultPollingScheduler,
 		schedulerEntryFrequencyMinInterval: defaultSchedulerEntryFrequencyMinInterval,
 		schedulerEntryFrequencyMaxInterval: defaultSchedulerEntryFrequencyMaxInterval,
+		pollingParsingErrorLimit:           defaultPollingParsingErrorLimit,
 		workerPoolSize:                     defaultWorkerPoolSize,
 		createAdmin:                        defaultCreateAdmin,
 		proxyImages:                        defaultProxyImages,
@@ -318,6 +321,11 @@ func (o *Options) SchedulerEntryFrequencyMinInterval() int {
 	return o.schedulerEntryFrequencyMinInterval
 }
 
+// PollingParsingErrorLimit returns the limit of errors when to stop polling.
+func (o *Options) PollingParsingErrorLimit() int {
+	return o.pollingParsingErrorLimit
+}
+
 // IsOAuth2UserCreationAllowed returns true if user creation is allowed for OAuth2 users.
 func (o *Options) IsOAuth2UserCreationAllowed() bool {
 	return o.oauth2UserCreationAllowed
@@ -493,6 +501,7 @@ func (o *Options) SortedOptions() []*Option {
 		"OAUTH2_USER_CREATION":                   o.oauth2UserCreationAllowed,
 		"POCKET_CONSUMER_KEY":                    o.pocketConsumerKey,
 		"POLLING_FREQUENCY":                      o.pollingFrequency,
+		"POLLING_PARSING_ERROR_LIMIT":            o.pollingParsingErrorLimit,
 		"POLLING_SCHEDULER":                      o.pollingScheduler,
 		"PROXY_IMAGES":                           o.proxyImages,
 		"ROOT_URL":                               o.rootURL,

config/parser.go

@@ -134,6 +134,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.schedulerEntryFrequencyMaxInterval = parseInt(value, defaultSchedulerEntryFrequencyMaxInterval)
 		case "SCHEDULER_ENTRY_FREQUENCY_MIN_INTERVAL":
 			p.opts.schedulerEntryFrequencyMinInterval = parseInt(value, defaultSchedulerEntryFrequencyMinInterval)
+		case "POLLING_PARSING_ERROR_LIMIT":
+			p.opts.pollingParsingErrorLimit = parseInt(value, defaultPollingParsingErrorLimit)
 		case "PROXY_IMAGES":
 			p.opts.proxyImages = parseString(value, defaultProxyImages)
 		case "CREATE_ADMIN":

miniflux.1

@@ -146,6 +146,11 @@ Minimum interval in minutes for the entry frequency scheduler\&.
 .br
 Default is 5 minutes\&.
 .TP
+.B POLLING_PARSING_ERROR_LIMIT
+The maximum number of parsing errors that the program will try before stopping polling a feed. Once the limit is reached, the user must refresh the feed manually. Set to 0 for unlimited.
+.br
+Default is 3\&.
+.TP
 .B DATABASE_URL
 Postgresql connection parameters\&.
 .br

storage/feed.go

@@ -9,6 +9,7 @@ import (
 	"errors"
 	"fmt"
 
+	"miniflux.app/config"
 	"miniflux.app/model"
 )
 
@@ -80,9 +81,13 @@ func (s *Storage) CountFeeds(userID int64) int {
 
 // CountUserFeedsWithErrors returns the number of feeds with parsing errors that belong to the given user.
 func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
+	pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
+	if pollingParsingErrorLimit <= 0 {
+		pollingParsingErrorLimit = 1
+	}
 	query := `SELECT count(*) FROM feeds WHERE user_id=$1 AND parsing_error_count >= $2`
 	var result int
-	err := s.db.QueryRow(query, userID, maxParsingError).Scan(&result)
+	err := s.db.QueryRow(query, userID, pollingParsingErrorLimit).Scan(&result)
 	if err != nil {
 		return 0
 	}
@@ -92,9 +97,13 @@ func (s *Storage) CountUserFeedsWithErrors(userID int64) int {
 
 // CountAllFeedsWithErrors returns the number of feeds with parsing errors.
 func (s *Storage) CountAllFeedsWithErrors() int {
+	pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
+	if pollingParsingErrorLimit <= 0 {
+		pollingParsingErrorLimit = 1
+	}
 	query := `SELECT count(*) FROM feeds WHERE parsing_error_count >= $1`
 	var result int
-	err := s.db.QueryRow(query, maxParsingError).Scan(&result)
+	err := s.db.QueryRow(query, pollingParsingErrorLimit).Scan(&result)
 	if err != nil {
 		return 0
 	}

storage/job.go

@@ -7,13 +7,13 @@ package storage // import "miniflux.app/storage"
 import (
 	"fmt"
 
+	"miniflux.app/config"
 	"miniflux.app/model"
 )
 
-const maxParsingError = 3
-
 // NewBatch returns a serie of jobs.
 func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
+	pollingParsingErrorLimit := config.Opts.PollingParsingErrorLimit()
 	query := `
 		SELECT
 			id,
@@ -21,10 +21,11 @@ func (s *Storage) NewBatch(batchSize int) (jobs model.JobList, err error) {
 		FROM
 			feeds
 		WHERE
-			parsing_error_count < $1 AND disabled is false AND next_check_at < now()
-		ORDER BY next_check_at ASC LIMIT %d
+			disabled is false AND next_check_at < now() AND 
+			CASE WHEN $1 > 0 THEN parsing_error_count < $1 ELSE parsing_error_count >= 0 END
+		ORDER BY next_check_at ASC LIMIT $2
 	`
-	return s.fetchBatchRows(fmt.Sprintf(query, batchSize), maxParsingError)
+	return s.fetchBatchRows(query, pollingParsingErrorLimit, batchSize)
 }
 
 // NewUserBatch returns a serie of jobs but only for a given user.