Add the possibility to add rules during feed creation

api/feed.go

@@ -49,6 +49,8 @@ func (h *handler) createFeed(w http.ResponseWriter, r *http.Request) {
 		feedInfo.UserAgent,
 		feedInfo.Username,
 		feedInfo.Password,
+		feedInfo.ScraperRules,
+		feedInfo.RewriteRules,
 	)
 	if err != nil {
 		json.ServerError(w, r, err)

api/payload.go

@@ -24,12 +24,14 @@ type entriesResponse struct {
 }
 
 type feedCreation struct {
-	FeedURL    string `json:"feed_url"`
-	CategoryID int64  `json:"category_id"`
-	UserAgent  string `json:"user_agent"`
-	Username   string `json:"username"`
-	Password   string `json:"password"`
-	Crawler    bool   `json:"crawler"`
+	FeedURL      string `json:"feed_url"`
+	CategoryID   int64  `json:"category_id"`
+	UserAgent    string `json:"user_agent"`
+	Username     string `json:"username"`
+	Password     string `json:"password"`
+	Crawler      bool   `json:"crawler"`
+	ScraperRules string `json:"scraper_rules"`
+	RewriteRules string `json:"rewrite_rules"`
 }
 
 type subscriptionDiscovery struct {

model/feed.go

@@ -61,11 +61,13 @@ func (f *Feed) WithCategoryID(categoryID int64) {
 }
 
 // WithBrowsingParameters defines browsing parameters.
-func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password string) {
+func (f *Feed) WithBrowsingParameters(crawler bool, userAgent, username, password, scraperRules, rewriteRules string) {
 	f.Crawler = crawler
 	f.UserAgent = userAgent
 	f.Username = username
 	f.Password = password
+	f.ScraperRules = scraperRules
+	f.RewriteRules = rewriteRules
 }
 
 // WithError adds a new error message and increment the error counter.

model/feed_test.go

@@ -44,7 +44,7 @@ func TestFeedCategorySetter(t *testing.T) {
 
 func TestFeedBrowsingParams(t *testing.T) {
 	feed := &Feed{}
-	feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret")
+	feed.WithBrowsingParameters(true, "Custom User Agent", "Username", "Secret", "Some Rule", "Another Rule")
 
 	if !feed.Crawler {
 		t.Error(`The crawler must be activated`)
@@ -61,6 +61,14 @@ func TestFeedBrowsingParams(t *testing.T) {
 	if feed.Password != "Secret" {
 		t.Error(`The password must be set`)
 	}
+
+	if feed.ScraperRules != "Some Rule" {
+		t.Errorf(`The scraper rules must be set`)
+	}
+
+	if feed.RewriteRules != "Another Rule" {
+		t.Errorf(`The rewrite rules must be set`)
+	}
 }
 
 func TestFeedErrorCounter(t *testing.T) {

reader/feed/handler.go

@@ -29,11 +29,11 @@ var (
 
 // Handler contains all the logic to create and refresh feeds.
 type Handler struct {
-	store      *storage.Storage
+	store *storage.Storage
 }
 
 // CreateFeed fetch, parse and store a new feed.
-func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password string) (*model.Feed, error) {
+func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool, userAgent, username, password, scraperRules, rewriteRules string) (*model.Feed, error) {
 	defer timer.ExecutionTime(time.Now(), fmt.Sprintf("[Handler:CreateFeed] feedUrl=%s", url))
 
 	if !h.store.CategoryExists(userID, categoryID) {
@@ -59,7 +59,7 @@ func (h *Handler) CreateFeed(userID, categoryID int64, url string, crawler bool,
 
 	subscription.UserID = userID
 	subscription.WithCategoryID(categoryID)
-	subscription.WithBrowsingParameters(crawler, userAgent, username, password)
+	subscription.WithBrowsingParameters(crawler, userAgent, username, password, scraperRules, rewriteRules)
 	subscription.WithClientResponse(response)
 	subscription.CheckedNow()
 
@@ -160,4 +160,4 @@ func checkFeedIcon(store *storage.Storage, feedID int64, websiteURL string) {
 			}
 		}
 	}
-}
+}

reader/rewrite/rewriter.go

@@ -37,7 +37,7 @@ func Rewriter(entryURL, entryContent, customRewriteRules string) string {
 			entryContent = addPDFLink(entryURL, entryContent)
 		case "nl2br":
 			entryContent = replaceLineFeeds(entryContent)
-		case "convert_text_link":
+		case "convert_text_link", "convert_text_links":
 			entryContent = replaceTextLinks(entryContent)
 		}
 	}

storage/feed.go

@@ -346,10 +346,12 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 			user_agent,
 			username,
 			password,
-			disabled
+			disabled,
+			scraper_rules,
+			rewrite_rules
 		)
 		VALUES
-			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
+			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
 		RETURNING
 			id
 	`
@@ -367,6 +369,8 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 		feed.Username,
 		feed.Password,
 		feed.Disabled,
+		feed.ScraperRules,
+		feed.RewriteRules,
 	).Scan(&feed.ID)
 	if err != nil {
 		return fmt.Errorf(`store: unable to create feed %q: %v`, feed.FeedURL, err)

template/html/add_subscription.html

@@ -56,6 +56,12 @@
                     - Using a different input name doesn't change anything
                 -->
                 <input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
+
+                <label for="form-scraper-rules">{{ t "form.feed.label.scraper_rules" }}</label>
+                <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
+
+                <label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
+                <input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
             </div>
         </details>
 

template/html/choose_subscription.html

@@ -22,6 +22,8 @@
     <input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
     <input type="hidden" name="feed_username" value="{{ .form.Username }}">
     <input type="hidden" name="feed_password" value="{{ .form.Password }}">
+    <input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
+    <input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
     {{ if .form.Crawler }}
         <input type="hidden" name="crawler" value="1">
     {{ end }}

template/views.go

@@ -87,6 +87,12 @@ var templateViewsMap = map[string]string{
                     - Using a different input name doesn't change anything
                 -->
                 <input type="text" name="feed_password" id="form-feed-password" value="{{ .form.Password }}">
+
+                <label for="form-scraper-rules">{{ t "form.feed.label.scraper_rules" }}</label>
+                <input type="text" name="scraper_rules" id="form-scraper-rules" value="{{ .form.ScraperRules }}">
+
+                <label for="form-rewrite-rules">{{ t "form.feed.label.rewrite_rules" }}</label>
+                <input type="text" name="rewrite_rules" id="form-rewrite-rules" value="{{ .form.RewriteRules }}">
             </div>
         </details>
 
@@ -311,6 +317,8 @@ var templateViewsMap = map[string]string{
     <input type="hidden" name="user_agent" value="{{ .form.UserAgent }}">
     <input type="hidden" name="feed_username" value="{{ .form.Username }}">
     <input type="hidden" name="feed_password" value="{{ .form.Password }}">
+    <input type="hidden" name="scraper_rules" value="{{ .form.ScraperRules }}">
+    <input type="hidden" name="rewrite_rules" value="{{ .form.RewriteRules }}">
     {{ if .form.Crawler }}
         <input type="hidden" name="crawler" value="1">
     {{ end }}
@@ -1343,12 +1351,12 @@ var templateViewsMap = map[string]string{
 
 var templateViewsMapChecksums = map[string]string{
 	"about":               "4035658497363d7af7f79be83190404eb21ec633fe8ec636bdfc219d9fc78cfc",
-	"add_subscription":    "a0f1d2bc02b6adc83dbeae593f74d9b936102cd6dd73302cdbec2137cafdcdd9",
+	"add_subscription":    "5bae8c60989593257f515f9c73b35c854a94bee4b2fb08ad38aa93b17be9c1b5",
 	"bookmark_entries":    "65588da78665699dd3f287f68325e9777d511f1a57fee4131a5bb6d00bb68df8",
 	"categories":          "2c5dd0ed6355bd5acc393bbf6117d20458b5581aab82036008324f6bbbe2af75",
 	"category_entries":    "dee7b9cd60c6c46f01dd4289940679df31c1fce28ce4aa7249fa459023e1eeb4",
 	"category_feeds":      "527c2ffbc4fcec775071424ba1022ae003525dba53a28cc41f48fb7b30aa984b",
-	"choose_subscription": "33c04843d7c1b608d034e605e52681822fc6d79bc6b900c04915dd9ebae584e2",
+	"choose_subscription": "5f21556e6cecfd64b1cff30e22ef313d2f09698cebb03f711cbac8ec7f2c1d04",
 	"create_category":     "6b22b5ce51abf4e225e23a79f81be09a7fb90acb265e93a8faf9446dff74018d",
 	"create_user":         "9b73a55233615e461d1f07d99ad1d4d3b54532588ab960097ba3e090c85aaf3a",
 	"edit_category":       "b1c0b38f1b714c5d884edcd61e5b5295a5f1c8b71c469b35391e4dcc97cc6d36",

ui/form/subscription.go

@@ -13,12 +13,14 @@ import (
 
 // SubscriptionForm represents the subscription form.
 type SubscriptionForm struct {
-	URL        string
-	CategoryID int64
-	Crawler    bool
-	UserAgent  string
-	Username   string
-	Password   string
+	URL          string
+	CategoryID   int64
+	Crawler      bool
+	UserAgent    string
+	Username     string
+	Password     string
+	ScraperRules string
+	RewriteRules string
 }
 
 // Validate makes sure the form values are valid.
@@ -38,11 +40,13 @@ func NewSubscriptionForm(r *http.Request) *SubscriptionForm {
 	}
 
 	return &SubscriptionForm{
-		URL:        r.FormValue("url"),
-		Crawler:    r.FormValue("crawler") == "1",
-		CategoryID: int64(categoryID),
-		UserAgent:  r.FormValue("user_agent"),
-		Username:   r.FormValue("feed_username"),
-		Password:   r.FormValue("feed_password"),
+		URL:          r.FormValue("url"),
+		Crawler:      r.FormValue("crawler") == "1",
+		CategoryID:   int64(categoryID),
+		UserAgent:    r.FormValue("user_agent"),
+		Username:     r.FormValue("feed_username"),
+		Password:     r.FormValue("feed_password"),
+		ScraperRules: r.FormValue("scraper_rules"),
+		RewriteRules: r.FormValue("rewrite_rules"),
 	}
 }

ui/subscription_choose.go

@@ -8,8 +8,8 @@ import (
 	"net/http"
 
 	"miniflux.app/http/client"
-	"miniflux.app/http/response/html"
 	"miniflux.app/http/request"
+	"miniflux.app/http/response/html"
 	"miniflux.app/http/route"
 	"miniflux.app/ui/form"
 	"miniflux.app/ui/session"
@@ -55,6 +55,8 @@ func (h *handler) showChooseSubscriptionPage(w http.ResponseWriter, r *http.Requ
 		subscriptionForm.UserAgent,
 		subscriptionForm.Username,
 		subscriptionForm.Password,
+		subscriptionForm.ScraperRules,
+		subscriptionForm.RewriteRules,
 	)
 	if err != nil {
 		view.Set("form", subscriptionForm)

ui/subscription_submit.go

@@ -2,14 +2,14 @@
 // Use of this source code is governed by the Apache 2.0
 // license that can be found in the LICENSE file.
 
-package ui  // import "miniflux.app/ui"
+package ui // import "miniflux.app/ui"
 
 import (
 	"net/http"
 
 	"miniflux.app/http/client"
-	"miniflux.app/http/response/html"
 	"miniflux.app/http/request"
+	"miniflux.app/http/response/html"
 	"miniflux.app/http/route"
 	"miniflux.app/logger"
 	"miniflux.app/reader/subscription"
@@ -80,6 +80,8 @@ func (h *handler) submitSubscription(w http.ResponseWriter, r *http.Request) {
 			subscriptionForm.UserAgent,
 			subscriptionForm.Username,
 			subscriptionForm.Password,
+			subscriptionForm.ScraperRules,
+			subscriptionForm.RewriteRules,
 		)
 		if err != nil {
 			v.Set("form", subscriptionForm)