content_rewrite.go 7.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219
  1. // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
  2. // SPDX-License-Identifier: Apache-2.0
  3. package rewrite // import "miniflux.app/v2/internal/reader/rewrite"
  4. import (
  5. "log/slog"
  6. "strconv"
  7. "strings"
  8. "text/scanner"
  9. "miniflux.app/v2/internal/model"
  10. "miniflux.app/v2/internal/urllib"
  11. )
  12. type rule struct {
  13. name string
  14. args []string
  15. }
  16. // ruleSpec describes a rewrite rule: the minimum number of arguments it requires
  17. // (used for save-time validation), whether its first argument is a regular
  18. // expression or a CSS selector (so it can be compile-checked at save time), and
  19. // the handler that applies it.
  20. type ruleSpec struct {
  21. minArgs int
  22. regexArg bool
  23. selectorArg bool
  24. apply func(entryURL string, entry *model.Entry, args []string)
  25. }
  26. // convertTextLinksRule is shared by the "convert_text_link" and
  27. // "convert_text_links" aliases.
  28. var convertTextLinksRule = ruleSpec{apply: func(_ string, entry *model.Entry, _ []string) {
  29. entry.Content = replaceTextLinks(entry.Content)
  30. }}
  31. // knownRules is the single source of truth for rewrite rules: it maps each valid
  32. // rule name to its minimum argument count and its handler. Validation and
  33. // dispatch both read this table, so the two can never drift.
  34. var knownRules = map[string]ruleSpec{
  35. "add_image_title": {apply: func(_ string, entry *model.Entry, _ []string) {
  36. entry.Content = addImageTitle(entry.Content)
  37. }},
  38. "add_mailto_subject": {apply: func(_ string, entry *model.Entry, _ []string) {
  39. entry.Content = addMailtoSubject(entry.Content)
  40. }},
  41. "add_dynamic_image": {apply: func(_ string, entry *model.Entry, _ []string) {
  42. entry.Content = addDynamicImage(entry.Content)
  43. }},
  44. "add_dynamic_iframe": {apply: func(_ string, entry *model.Entry, _ []string) {
  45. entry.Content = addDynamicIframe(entry.Content)
  46. }},
  47. "add_youtube_video": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  48. entry.Content = addYoutubeVideoRewriteRule(entryURL, entry.Content)
  49. }},
  50. "add_invidious_video": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  51. entry.Content = addInvidiousVideo(entryURL, entry.Content)
  52. }},
  53. "add_youtube_video_using_invidious_player": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  54. entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content)
  55. }},
  56. "add_youtube_video_from_id": {apply: func(_ string, entry *model.Entry, _ []string) {
  57. entry.Content = addYoutubeVideoFromId(entry.Content)
  58. }},
  59. "add_pdf_download_link": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  60. entry.Content = addPDFLink(entryURL, entry.Content)
  61. }},
  62. "nl2br": {apply: func(_ string, entry *model.Entry, _ []string) {
  63. entry.Content = strings.ReplaceAll(entry.Content, "\n", "<br>")
  64. }},
  65. "convert_text_link": convertTextLinksRule,
  66. "convert_text_links": convertTextLinksRule,
  67. "fix_medium_images": {apply: func(_ string, entry *model.Entry, _ []string) {
  68. entry.Content = fixMediumImages(entry.Content)
  69. }},
  70. "use_noscript_figure_images": {apply: func(_ string, entry *model.Entry, _ []string) {
  71. entry.Content = useNoScriptImages(entry.Content)
  72. }},
  73. "replace": {minArgs: 2, regexArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  74. // Format: replace("search-term"|"replace-term")
  75. entry.Content = replaceCustom(entry.Content, args[0], args[1])
  76. }},
  77. "replace_title": {minArgs: 2, regexArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  78. // Format: replace_title("search-term"|"replace-term")
  79. entry.Title = replaceCustom(entry.Title, args[0], args[1])
  80. }},
  81. "remove": {minArgs: 1, selectorArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  82. // Format: remove("#selector > .element, .another")
  83. entry.Content = removeCustom(entry.Content, args[0])
  84. }},
  85. "add_enclosure_links": {apply: func(_ string, entry *model.Entry, _ []string) {
  86. entry.Content = addEnclosureLinks(entry)
  87. }},
  88. "add_castopod_episode": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  89. entry.Content = addCastopodEpisode(entryURL, entry.Content)
  90. }},
  91. "base64_decode": {selectorArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  92. selector := "body"
  93. if len(args) >= 1 {
  94. selector = args[0]
  95. }
  96. entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content)
  97. }},
  98. "add_hn_links_using_hack": {apply: func(_ string, entry *model.Entry, _ []string) {
  99. entry.Content = addHackerNewsLinksUsing(entry.Content, "hack")
  100. }},
  101. "add_hn_links_using_opener": {apply: func(_ string, entry *model.Entry, _ []string) {
  102. entry.Content = addHackerNewsLinksUsing(entry.Content, "opener")
  103. }},
  104. "remove_tables": {apply: func(_ string, entry *model.Entry, _ []string) {
  105. entry.Content = removeTables(entry.Content)
  106. }},
  107. "remove_clickbait": {apply: func(_ string, entry *model.Entry, _ []string) {
  108. entry.Title = titlelize(entry.Title)
  109. }},
  110. "fix_ghost_cards": {apply: func(_ string, entry *model.Entry, _ []string) {
  111. entry.Content = fixGhostCards(entry.Content)
  112. }},
  113. "remove_img_blur_params": {apply: func(_ string, entry *model.Entry, _ []string) {
  114. entry.Content = removeImgBlurParams(entry.Content)
  115. }},
  116. }
  117. // applyRule applies a single rewrite rule to the entry. Unknown names and rules
  118. // with too few arguments are skipped: malformed rules are rejected at save time,
  119. // but this path also runs predefined and previously-stored rules, which are
  120. // warn-only for backward compatibility.
  121. func (rule rule) applyRule(entryURL string, entry *model.Entry) {
  122. spec, ok := knownRules[rule.name]
  123. if !ok {
  124. return
  125. }
  126. if len(rule.args) < spec.minArgs {
  127. slog.Warn("Not enough arguments for rewrite rule",
  128. slog.Any("rule", rule),
  129. slog.String("entry_url", entryURL),
  130. )
  131. return
  132. }
  133. spec.apply(entryURL, entry, rule.args)
  134. }
  135. func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string) {
  136. rulesList := getPredefinedRewriteRules(entry.URL)
  137. if customRewriteRules != "" {
  138. rulesList = customRewriteRules
  139. }
  140. rules, errs := parseRules(rulesList)
  141. for _, e := range errs {
  142. slog.Warn("Malformed rewrite rule ignored",
  143. slog.String("entry_url", entry.URL),
  144. slog.String("rule", e.Rule),
  145. slog.String("token", e.Token),
  146. slog.Int("column", e.Pos.Column),
  147. slog.String("detail", e.Message),
  148. )
  149. }
  150. rules = append(rules, rule{name: "add_pdf_download_link"})
  151. slog.Debug("Rewrite rules applied",
  152. slog.Any("rules", rules),
  153. slog.String("entry_url", entry.URL),
  154. )
  155. for _, rule := range rules {
  156. rule.applyRule(entry.URL, entry)
  157. }
  158. }
  159. func parseRules(rulesText string) (rules []rule, errs []RuleError) {
  160. scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings}
  161. scan.Init(strings.NewReader(rulesText))
  162. scan.Error = func(s *scanner.Scanner, msg string) {
  163. errs = append(errs, RuleError{Pos: s.Pos(), Kind: RuleErrLexical, Message: msg})
  164. }
  165. for {
  166. switch scan.Scan() {
  167. case scanner.Ident:
  168. rules = append(rules, rule{name: scan.TokenText()})
  169. case scanner.String:
  170. if l := len(rules) - 1; l >= 0 {
  171. text, err := strconv.Unquote(scan.TokenText())
  172. if err != nil {
  173. errs = append(errs, RuleError{
  174. Pos: scan.Position,
  175. Kind: RuleErrUnquote,
  176. Rule: rules[l].name,
  177. Token: scan.TokenText(),
  178. Message: err.Error(),
  179. })
  180. continue
  181. }
  182. rules[l].args = append(rules[l].args, text)
  183. } else {
  184. errs = append(errs, RuleError{
  185. Pos: scan.Position,
  186. Kind: RuleErrOrphanArg,
  187. Token: scan.TokenText(),
  188. Message: "quoted string before any rule name",
  189. })
  190. }
  191. case scanner.EOF:
  192. errs = append(errs, validateParsedRules(rules)...)
  193. return rules, errs
  194. }
  195. }
  196. }
  197. func getPredefinedRewriteRules(entryURL string) string {
  198. urlDomain := urllib.DomainWithoutWWW(entryURL)
  199. if rules, ok := predefinedRules[urlDomain]; ok {
  200. return rules
  201. }
  202. return ""
  203. }