content_rewrite.go 7.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218
  1. // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
  2. // SPDX-License-Identifier: Apache-2.0
  3. package rewrite // import "miniflux.app/v2/internal/reader/rewrite"
  4. import (
  5. "log/slog"
  6. "strconv"
  7. "strings"
  8. "text/scanner"
  9. "miniflux.app/v2/internal/model"
  10. "miniflux.app/v2/internal/urllib"
  11. )
  12. type rule struct {
  13. name string
  14. args []string
  15. }
  16. // ruleSpec describes a rewrite rule: the minimum number of arguments it requires
  17. // (used for save-time validation), whether its first argument is a regular
  18. // expression (so it can be compile-checked at save time), and the handler that
  19. // applies it.
  20. type ruleSpec struct {
  21. minArgs int
  22. regexArg bool
  23. apply func(entryURL string, entry *model.Entry, args []string)
  24. }
  25. // convertTextLinksRule is shared by the "convert_text_link" and
  26. // "convert_text_links" aliases.
  27. var convertTextLinksRule = ruleSpec{apply: func(_ string, entry *model.Entry, _ []string) {
  28. entry.Content = replaceTextLinks(entry.Content)
  29. }}
  30. // knownRules is the single source of truth for rewrite rules: it maps each valid
  31. // rule name to its minimum argument count and its handler. Validation and
  32. // dispatch both read this table, so the two can never drift.
  33. var knownRules = map[string]ruleSpec{
  34. "add_image_title": {apply: func(_ string, entry *model.Entry, _ []string) {
  35. entry.Content = addImageTitle(entry.Content)
  36. }},
  37. "add_mailto_subject": {apply: func(_ string, entry *model.Entry, _ []string) {
  38. entry.Content = addMailtoSubject(entry.Content)
  39. }},
  40. "add_dynamic_image": {apply: func(_ string, entry *model.Entry, _ []string) {
  41. entry.Content = addDynamicImage(entry.Content)
  42. }},
  43. "add_dynamic_iframe": {apply: func(_ string, entry *model.Entry, _ []string) {
  44. entry.Content = addDynamicIframe(entry.Content)
  45. }},
  46. "add_youtube_video": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  47. entry.Content = addYoutubeVideoRewriteRule(entryURL, entry.Content)
  48. }},
  49. "add_invidious_video": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  50. entry.Content = addInvidiousVideo(entryURL, entry.Content)
  51. }},
  52. "add_youtube_video_using_invidious_player": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  53. entry.Content = addYoutubeVideoUsingInvidiousPlayer(entryURL, entry.Content)
  54. }},
  55. "add_youtube_video_from_id": {apply: func(_ string, entry *model.Entry, _ []string) {
  56. entry.Content = addYoutubeVideoFromId(entry.Content)
  57. }},
  58. "add_pdf_download_link": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  59. entry.Content = addPDFLink(entryURL, entry.Content)
  60. }},
  61. "nl2br": {apply: func(_ string, entry *model.Entry, _ []string) {
  62. entry.Content = strings.ReplaceAll(entry.Content, "\n", "<br>")
  63. }},
  64. "convert_text_link": convertTextLinksRule,
  65. "convert_text_links": convertTextLinksRule,
  66. "fix_medium_images": {apply: func(_ string, entry *model.Entry, _ []string) {
  67. entry.Content = fixMediumImages(entry.Content)
  68. }},
  69. "use_noscript_figure_images": {apply: func(_ string, entry *model.Entry, _ []string) {
  70. entry.Content = useNoScriptImages(entry.Content)
  71. }},
  72. "replace": {minArgs: 2, regexArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  73. // Format: replace("search-term"|"replace-term")
  74. entry.Content = replaceCustom(entry.Content, args[0], args[1])
  75. }},
  76. "replace_title": {minArgs: 2, regexArg: true, apply: func(_ string, entry *model.Entry, args []string) {
  77. // Format: replace_title("search-term"|"replace-term")
  78. entry.Title = replaceCustom(entry.Title, args[0], args[1])
  79. }},
  80. "remove": {minArgs: 1, apply: func(_ string, entry *model.Entry, args []string) {
  81. // Format: remove("#selector > .element, .another")
  82. entry.Content = removeCustom(entry.Content, args[0])
  83. }},
  84. "add_enclosure_links": {apply: func(_ string, entry *model.Entry, _ []string) {
  85. entry.Content = addEnclosureLinks(entry)
  86. }},
  87. "add_castopod_episode": {apply: func(entryURL string, entry *model.Entry, _ []string) {
  88. entry.Content = addCastopodEpisode(entryURL, entry.Content)
  89. }},
  90. "base64_decode": {apply: func(_ string, entry *model.Entry, args []string) {
  91. selector := "body"
  92. if len(args) >= 1 {
  93. selector = args[0]
  94. }
  95. entry.Content = applyFuncOnTextContent(entry.Content, selector, decodeBase64Content)
  96. }},
  97. "add_hn_links_using_hack": {apply: func(_ string, entry *model.Entry, _ []string) {
  98. entry.Content = addHackerNewsLinksUsing(entry.Content, "hack")
  99. }},
  100. "add_hn_links_using_opener": {apply: func(_ string, entry *model.Entry, _ []string) {
  101. entry.Content = addHackerNewsLinksUsing(entry.Content, "opener")
  102. }},
  103. "remove_tables": {apply: func(_ string, entry *model.Entry, _ []string) {
  104. entry.Content = removeTables(entry.Content)
  105. }},
  106. "remove_clickbait": {apply: func(_ string, entry *model.Entry, _ []string) {
  107. entry.Title = titlelize(entry.Title)
  108. }},
  109. "fix_ghost_cards": {apply: func(_ string, entry *model.Entry, _ []string) {
  110. entry.Content = fixGhostCards(entry.Content)
  111. }},
  112. "remove_img_blur_params": {apply: func(_ string, entry *model.Entry, _ []string) {
  113. entry.Content = removeImgBlurParams(entry.Content)
  114. }},
  115. }
  116. // applyRule applies a single rewrite rule to the entry. Unknown names and rules
  117. // with too few arguments are skipped: malformed rules are rejected at save time,
  118. // but this path also runs predefined and previously-stored rules, which are
  119. // warn-only for backward compatibility.
  120. func (rule rule) applyRule(entryURL string, entry *model.Entry) {
  121. spec, ok := knownRules[rule.name]
  122. if !ok {
  123. return
  124. }
  125. if len(rule.args) < spec.minArgs {
  126. slog.Warn("Not enough arguments for rewrite rule",
  127. slog.Any("rule", rule),
  128. slog.String("entry_url", entryURL),
  129. )
  130. return
  131. }
  132. spec.apply(entryURL, entry, rule.args)
  133. }
  134. func ApplyContentRewriteRules(entry *model.Entry, customRewriteRules string) {
  135. rulesList := getPredefinedRewriteRules(entry.URL)
  136. if customRewriteRules != "" {
  137. rulesList = customRewriteRules
  138. }
  139. rules, errs := parseRules(rulesList)
  140. for _, e := range errs {
  141. slog.Warn("Malformed rewrite rule ignored",
  142. slog.String("entry_url", entry.URL),
  143. slog.String("rule", e.Rule),
  144. slog.String("token", e.Token),
  145. slog.Int("column", e.Pos.Column),
  146. slog.String("detail", e.Message),
  147. )
  148. }
  149. rules = append(rules, rule{name: "add_pdf_download_link"})
  150. slog.Debug("Rewrite rules applied",
  151. slog.Any("rules", rules),
  152. slog.String("entry_url", entry.URL),
  153. )
  154. for _, rule := range rules {
  155. rule.applyRule(entry.URL, entry)
  156. }
  157. }
  158. func parseRules(rulesText string) (rules []rule, errs []RuleError) {
  159. scan := scanner.Scanner{Mode: scanner.ScanIdents | scanner.ScanStrings}
  160. scan.Init(strings.NewReader(rulesText))
  161. scan.Error = func(s *scanner.Scanner, msg string) {
  162. errs = append(errs, RuleError{Pos: s.Pos(), Kind: RuleErrLexical, Message: msg})
  163. }
  164. for {
  165. switch scan.Scan() {
  166. case scanner.Ident:
  167. rules = append(rules, rule{name: scan.TokenText()})
  168. case scanner.String:
  169. if l := len(rules) - 1; l >= 0 {
  170. text, err := strconv.Unquote(scan.TokenText())
  171. if err != nil {
  172. errs = append(errs, RuleError{
  173. Pos: scan.Position,
  174. Kind: RuleErrUnquote,
  175. Rule: rules[l].name,
  176. Token: scan.TokenText(),
  177. Message: err.Error(),
  178. })
  179. continue
  180. }
  181. rules[l].args = append(rules[l].args, text)
  182. } else {
  183. errs = append(errs, RuleError{
  184. Pos: scan.Position,
  185. Kind: RuleErrOrphanArg,
  186. Token: scan.TokenText(),
  187. Message: "quoted string before any rule name",
  188. })
  189. }
  190. case scanner.EOF:
  191. errs = append(errs, validateParsedRules(rules)...)
  192. return rules, errs
  193. }
  194. }
  195. }
  196. func getPredefinedRewriteRules(entryURL string) string {
  197. urlDomain := urllib.DomainWithoutWWW(entryURL)
  198. if rules, ok := predefinedRules[urlDomain]; ok {
  199. return rules
  200. }
  201. return ""
  202. }