sanitizer.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641
  1. // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
  2. // SPDX-License-Identifier: Apache-2.0
  3. package sanitizer // import "miniflux.app/v2/internal/reader/sanitizer"
  4. import (
  5. "io"
  6. "net/url"
  7. "slices"
  8. "strconv"
  9. "strings"
  10. "miniflux.app/v2/internal/config"
  11. "miniflux.app/v2/internal/reader/urlcleaner"
  12. "miniflux.app/v2/internal/urllib"
  13. "golang.org/x/net/html"
  14. )
  15. var (
  16. allowedHTMLTagsAndAttributes = map[string][]string{
  17. "a": {"href", "title", "id"},
  18. "abbr": {"title"},
  19. "acronym": {"title"},
  20. "aside": {},
  21. "audio": {"src"},
  22. "blockquote": {},
  23. "b": {},
  24. "br": {},
  25. "caption": {},
  26. "cite": {},
  27. "code": {},
  28. "dd": {"id"},
  29. "del": {},
  30. "dfn": {},
  31. "dl": {"id"},
  32. "dt": {"id"},
  33. "em": {},
  34. "figcaption": {},
  35. "figure": {},
  36. "h1": {"id"},
  37. "h2": {"id"},
  38. "h3": {"id"},
  39. "h4": {"id"},
  40. "h5": {"id"},
  41. "h6": {"id"},
  42. "hr": {},
  43. "i": {},
  44. "iframe": {"width", "height", "frameborder", "src", "allowfullscreen"},
  45. "img": {"alt", "title", "src", "srcset", "sizes", "width", "height", "fetchpriority", "decoding"},
  46. "ins": {},
  47. "kbd": {},
  48. "li": {"id"},
  49. "ol": {"id"},
  50. "p": {},
  51. "picture": {},
  52. "pre": {},
  53. "q": {"cite"},
  54. "rp": {},
  55. "rt": {},
  56. "rtc": {},
  57. "ruby": {},
  58. "s": {},
  59. "small": {},
  60. "samp": {},
  61. "source": {"src", "type", "srcset", "sizes", "media"},
  62. "strong": {},
  63. "sub": {},
  64. "sup": {"id"},
  65. "table": {},
  66. "td": {"rowspan", "colspan"},
  67. "tfoot": {},
  68. "th": {"rowspan", "colspan"},
  69. "thead": {},
  70. "time": {"datetime"},
  71. "tr": {},
  72. "u": {},
  73. "ul": {"id"},
  74. "var": {},
  75. "video": {"poster", "height", "width", "src"},
  76. "wbr": {},
  77. // MathML: https://w3c.github.io/mathml-core/ and https://developer.mozilla.org/en-US/docs/Web/MathML/Reference/Element
  78. "annotation": {},
  79. "annotation-xml": {},
  80. "maction": {},
  81. "math": {"xmlns"},
  82. "merror": {},
  83. "mfrac": {},
  84. "mi": {},
  85. "mmultiscripts": {},
  86. "mn": {},
  87. "mo": {},
  88. "mover": {},
  89. "mpadded": {},
  90. "mphantom": {},
  91. "mprescripts": {},
  92. "mroot": {},
  93. "mrow": {},
  94. "ms": {},
  95. "mspace": {},
  96. "msqrt": {},
  97. "mstyle": {},
  98. "msub": {},
  99. "msubsup": {},
  100. "msup": {},
  101. "mtable": {},
  102. "mtd": {},
  103. "mtext": {},
  104. "mtr": {},
  105. "munder": {},
  106. "munderover": {},
  107. "semantics": {},
  108. }
  109. iframeAllowList = map[string]struct{}{
  110. "bandcamp.com": {},
  111. "cdn.embedly.com": {},
  112. "dailymotion.com": {},
  113. "framatube.org": {},
  114. "open.spotify.com": {},
  115. "player.bilibili.com": {},
  116. "player.twitch.tv": {},
  117. "player.vimeo.com": {},
  118. "soundcloud.com": {},
  119. "vk.com": {},
  120. "w.soundcloud.com": {},
  121. "youtube-nocookie.com": {},
  122. "youtube.com": {},
  123. }
  124. blockedResourceURLSubstrings = []string{
  125. "api.flattr.com",
  126. "www.facebook.com/sharer.php",
  127. "feeds.feedburner.com",
  128. "feedsportal.com",
  129. "linkedin.com/shareArticle",
  130. "pinterest.com/pin/create/button/",
  131. "stats.wordpress.com",
  132. "twitter.com/intent/tweet",
  133. "twitter.com/share",
  134. "x.com/intent/tweet",
  135. "x.com/share",
  136. }
  137. dataAttributeAllowedPrefixes = []string{
  138. "data:image/avif",
  139. "data:image/apng",
  140. "data:image/png",
  141. "data:image/svg",
  142. "data:image/svg+xml",
  143. "data:image/jpg",
  144. "data:image/jpeg",
  145. "data:image/gif",
  146. "data:image/webp",
  147. }
  148. )
  149. // SanitizerOptions holds options for the HTML sanitizer.
  150. type SanitizerOptions struct {
  151. OpenLinksInNewTab bool
  152. }
  153. // SanitizeHTML takes raw HTML input and removes any disallowed tags and attributes.
  154. func SanitizeHTML(baseURL, rawHTML string, sanitizerOptions *SanitizerOptions) string {
  155. var buffer strings.Builder
  156. // Educated guess about how big the sanitized HTML will be,
  157. // to reduce the amount of buffer re-allocations in this function.
  158. estimatedRatio := len(rawHTML) * 3 / 4
  159. buffer.Grow(estimatedRatio)
  160. // We need to surround `rawHTML` with body tags so that html.Parse
  161. // will consider it a valid html document.
  162. doc, err := html.Parse(io.MultiReader(
  163. strings.NewReader("<body>"),
  164. strings.NewReader(rawHTML),
  165. strings.NewReader("</body>"),
  166. ))
  167. if err != nil {
  168. return ""
  169. }
  170. /* The structure of `doc` is always:
  171. <html>
  172. <head>...</head>
  173. <body>..</body>
  174. </html>
  175. */
  176. body := doc.FirstChild.FirstChild.NextSibling
  177. // Errors are a non-issue, so they're handled in filterAndRenderHTML
  178. parsedBaseUrl, _ := url.Parse(baseURL)
  179. for c := body.FirstChild; c != nil; c = c.NextSibling {
  180. if err := filterAndRenderHTML(&buffer, c, parsedBaseUrl, sanitizerOptions); err != nil {
  181. return ""
  182. }
  183. }
  184. return buffer.String()
  185. }
  186. func findAllowedIframeSourceDomain(iframeSourceURL string) (string, bool) {
  187. parsedURL, err := url.Parse(iframeSourceURL)
  188. if err != nil {
  189. return "", false
  190. }
  191. switch parsedURL.Scheme {
  192. case "http", "https":
  193. // Only web URLs are allowed for embedded content.
  194. case "":
  195. // Preserve support for protocol-relative iframe URLs.
  196. default:
  197. return "", false
  198. }
  199. iframeSourceDomain := strings.TrimPrefix(parsedURL.Host, "www.")
  200. if iframeSourceDomain == "" {
  201. return "", false
  202. }
  203. if _, ok := iframeAllowList[iframeSourceDomain]; ok {
  204. return iframeSourceDomain, true
  205. }
  206. if ytDomain := config.Opts.YouTubeEmbedDomain(); ytDomain != "" && iframeSourceDomain == strings.TrimPrefix(ytDomain, "www.") {
  207. return iframeSourceDomain, true
  208. }
  209. if invidiousInstance := config.Opts.InvidiousInstance(); invidiousInstance != "" && iframeSourceDomain == strings.TrimPrefix(invidiousInstance, "www.") {
  210. return iframeSourceDomain, true
  211. }
  212. return "", false
  213. }
  214. func filterAndRenderHTML(buf *strings.Builder, n *html.Node, parsedBaseUrl *url.URL, sanitizerOptions *SanitizerOptions) error {
  215. if n == nil {
  216. return nil
  217. }
  218. switch n.Type {
  219. case html.TextNode:
  220. buf.WriteString(html.EscapeString(n.Data))
  221. case html.ElementNode:
  222. tag := n.Data
  223. if shouldIgnoreTag(n, tag) {
  224. return nil
  225. }
  226. _, ok := allowedHTMLTagsAndAttributes[tag]
  227. if !ok {
  228. // The tag isn't allowed, but we're still interested in its content
  229. return filterAndRenderHTMLChildren(buf, n, parsedBaseUrl, sanitizerOptions)
  230. }
  231. htmlAttributes, hasAllRequiredAttributes := sanitizeAttributes(parsedBaseUrl, tag, n.Attr, sanitizerOptions)
  232. if !hasAllRequiredAttributes {
  233. if tag == "iframe" {
  234. // A blocked iframe should not have its inner content rendered.
  235. return nil
  236. }
  237. // The tag doesn't have every required attributes but we're still interested in its content
  238. return filterAndRenderHTMLChildren(buf, n, parsedBaseUrl, sanitizerOptions)
  239. }
  240. buf.WriteByte('<')
  241. buf.WriteString(n.Data)
  242. if htmlAttributes != "" {
  243. buf.WriteByte(' ')
  244. buf.WriteString(htmlAttributes)
  245. }
  246. buf.WriteByte('>')
  247. if isSelfContainedTag(tag) {
  248. return nil
  249. }
  250. if tag != "iframe" {
  251. // iframes aren't allowed to have child nodes.
  252. filterAndRenderHTMLChildren(buf, n, parsedBaseUrl, sanitizerOptions)
  253. }
  254. buf.WriteString("</")
  255. buf.WriteString(n.Data)
  256. buf.WriteByte('>')
  257. default:
  258. }
  259. return nil
  260. }
  261. func filterAndRenderHTMLChildren(buf *strings.Builder, n *html.Node, parsedBaseUrl *url.URL, sanitizerOptions *SanitizerOptions) error {
  262. for c := n.FirstChild; c != nil; c = c.NextSibling {
  263. if err := filterAndRenderHTML(buf, c, parsedBaseUrl, sanitizerOptions); err != nil {
  264. return err
  265. }
  266. }
  267. return nil
  268. }
  269. func hasRequiredAttributes(s *mandatoryAttributesStruct, tagName string) bool {
  270. switch tagName {
  271. case "a":
  272. return s.href
  273. case "iframe":
  274. return s.src
  275. case "source", "img":
  276. return s.src || s.srcset
  277. }
  278. return true
  279. }
  280. func isBlockedResource(absoluteURL string) bool {
  281. for _, blockedURL := range blockedResourceURLSubstrings {
  282. if strings.Contains(absoluteURL, blockedURL) {
  283. return true
  284. }
  285. }
  286. return false
  287. }
  288. func isBlockedTag(tagName string) bool {
  289. switch tagName {
  290. case "noscript", "script", "style":
  291. return true
  292. }
  293. return false
  294. }
  295. func isExternalResourceAttribute(attribute string) bool {
  296. switch attribute {
  297. case "src", "href", "poster", "cite":
  298. return true
  299. default:
  300. return false
  301. }
  302. }
  303. func isHidden(n *html.Node) bool {
  304. for _, attr := range n.Attr {
  305. if attr.Key == "hidden" {
  306. return true
  307. }
  308. }
  309. return false
  310. }
  311. func isPixelTracker(tagName string, attributes []html.Attribute) bool {
  312. if tagName != "img" {
  313. return false
  314. }
  315. hasHeight := false
  316. hasWidth := false
  317. for _, attribute := range attributes {
  318. if attribute.Val == "1" || attribute.Val == "0" {
  319. switch attribute.Key {
  320. case "height":
  321. hasHeight = true
  322. case "width":
  323. hasWidth = true
  324. }
  325. }
  326. }
  327. return hasHeight && hasWidth
  328. }
  329. func isPositiveInteger(value string) bool {
  330. if value == "" {
  331. return false
  332. }
  333. if number, err := strconv.Atoi(value); err == nil {
  334. return number > 0
  335. }
  336. return false
  337. }
  338. func isSelfContainedTag(tag string) bool {
  339. switch tag {
  340. case "area", "base", "br", "col", "embed", "hr", "img", "input",
  341. "link", "meta", "param", "source", "track", "wbr":
  342. return true
  343. }
  344. return false
  345. }
  346. func isValidDataAttribute(value string) bool {
  347. for _, prefix := range dataAttributeAllowedPrefixes {
  348. if strings.HasPrefix(value, prefix) {
  349. return true
  350. }
  351. }
  352. return false
  353. }
  354. func isValidDecodingValue(value string) bool {
  355. switch value {
  356. case "sync", "async", "auto":
  357. return true
  358. }
  359. return false
  360. }
  361. func isValidFetchPriorityValue(value string) bool {
  362. switch value {
  363. case "high", "low", "auto":
  364. return true
  365. }
  366. return false
  367. }
  368. func rewriteIframeURL(link string) string {
  369. u, err := url.Parse(link)
  370. if err != nil {
  371. return link
  372. }
  373. switch strings.TrimPrefix(u.Hostname(), "www.") {
  374. case "youtube.com":
  375. if pathWithoutEmbed, ok := strings.CutPrefix(u.Path, "/embed/"); ok {
  376. if len(u.RawQuery) > 0 {
  377. return config.Opts.YouTubeEmbedUrlOverride() + pathWithoutEmbed + "?" + u.RawQuery
  378. }
  379. return config.Opts.YouTubeEmbedUrlOverride() + pathWithoutEmbed
  380. }
  381. case "player.vimeo.com":
  382. // See https://help.vimeo.com/hc/en-us/articles/12426260232977-About-Player-parameters
  383. if strings.HasPrefix(u.Path, "/video/") {
  384. if len(u.RawQuery) > 0 {
  385. return link + "&dnt=1"
  386. }
  387. return link + "?dnt=1"
  388. }
  389. }
  390. return link
  391. }
  392. type mandatoryAttributesStruct struct {
  393. href bool
  394. src bool
  395. srcset bool
  396. }
  397. func trackAttributes(s *mandatoryAttributesStruct, attributeName string) {
  398. switch attributeName {
  399. case "href":
  400. s.href = true
  401. case "src":
  402. s.src = true
  403. case "srcset":
  404. s.srcset = true
  405. }
  406. }
  407. func sanitizeAttributes(parsedBaseUrl *url.URL, tagName string, attributes []html.Attribute, sanitizerOptions *SanitizerOptions) (string, bool) {
  408. var htmlAttrs strings.Builder
  409. // Rough estimate: most attributes are short; ~24 bytes (key + ="value") is
  410. // a reasonable starting point. Avoids early grows for typical elements.
  411. htmlAttrs.Grow(len(attributes) * 24)
  412. // writeAttr appends key="value" to htmlAttrs, prefixing with a single
  413. // space when not the first written attribute. value is HTML-escaped.
  414. writeAttr := func(key, value string) {
  415. htmlAttrs.WriteByte(' ')
  416. htmlAttrs.WriteString(key)
  417. htmlAttrs.WriteString(`="`)
  418. htmlAttrs.WriteString(html.EscapeString(value))
  419. htmlAttrs.WriteByte('"')
  420. }
  421. // Keep track of mandatory attributes for some tags
  422. mandatoryAttributes := mandatoryAttributesStruct{false, false, false}
  423. var isAnchorLink bool
  424. var isYouTubeEmbed bool
  425. // We know the element is present, as the tag was validated in the caller of `sanitizeAttributes`
  426. allowedAttributes := allowedHTMLTagsAndAttributes[tagName]
  427. for _, attribute := range attributes {
  428. if !slices.Contains(allowedAttributes, attribute.Key) {
  429. continue
  430. }
  431. value := attribute.Val
  432. switch tagName {
  433. case "math":
  434. if attribute.Key == "xmlns" {
  435. value = "http://www.w3.org/1998/Math/MathML"
  436. }
  437. case "img":
  438. switch attribute.Key {
  439. case "fetchpriority":
  440. if !isValidFetchPriorityValue(value) {
  441. continue
  442. }
  443. case "decoding":
  444. if !isValidDecodingValue(value) {
  445. continue
  446. }
  447. case "width", "height":
  448. if !isPositiveInteger(value) {
  449. continue
  450. }
  451. case "srcset":
  452. value = sanitizeSrcsetAttr(parsedBaseUrl, value)
  453. if value == "" {
  454. continue
  455. }
  456. }
  457. case "source":
  458. if attribute.Key == "srcset" {
  459. value = sanitizeSrcsetAttr(parsedBaseUrl, value)
  460. if value == "" {
  461. continue
  462. }
  463. }
  464. }
  465. if isExternalResourceAttribute(attribute.Key) {
  466. switch {
  467. case tagName == "iframe":
  468. iframeSourceDomain, trustedIframeDomain := findAllowedIframeSourceDomain(attribute.Val)
  469. if !trustedIframeDomain {
  470. return "", false
  471. }
  472. value = rewriteIframeURL(attribute.Val)
  473. if iframeSourceDomain == "youtube.com" || iframeSourceDomain == "youtube-nocookie.com" {
  474. isYouTubeEmbed = true
  475. }
  476. case tagName == "img" && attribute.Key == "src" && isValidDataAttribute(attribute.Val):
  477. value = attribute.Val
  478. case tagName == "a" && attribute.Key == "href" && strings.HasPrefix(attribute.Val, "#"):
  479. value = attribute.Val
  480. isAnchorLink = true
  481. default:
  482. if isBlockedResource(value) {
  483. return "", false
  484. }
  485. var err error
  486. value, err = urllib.ResolveToAbsoluteURLWithParsedBaseURL(parsedBaseUrl, value)
  487. if err != nil {
  488. continue
  489. }
  490. if !HasValidURIScheme(value) {
  491. continue
  492. }
  493. // Skip the parse + RemoveTrackingParameters round trip when there
  494. // is no query string to clean, which is common for <img>.
  495. if strings.IndexByte(value, '?') >= 0 {
  496. parsedValueUrl, _ := url.Parse(value)
  497. // TODO use feedURL instead of baseURL twice.
  498. if cleanedURL, err := urlcleaner.RemoveTrackingParameters(parsedBaseUrl, parsedBaseUrl, parsedValueUrl); err == nil {
  499. value = cleanedURL
  500. }
  501. }
  502. }
  503. }
  504. trackAttributes(&mandatoryAttributes, attribute.Key)
  505. writeAttr(attribute.Key, value)
  506. }
  507. if !hasRequiredAttributes(&mandatoryAttributes, tagName) {
  508. return "", false
  509. }
  510. if !isAnchorLink {
  511. switch tagName {
  512. case "a":
  513. writeAttr("rel", "noopener noreferrer")
  514. writeAttr("referrerpolicy", "no-referrer")
  515. if sanitizerOptions.OpenLinksInNewTab {
  516. writeAttr("target", "_blank")
  517. }
  518. case "video", "audio":
  519. htmlAttrs.WriteString(" controls")
  520. case "iframe":
  521. writeAttr("sandbox", "allow-scripts allow-same-origin allow-popups allow-popups-to-escape-sandbox")
  522. writeAttr("loading", "lazy")
  523. // Note: the referrerpolicy seems to be required to avoid YouTube error 153 video player configuration error
  524. // See https://developers.google.com/youtube/terms/required-minimum-functionality#embedded-player-api-client-identity
  525. if isYouTubeEmbed {
  526. writeAttr("referrerpolicy", "strict-origin-when-cross-origin")
  527. }
  528. case "img":
  529. writeAttr("loading", "lazy")
  530. }
  531. }
  532. return strings.TrimLeft(htmlAttrs.String(), " "), true
  533. }
  534. func sanitizeSrcsetAttr(parsedBaseURL *url.URL, value string) string {
  535. candidates := ParseSrcSetAttribute(value)
  536. if len(candidates) == 0 {
  537. return ""
  538. }
  539. sanitizedCandidates := make([]*imageCandidate, 0, len(candidates))
  540. for _, imageCandidate := range candidates {
  541. absoluteURL, err := urllib.ResolveToAbsoluteURLWithParsedBaseURL(parsedBaseURL, imageCandidate.ImageURL)
  542. if err != nil {
  543. continue
  544. }
  545. if !HasValidURIScheme(absoluteURL) || isBlockedResource(absoluteURL) {
  546. continue
  547. }
  548. imageCandidate.ImageURL = absoluteURL
  549. sanitizedCandidates = append(sanitizedCandidates, imageCandidate)
  550. }
  551. return imageCandidates(sanitizedCandidates).String()
  552. }
  553. func shouldIgnoreTag(n *html.Node, tag string) bool {
  554. if isPixelTracker(tag, n.Attr) {
  555. return true
  556. }
  557. if isBlockedTag(tag) {
  558. return true
  559. }
  560. if isHidden(n) {
  561. return true
  562. }
  563. return false
  564. }