Sfoglia il codice sorgente

feat(readability): avoid removing elements with `content` class

Frédéric Guillot 9 mesi fa
parent
commit
66b269e6cd
1 ha cambiato i file con 9 aggiunte e 9 eliminazioni
  1. 9 9
      internal/reader/readability/readability.go

+ 9 - 9
internal/reader/readability/readability.go

@@ -18,9 +18,9 @@ import (
 const defaultTagsToScore = "section,h2,h3,h4,h5,h6,p,td,pre,div"
 
 var (
-	strongCandidates  = [...]string{"popupbody", "-ad", "g-plus"}
-	maybeCandidate    = [...]string{"and", "article", "body", "column", "main", "shadow"}
-	unlikelyCandidate = [...]string{"banner", "breadcrumbs", "combx", "comment", "community", "cover-wrap", "disqus", "extra", "foot", "header", "legends", "menu", "modal", "related", "remark", "replies", "rss", "shoutbox", "sidebar", "skyscraper", "social", "sponsor", "supplemental", "ad-break", "agegate", "pagination", "pager", "popup", "yom-remote"}
+	strongCandidatesToRemove  = [...]string{"popupbody", "-ad", "g-plus"}
+	maybeCandidateToRemove    = [...]string{"and", "article", "body", "column", "main", "shadow", "content"}
+	unlikelyCandidateToRemove = [...]string{"banner", "breadcrumbs", "combx", "comment", "community", "cover-wrap", "disqus", "extra", "foot", "header", "legends", "menu", "modal", "related", "remark", "replies", "rss", "shoutbox", "sidebar", "skyscraper", "social", "sponsor", "supplemental", "ad-break", "agegate", "pagination", "pager", "popup", "yom-remote"}
 
 	positiveKeywords = [...]string{"article", "blog", "body", "content", "entry", "h-entry", "hentry", "main", "page", "pagination", "post", "story", "text"}
 	negativeKeywords = [...]string{"author", "banner", "byline", "com-", "combx", "comment", "contact", "dateline", "foot", "hid", "masthead", "media", "meta", "modal", "outbrain", "promo", "related", "scroll", "share", "shopping", "shoutbox", "sidebar", "skyscraper", "sponsor", "tags", "tool", "widget", "writtenby"}
@@ -185,17 +185,17 @@ func shouldRemoveCandidate(str string) bool {
 	str = strings.ToLower(str)
 
 	// Those candidates have no false-positives, no need to check against `maybeCandidate`
-	for _, strongCandidate := range strongCandidates {
-		if strings.Contains(str, strongCandidate) {
+	for _, strongCandidateToRemove := range strongCandidatesToRemove {
+		if strings.Contains(str, strongCandidateToRemove) {
 			return true
 		}
 	}
 
-	for _, unlikelyCandidate := range unlikelyCandidate {
-		if strings.Contains(str, unlikelyCandidate) {
+	for _, unlikelyCandidateToRemove := range unlikelyCandidateToRemove {
+		if strings.Contains(str, unlikelyCandidateToRemove) {
 			// Do we have a false positive?
-			for _, maybe := range maybeCandidate {
-				if strings.Contains(str, maybe) {
+			for _, maybeCandidateToRemove := range maybeCandidateToRemove {
+				if strings.Contains(str, maybeCandidateToRemove) {
 					return false
 				}
 			}