Browse Source

perf(sanitizer): avoid Token() allocations in stripIter

Tokenizer.Token() allocates a Token struct and parses the full
attribute slice for every start tag, even though stripIter only
consumes text tokens. Switch to checking the TokenType from Next()
and calling Text() only for text tokens.

Shared by StripTags and TruncateHTML, both called per entry:

  StripTags     247KB -> 71KB/op (-71%), 12623 -> 4423 allocs (-65%)
  TruncateHTML  19KB  -> 7.9KB/op (-59%),   819 ->  297 allocs (-64%)

Both roughly 2x faster on tag-heavy input.
jvoisin 2 tháng trước cách đây
mục cha
commit
e43381d17a
1 tập tin đã thay đổi với 9 bổ sung và 4 xóa
  1. 9 4
      internal/reader/sanitizer/strip_tags.go

+ 9 - 4
internal/reader/sanitizer/strip_tags.go

@@ -33,13 +33,18 @@ func StripTags(input string) string {
 func stripIter(src io.Reader, yield func(string) bool) error {
 	tokenizer := html.NewTokenizer(src)
 
-	for tokenizer.Next() != html.ErrorToken {
-		token := tokenizer.Token()
-		if token.Type != html.TextToken {
+	for {
+		tokenType := tokenizer.Next()
+		if tokenType == html.ErrorToken {
+			break
+		}
+		if tokenType != html.TextToken {
 			continue
 		}
 
-		if !yield(token.Data) {
+		// Use Text() instead of Token() to avoid allocating a Token
+		// struct (and parsing attribute slices) for every non-text tag.
+		if !yield(string(tokenizer.Text())) {
 			break
 		}
 	}