|
|
@@ -7,33 +7,37 @@ package readingtime
|
|
|
import (
|
|
|
"math"
|
|
|
"strings"
|
|
|
+ "unicode"
|
|
|
"unicode/utf8"
|
|
|
|
|
|
"miniflux.app/v2/internal/reader/sanitizer"
|
|
|
-
|
|
|
- "github.com/abadojack/whatlanggo"
|
|
|
)
|
|
|
|
|
|
// EstimateReadingTime returns the estimated reading time of an article in minute.
|
|
|
func EstimateReadingTime(content string, defaultReadingSpeed, cjkReadingSpeed int) int {
|
|
|
sanitizedContent := sanitizer.StripTags(content)
|
|
|
+ truncationPoint := min(len(sanitizedContent), 50)
|
|
|
|
|
|
- // Litterature on language detection says that around 100 signes is enough, we're safe here.
|
|
|
- truncationPoint := min(len(sanitizedContent), 250)
|
|
|
-
|
|
|
- // We're only interested in identifying Japanse/Chinese/Korean
|
|
|
- options := whatlanggo.Options{
|
|
|
- Whitelist: map[whatlanggo.Lang]bool{
|
|
|
- whatlanggo.Jpn: true,
|
|
|
- whatlanggo.Cmn: true,
|
|
|
- whatlanggo.Kor: true,
|
|
|
- },
|
|
|
+ if isCJK(sanitizedContent[:truncationPoint]) {
|
|
|
+ return int(math.Ceil(float64(utf8.RuneCountInString(sanitizedContent)) / float64(cjkReadingSpeed)))
|
|
|
}
|
|
|
- langInfo := whatlanggo.DetectWithOptions(sanitizedContent[:truncationPoint], options)
|
|
|
+ return int(math.Ceil(float64(len(strings.Fields(sanitizedContent))) / float64(defaultReadingSpeed)))
|
|
|
+}
|
|
|
|
|
|
- if langInfo.IsReliable() {
|
|
|
- return int(math.Ceil(float64(utf8.RuneCountInString(sanitizedContent)) / float64(cjkReadingSpeed)))
|
|
|
+func isCJK(text string) bool {
|
|
|
+ totalCJK := 0
|
|
|
+
|
|
|
+ for _, r := range text[:min(len(text), 50)] {
|
|
|
+ if unicode.Is(unicode.Han, r) ||
|
|
|
+ unicode.Is(unicode.Hangul, r) ||
|
|
|
+ unicode.Is(unicode.Hiragana, r) ||
|
|
|
+ unicode.Is(unicode.Katakana, r) ||
|
|
|
+ unicode.Is(unicode.Yi, r) ||
|
|
|
+ unicode.Is(unicode.Bopomofo, r) {
|
|
|
+ totalCJK++
|
|
|
+ }
|
|
|
}
|
|
|
- nbOfWords := len(strings.Fields(sanitizedContent))
|
|
|
- return int(math.Ceil(float64(nbOfWords) / float64(defaultReadingSpeed)))
|
|
|
+
|
|
|
+ // if at least 50% of the text is CJK, odds are that the text is in CJK.
|
|
|
+ return totalCJK > len(text)/50
|
|
|
}
|
Julien Voisin