feat: optionally fetch watch time from YouTube API instead of website

internal/config/config_test.go

@@ -2116,6 +2116,24 @@ func TestFetchYouTubeWatchTime(t *testing.T) {
 	}
 }
 
+func TestYouTubeApiKey(t *testing.T) {
+	os.Clearenv()
+	os.Setenv("YOUTUBE_API_KEY", "AAAAAAAAAAAAAaaaaaaaaaaaaa0000000000000")
+
+	parser := NewParser()
+	opts, err := parser.ParseEnvironmentVariables()
+	if err != nil {
+		t.Fatalf(`Parsing failure: %v`, err)
+	}
+
+	expected := "AAAAAAAAAAAAAaaaaaaaaaaaaa0000000000000"
+	result := opts.YouTubeApiKey()
+
+	if result != expected {
+		t.Fatalf(`Unexpected YOUTUBE_API_KEY value, got %v instead of %v`, result, expected)
+	}
+}
+
 func TestYouTubeEmbedUrlOverride(t *testing.T) {
 	os.Clearenv()
 	os.Setenv("YOUTUBE_EMBED_URL_OVERRIDE", "https://invidious.custom/embed/")

internal/config/options.go

@@ -60,6 +60,7 @@ const (
 	defaultFetchNebulaWatchTime               = false
 	defaultFetchOdyseeWatchTime               = false
 	defaultFetchYouTubeWatchTime              = false
+	defaultYouTubeApiKey                      = ""
 	defaultYouTubeEmbedUrlOverride            = "https://www.youtube-nocookie.com/embed/"
 	defaultCreateAdmin                        = false
 	defaultAdminUsername                      = ""
@@ -149,6 +150,7 @@ type Options struct {
 	fetchOdyseeWatchTime               bool
 	fetchYouTubeWatchTime              bool
 	filterEntryMaxAgeDays              int
+	youTubeApiKey                      string
 	youTubeEmbedUrlOverride            string
 	oauth2UserCreationAllowed          bool
 	oauth2ClientID                     string
@@ -228,6 +230,7 @@ func NewOptions() *Options {
 		fetchNebulaWatchTime:               defaultFetchNebulaWatchTime,
 		fetchOdyseeWatchTime:               defaultFetchOdyseeWatchTime,
 		fetchYouTubeWatchTime:              defaultFetchYouTubeWatchTime,
+		youTubeApiKey:                      defaultYouTubeApiKey,
 		youTubeEmbedUrlOverride:            defaultYouTubeEmbedUrlOverride,
 		oauth2UserCreationAllowed:          defaultOAuth2UserCreation,
 		oauth2ClientID:                     defaultOAuth2ClientID,
@@ -503,6 +506,11 @@ func (o *Options) FetchYouTubeWatchTime() bool {
 	return o.fetchYouTubeWatchTime
 }
 
+// YouTubeApiKey returns the YouTube API key if defined.
+func (o *Options) YouTubeApiKey() string {
+	return o.youTubeApiKey
+}
+
 // YouTubeEmbedUrlOverride returns YouTube URL which will be used for embeds
 func (o *Options) YouTubeEmbedUrlOverride() string {
 	return o.youTubeEmbedUrlOverride
@@ -733,6 +741,7 @@ func (o *Options) SortedOptions(redactSecret bool) []*Option {
 		"SERVER_TIMING_HEADER":                   o.serverTimingHeader,
 		"WATCHDOG":                               o.watchdog,
 		"WORKER_POOL_SIZE":                       o.workerPoolSize,
+		"YOUTUBE_API_KEY":                        redactSecretValue(o.youTubeApiKey, redactSecret),
 		"YOUTUBE_EMBED_URL_OVERRIDE":             o.youTubeEmbedUrlOverride,
 		"WEBAUTHN":                               o.webAuthn,
 	}

internal/config/parser.go

@@ -271,6 +271,8 @@ func (p *Parser) parseLines(lines []string) (err error) {
 			p.opts.fetchOdyseeWatchTime = parseBool(value, defaultFetchOdyseeWatchTime)
 		case "FETCH_YOUTUBE_WATCH_TIME":
 			p.opts.fetchYouTubeWatchTime = parseBool(value, defaultFetchYouTubeWatchTime)
+		case "YOUTUBE_API_KEY":
+			p.opts.youTubeApiKey = parseString(value, defaultYouTubeApiKey)
 		case "YOUTUBE_EMBED_URL_OVERRIDE":
 			p.opts.youTubeEmbedUrlOverride = parseString(value, defaultYouTubeEmbedUrlOverride)
 		case "WATCHDOG":

internal/reader/processor/youtube.go

@@ -4,9 +4,11 @@
 package processor
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"log/slog"
+	"net/url"
 	"regexp"
 	"strconv"
 	"time"
@@ -33,6 +35,14 @@ func shouldFetchYouTubeWatchTime(entry *model.Entry) bool {
 }
 
 func fetchYouTubeWatchTime(websiteURL string) (int, error) {
+	if config.Opts.YouTubeApiKey() == "" {
+		return fetchYouTubeWatchTimeFromWebsite(websiteURL)
+	} else {
+		return fetchYouTubeWatchTimeFromApi(websiteURL)
+	}
+}
+
+func fetchYouTubeWatchTimeFromWebsite(websiteURL string) (int, error) {
 	requestBuilder := fetcher.NewRequestBuilder()
 	requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
 	requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
@@ -63,6 +73,61 @@ func fetchYouTubeWatchTime(websiteURL string) (int, error) {
 	return int(dur.Minutes()), nil
 }
 
+func fetchYouTubeWatchTimeFromApi(websiteURL string) (int, error) {
+	requestBuilder := fetcher.NewRequestBuilder()
+	requestBuilder.WithTimeout(config.Opts.HTTPClientTimeout())
+	requestBuilder.WithProxy(config.Opts.HTTPClientProxy())
+
+	parsedWebsiteURL, err := url.Parse(websiteURL)
+	if err != nil {
+		return 0, fmt.Errorf("unable to parse URL: %v", err)
+	}
+
+	apiQuery := url.Values{}
+	apiQuery.Set("id", parsedWebsiteURL.Query().Get("v"))
+	apiQuery.Set("key", config.Opts.YouTubeApiKey())
+	apiQuery.Set("part", "contentDetails")
+
+	apiURL := url.URL{
+		Scheme:   "https",
+		Host:     "www.googleapis.com",
+		Path:     "youtube/v3/videos",
+		RawQuery: apiQuery.Encode(),
+	}
+
+	responseHandler := fetcher.NewResponseHandler(requestBuilder.ExecuteRequest(apiURL.String()))
+	defer responseHandler.Close()
+
+	if localizedError := responseHandler.LocalizedError(); localizedError != nil {
+		slog.Warn("Unable to fetch contentDetails from YouTube API", slog.String("website_url", websiteURL), slog.Any("error", localizedError.Error()))
+		return 0, localizedError.Error()
+	}
+
+	var videos struct {
+		Items []struct {
+			ContentDetails struct {
+				Duration string `json:"duration"`
+			} `json:"contentDetails"`
+		} `json:"items"`
+	}
+
+	if err := json.NewDecoder(responseHandler.Body(config.Opts.HTTPClientMaxBodySize())).Decode(&videos); err != nil {
+		return 0, fmt.Errorf("unable to decode JSON: %v", err)
+	}
+
+	if n := len(videos.Items); n != 1 {
+		return 0, fmt.Errorf("invalid items length: %d", n)
+	}
+
+	durs := videos.Items[0].ContentDetails.Duration
+	dur, err := parseISO8601(durs)
+	if err != nil {
+		return 0, fmt.Errorf("unable to parse duration %s: %v", durs, err)
+	}
+
+	return int(dur.Minutes()), nil
+}
+
 func parseISO8601(from string) (time.Duration, error) {
 	var match []string
 	var d time.Duration

miniflux.1

@@ -555,6 +555,11 @@ Number of background workers\&.
 .br
 Default is 16 workers\&.
 .TP
+.B YOUTUBE_API_KEY
+YouTube API key for use with FETCH_YOUTUBE_WATCH_TIME. If nonempty, the duration will be fetched from the YouTube API. Otherwise, the duration will be fetched from the YouTube website\&.
+.br
+Default is empty\&.
+.TP
 .B YOUTUBE_EMBED_URL_OVERRIDE
 YouTube URL which will be used for embeds\&.
 .br