fix(mediaproxy): always forward the browser Accept header

Tumblr CDN is blocking the Accept header used to fetch feeds

internal/reader/fetcher/request_builder.go

@@ -18,21 +18,21 @@ import (
 )
 
 const (
-	defaultHTTPClientTimeout     = 20
-	defaultHTTPClientMaxBodySize = 15 * 1024 * 1024
-	defaultAcceptHeader          = "application/xml, application/atom+xml, application/rss+xml, application/rdf+xml, application/feed+json, text/html, */*;q=0.9"
+	defaultHTTPClientTimeout = 20
+	defaultAcceptHeader      = "application/xml, application/atom+xml, application/rss+xml, application/rdf+xml, application/feed+json, text/html, */*;q=0.9"
 )
 
 type RequestBuilder struct {
-	headers          http.Header
-	clientProxyURL   *url.URL
-	clientTimeout    int
-	useClientProxy   bool
-	withoutRedirects bool
-	ignoreTLSErrors  bool
-	disableHTTP2     bool
-	proxyRotator     *proxyrotator.ProxyRotator
-	feedProxyURL     string
+	headers            http.Header
+	clientProxyURL     *url.URL
+	clientTimeout      int
+	useClientProxy     bool
+	withoutRedirects   bool
+	ignoreTLSErrors    bool
+	disableHTTP2       bool
+	disableCompression bool
+	proxyRotator       *proxyrotator.ProxyRotator
+	feedProxyURL       string
 }
 
 func NewRequestBuilder() *RequestBuilder {
@@ -124,6 +124,11 @@ func (r *RequestBuilder) IgnoreTLSErrors(value bool) *RequestBuilder {
 	return r
 }
 
+func (r *RequestBuilder) WithoutCompression() *RequestBuilder {
+	r.disableCompression = true
+	return r
+}
+
 func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, error) {
 	transport := &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
@@ -197,8 +202,18 @@ func (r *RequestBuilder) ExecuteRequest(requestURL string) (*http.Response, erro
 	}
 
 	req.Header = r.headers
-	req.Header.Set("Accept-Encoding", "br, gzip")
-	req.Header.Set("Accept", defaultAcceptHeader)
+	if r.disableCompression {
+		req.Header.Set("Accept-Encoding", "identity")
+	} else {
+		req.Header.Set("Accept-Encoding", "br, gzip")
+	}
+
+	// Set default Accept header if not already set.
+	// Note that for the media proxy requests, we need to forward the browser Accept header.
+	if req.Header.Get("Accept") == "" {
+		req.Header.Set("Accept", defaultAcceptHeader)
+	}
+
 	req.Header.Set("Connection", "close")
 
 	slog.Debug("Making outgoing request", slog.Group("request",

internal/ui/proxy.go

@@ -88,6 +88,9 @@ func (h *handler) mediaProxy(w http.ResponseWriter, r *http.Request) {
 	requestBuilder := fetcher.NewRequestBuilder()
 	requestBuilder.WithTimeout(config.Opts.MediaProxyHTTPClientTimeout())
 
+	// Disable compression for the media proxy requests (not implemented).
+	requestBuilder.WithoutCompression()
+
 	if referer := rewrite.GetRefererForURL(mediaURL); referer != "" {
 		requestBuilder.WithHeader("Referer", referer)
 	}