// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved. // SPDX-License-Identifier: Apache-2.0 package fetcher // import "miniflux.app/v2/internal/reader/fetcher" import ( "net/http" "net/http/httptest" "net/url" "strings" "testing" "time" "miniflux.app/v2/internal/config" "miniflux.app/v2/internal/proxyrotator" ) func TestNewRequestBuilder(t *testing.T) { builder := NewRequestBuilder() if builder == nil { t.Fatal("NewRequestBuilder should not return nil") } if builder.clientTimeout != defaultHTTPClientTimeout { t.Errorf("Expected default timeout %d, got %d", defaultHTTPClientTimeout, builder.clientTimeout) } if builder.headers == nil { t.Fatal("Headers should be initialized") } } func TestRequestBuilder_WithHeader(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Custom-Header") != "custom-value" { t.Errorf("Expected Custom-Header to be 'custom-value', got '%s'", r.Header.Get("Custom-Header")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithHeader("Custom-Header", "custom-value").ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_WithETag(t *testing.T) { tests := []struct { name string etag string expected string }{ {"with etag", "test-etag", "test-etag"}, {"empty etag", "", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("If-None-Match") != tt.expected { t.Errorf("Expected If-None-Match to be '%s', got '%s'", tt.expected, r.Header.Get("If-None-Match")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithETag(tt.etag).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() }) } } func TestRequestBuilder_WithLastModified(t *testing.T) { tests := []struct { name string lastModified string expected string }{ {"with last modified", "Mon, 02 Jan 2006 15:04:05 GMT", "Mon, 02 Jan 2006 15:04:05 GMT"}, {"empty last modified", "", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("If-Modified-Since") != tt.expected { t.Errorf("Expected If-Modified-Since to be '%s', got '%s'", tt.expected, r.Header.Get("If-Modified-Since")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithLastModified(tt.lastModified).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() }) } } func TestRequestBuilder_WithUserAgent(t *testing.T) { tests := []struct { name string userAgent string defaultAgent string expectedHeader string }{ {"custom user agent", "CustomAgent/1.0", "DefaultAgent/1.0", "CustomAgent/1.0"}, {"default user agent", "", "DefaultAgent/1.0", "DefaultAgent/1.0"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("User-Agent") != tt.expectedHeader { t.Errorf("Expected User-Agent to be '%s', got '%s'", tt.expectedHeader, r.Header.Get("User-Agent")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithUserAgent(tt.userAgent, tt.defaultAgent).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() }) } } func TestRequestBuilder_WithCookie(t *testing.T) { tests := []struct { name string cookie string expected string }{ {"with cookie", "session=abc123; lang=en", "session=abc123; lang=en"}, {"empty cookie", "", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Cookie") != tt.expected { t.Errorf("Expected Cookie to be '%s', got '%s'", tt.expected, r.Header.Get("Cookie")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithCookie(tt.cookie, server.URL).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() }) } } func TestRequestBuilder_WithCookieOnlySentToSameSite(t *testing.T) { // The proxy receives the requests sent to any host, so no DNS is needed. receivedCookies := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case receivedCookies <- r.Header.Get("Cookie"): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() proxyURL, err := url.Parse(proxyServer.URL) if err != nil { t.Fatalf("Unable to parse proxy URL: %v", err) } // The same builder is used for all requests, like when scraping the entries of a feed. builder := NewRequestBuilder(). WithCustomApplicationProxyURL(proxyURL). UseCustomApplicationProxyURL(true). WithCookie("session=secret", "https://feeds.example.org/rss.xml") tests := []struct { requestURL string expected string }{ {"http://feeds.example.org/rss.xml", "session=secret"}, {"http://www.example.org/article.html", "session=secret"}, {"http://example.org/favicon.ico", "session=secret"}, {"http://example.net/article.html", ""}, {"http://feeds.example.org.example.net/rss.xml", ""}, {"http://cdn.example.com/icon.png", ""}, } for _, tt := range tests { resp, err := builder.ExecuteRequest(tt.requestURL) if err != nil { t.Fatalf("Expected no error for %q, got %v", tt.requestURL, err) } resp.Body.Close() select { case cookie := <-receivedCookies: if cookie != tt.expected { t.Errorf("Expected Cookie for %q to be %q, got %q", tt.requestURL, tt.expected, cookie) } default: t.Fatalf("Expected request to %q to be sent through the proxy", tt.requestURL) } } } func TestRequestBuilder_WithCookieNotForwardedOnRedirectToAnotherSite(t *testing.T) { type receivedRequest struct { host string cookie string } receivedRequests := make(chan receivedRequest, 2) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case receivedRequests <- receivedRequest{r.Host, r.Header.Get("Cookie")}: default: } if r.Host == "www.example.org" { http.Redirect(w, r, "http://example.net/article.html", http.StatusFound) return } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() proxyURL, err := url.Parse(proxyServer.URL) if err != nil { t.Fatalf("Unable to parse proxy URL: %v", err) } builder := NewRequestBuilder(). WithCustomApplicationProxyURL(proxyURL). UseCustomApplicationProxyURL(true). WithCookie("session=secret", "https://feeds.example.org/rss.xml") resp, err := builder.ExecuteRequest("http://www.example.org/article.html") if err != nil { t.Fatalf("Expected no error, got %v", err) } resp.Body.Close() if len(receivedRequests) != 2 { t.Fatalf("Expected the redirect to be followed, got %d requests", len(receivedRequests)) } if request := <-receivedRequests; request.cookie != "session=secret" { t.Errorf("Expected Cookie for %q to be %q, got %q", request.host, "session=secret", request.cookie) } if request := <-receivedRequests; request.host != "example.net" || request.cookie != "" { t.Errorf("Expected no Cookie to be forwarded to example.net, got %q for %q", request.cookie, request.host) } } func TestIsSameSite(t *testing.T) { tests := []struct { host string siteHost string expected bool }{ {"example.org", "example.org", true}, {"www.example.org", "example.org", true}, {"example.org", "www.example.org", true}, {"www.example.org", "feeds.example.org", true}, {"WWW.Example.ORG", "feeds.example.org", true}, {"news.example.co.uk", "www.example.co.uk", true}, {"xn--bcher-kva.example", "www.bücher.example", true}, {"example.net", "example.org", false}, {"notexample.org", "example.org", false}, {"example.org.example.net", "example.org", false}, {"attacker.co.uk", "example.co.uk", false}, {"attacker.CO.UK", "www.example.CO.UK", false}, {"bob.github.io", "alice.github.io", false}, {"127.0.0.1", "127.0.0.1", true}, {"10.0.0.1", "192.168.0.1", false}, {"::1", "::1", true}, {"localhost", "localhost", true}, {"intranet", "localhost", false}, {"", "example.org", false}, {"example.org", "", false}, {"", "", false}, } for _, tt := range tests { if got := isSameSite(tt.host, tt.siteHost); got != tt.expected { t.Errorf("isSameSite(%q, %q) = %v, expected %v", tt.host, tt.siteHost, got, tt.expected) } } } func TestRequestBuilder_WithUsernameAndPassword(t *testing.T) { tests := []struct { name string username string password string expected string }{ {"with credentials", "test", "password", "Basic dGVzdDpwYXNzd29yZA=="}, // base64 of "test:password" {"empty username", "", "password", ""}, {"empty password", "test", "", ""}, {"both empty", "", "", ""}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != tt.expected { t.Errorf("Expected Authorization to be '%s', got '%s'", tt.expected, r.Header.Get("Authorization")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithUsernameAndPassword(tt.username, tt.password).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() }) } } func TestRequestBuilder_DefaultAcceptHeader(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Accept") != defaultAcceptHeader { t.Errorf("Expected Accept to be '%s', got '%s'", defaultAcceptHeader, r.Header.Get("Accept")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_CustomAcceptHeaderNotOverridden(t *testing.T) { customAccept := "application/json" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Accept") != customAccept { t.Errorf("Expected Accept to be '%s', got '%s'", customAccept, r.Header.Get("Accept")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithHeader("Accept", customAccept).ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_WithTimeout(t *testing.T) { builder := NewRequestBuilder() builder = builder.WithTimeout(30 * time.Second) if builder.clientTimeout != 30*time.Second { t.Errorf("Expected timeout to be 30, got %d", builder.clientTimeout) } } func TestRequestBuilder_WithoutRedirects(t *testing.T) { // Create a redirect server redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer redirectServer.Close() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, redirectServer.URL, http.StatusFound) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithoutRedirects().ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() if resp.StatusCode != http.StatusFound { t.Errorf("Expected status code %d, got %d", http.StatusFound, resp.StatusCode) } } func TestRequestBuilder_Clone(t *testing.T) { original := NewRequestBuilder().WithHeader("X-Shared", "value") clone := original.Clone().WithoutRedirects() clone.WithHeader("X-Clone-Only", "value") if original.withoutRedirects { t.Error("Mutating the clone should not disable redirects on the original") } if original.headers.Get("X-Clone-Only") != "" { t.Error("Mutating the clone's headers should not affect the original") } if clone.headers.Get("X-Shared") != "value" { t.Error("Expected the clone to inherit the original headers") } if clone.clientTimeout != original.clientTimeout { t.Error("Expected the clone to inherit the original timeout") } } func TestRequestBuilder_DisableHTTP2(t *testing.T) { builder := NewRequestBuilder() builder = builder.DisableHTTP2(true) if !builder.disableHTTP2 { t.Error("Expected disableHTTP2 to be true") } } func TestRequestBuilder_IgnoreTLSErrors(t *testing.T) { builder := NewRequestBuilder() builder = builder.IgnoreTLSErrors(true) if !builder.ignoreTLSErrors { t.Error("Expected ignoreTLSErrors to be true") } } func TestRequestBuilder_WithoutCompression(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Accept-Encoding") != "identity" { t.Errorf("Expected Accept-Encoding to be 'identity', got '%s'", r.Header.Get("Accept-Encoding")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.WithoutCompression().ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_WithCompression(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Accept-Encoding") != "br,gzip" { t.Errorf("Expected Accept-Encoding to be 'br,gzip', got '%s'", r.Header.Get("Accept-Encoding")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_ConnectionCloseHeader(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Connection") != "close" { t.Errorf("Expected Connection to be 'close', got '%s'", r.Header.Get("Connection")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_WithCustomApplicationProxyURL(t *testing.T) { proxyURL, _ := url.Parse("http://proxy.example.com:8080") builder := NewRequestBuilder() builder = builder.WithCustomApplicationProxyURL(proxyURL) if builder.clientProxyURL != proxyURL { t.Error("Expected clientProxyURL to be set") } } func TestRequestBuilder_UseCustomApplicationProxyURL(t *testing.T) { builder := NewRequestBuilder() builder = builder.UseCustomApplicationProxyURL(true) if !builder.useClientProxy { t.Error("Expected useClientProxy to be true") } } func TestRequestBuilder_WithCustomFeedProxyURL(t *testing.T) { proxyURL := "http://feed-proxy.example.com:8080" builder := NewRequestBuilder() builder = builder.WithCustomFeedProxyURL(proxyURL) if builder.feedProxyURL != proxyURL { t.Errorf("Expected feedProxyURL to be '%s', got '%s'", proxyURL, builder.feedProxyURL) } } func TestRequestBuilder_ChainedMethods(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Check multiple headers if r.Header.Get("User-Agent") != "TestAgent/1.0" { t.Errorf("Expected User-Agent to be 'TestAgent/1.0', got '%s'", r.Header.Get("User-Agent")) } if r.Header.Get("Cookie") != "test=value" { t.Errorf("Expected Cookie to be 'test=value', got '%s'", r.Header.Get("Cookie")) } if r.Header.Get("If-None-Match") != "etag123" { t.Errorf("Expected If-None-Match to be 'etag123', got '%s'", r.Header.Get("If-None-Match")) } w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder. WithUserAgent("TestAgent/1.0", "DefaultAgent/1.0"). WithCookie("test=value", server.URL). WithETag("etag123"). WithTimeout(10 * time.Second). ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected no error, got %v", err) } defer resp.Body.Close() } func TestRequestBuilder_InvalidURL(t *testing.T) { builder := NewRequestBuilder() _, err := builder.ExecuteRequest("invalid-url") if err == nil { t.Error("Expected error for invalid URL") } } func TestRequestBuilder_RefusePrivateNetworkByDefault(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() _, err := builder.ExecuteRequest(server.URL) if err == nil { t.Fatal("Expected private network request to be rejected") } if !strings.Contains(err.Error(), "refusing to access private network host") { t.Fatalf("Unexpected error for private network request: %v", err) } } func TestRequestBuilder_AllowPrivateNetworkWhenEnabled(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "1") server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() builder := NewRequestBuilder() resp, err := builder.ExecuteRequest(server.URL) if err != nil { t.Fatalf("Expected private network request to succeed when enabled: %v", err) } defer resp.Body.Close() } func TestRequestBuilder_AllowPrivateConfiguredProxy(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") tests := []struct { name string configure func(t *testing.T, builder *RequestBuilder, proxyURL string) *RequestBuilder }{ { name: "application proxy", configure: func(t *testing.T, builder *RequestBuilder, proxyURL string) *RequestBuilder { t.Helper() parsedProxyURL, err := url.Parse(proxyURL) if err != nil { t.Fatalf("Unable to parse proxy URL: %v", err) } return builder.WithCustomApplicationProxyURL(parsedProxyURL).UseCustomApplicationProxyURL(true) }, }, { name: "proxy rotator", configure: func(t *testing.T, builder *RequestBuilder, proxyURL string) *RequestBuilder { t.Helper() rotator, err := proxyrotator.NewProxyRotator([]string{proxyURL}) if err != nil { t.Fatalf("Unable to create proxy rotator: %v", err) } return builder.WithProxyRotator(rotator) }, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { targetURL := "http://feed.invalid/rss.xml" proxyRequests := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case proxyRequests <- r.URL.String(): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() builder := tt.configure(t, NewRequestBuilder(), proxyServer.URL) resp, err := builder.ExecuteRequest(targetURL) if err != nil { t.Fatalf("Expected private proxy request to succeed: %v", err) } defer resp.Body.Close() select { case gotURL := <-proxyRequests: if gotURL != targetURL { t.Fatalf("Expected proxy request URL to be %q, got %q", targetURL, gotURL) } default: t.Fatal("Expected request to be sent through the proxy") } }) } } func TestRequestBuilder_RefusePrivateFeedProxyByDefault(t *testing.T) { configureFetcherPrivateNetworkOptions(t, "0", "0") proxyRequests := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case proxyRequests <- r.URL.String(): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() _, err := NewRequestBuilder(). WithCustomFeedProxyURL(proxyServer.URL). ExecuteRequest("http://feed.invalid/rss.xml") if err == nil { t.Fatal("Expected private feed proxy request to be rejected") } if !strings.Contains(err.Error(), "refusing to access private network host") { t.Fatalf("Unexpected error for private feed proxy request: %v", err) } select { case gotURL := <-proxyRequests: t.Fatalf("Expected private feed proxy to receive no request, got %q", gotURL) default: } } func TestRequestBuilder_AllowPrivateFeedProxyWhenEnabled(t *testing.T) { configureFetcherPrivateNetworkOptions(t, "0", "1") targetURL := "http://feed.invalid/rss.xml" proxyRequests := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case proxyRequests <- r.URL.String(): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() resp, err := NewRequestBuilder(). WithCustomFeedProxyURL(proxyServer.URL). ExecuteRequest(targetURL) if err != nil { t.Fatalf("Expected private feed proxy request to succeed: %v", err) } defer resp.Body.Close() select { case gotURL := <-proxyRequests: if gotURL != targetURL { t.Fatalf("Expected proxy request URL to be %q, got %q", targetURL, gotURL) } default: t.Fatal("Expected request to be sent through the private feed proxy") } } func TestRequestBuilder_AllowPrivateEnvironmentProxy(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") targetURL := "http://feed.invalid/rss.xml" proxyRequests := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case proxyRequests <- r.URL.String(): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() t.Setenv("HTTP_PROXY", proxyServer.URL) resp, err := NewRequestBuilder().ExecuteRequest(targetURL) if err != nil { t.Fatalf("Expected request through the environment proxy to succeed: %v", err) } defer resp.Body.Close() select { case gotURL := <-proxyRequests: if gotURL != targetURL { t.Fatalf("Expected proxy request URL to be %q, got %q", targetURL, gotURL) } default: t.Fatal("Expected request to be sent through the environment proxy") } } func TestRequestBuilder_RefusePrivateNetworkWhenExcludedFromEnvironmentProxy(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") privateServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer privateServer.Close() proxyRequests := make(chan string, 1) proxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case proxyRequests <- r.URL.String(): default: } w.WriteHeader(http.StatusOK) })) defer proxyServer.Close() t.Setenv("HTTP_PROXY", proxyServer.URL) t.Setenv("NO_PROXY", "feed.invalid") _, err := NewRequestBuilder().ExecuteRequest("http://feed.invalid/rss.xml") if err == nil { t.Fatal("Expected request excluded from the environment proxy to fail") } select { case gotURL := <-proxyRequests: t.Fatalf("Expected request to bypass the environment proxy, but the proxy received %q", gotURL) default: } } func TestRequestBuilder_AllowPrivateEnvironmentProxyOnRedirect(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") connectRequests := make(chan string, 1) httpsProxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { select { case connectRequests <- r.Host: default: } http.Error(w, "tunneling is not supported by this test proxy", http.StatusBadGateway) })) defer httpsProxyServer.Close() httpProxyServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "https://secure.invalid/rss.xml", http.StatusFound) })) defer httpProxyServer.Close() t.Setenv("HTTP_PROXY", httpProxyServer.URL) t.Setenv("HTTPS_PROXY", httpsProxyServer.URL) if _, err := NewRequestBuilder().ExecuteRequest("http://feed.invalid/rss.xml"); err != nil { if strings.Contains(err.Error(), "refusing to access private network host") { t.Fatalf("Expected the proxy selected after the redirect to be reachable: %v", err) } } select { case gotHost := <-connectRequests: if gotHost != "secure.invalid:443" { t.Fatalf("Expected the HTTPS proxy to receive a CONNECT request for %q, got %q", "secure.invalid:443", gotHost) } default: t.Fatal("Expected the redirect to be sent through the HTTPS environment proxy") } } func TestRequestBuilder_RefusePrivateNetworkOnRedirect(t *testing.T) { configureFetcherAllowPrivateNetworksOption(t, "0") // Target server on a loopback address (private). privateServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) })) defer privateServer.Close() // Redirector that sends the client to the private server. // Because the Control callback checks the IP at connection time, the // redirect target is also validated (unlike a pre-flight DNS check). redirectServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, privateServer.URL, http.StatusFound) })) defer redirectServer.Close() builder := NewRequestBuilder() _, err := builder.ExecuteRequest(redirectServer.URL) if err == nil { t.Fatal("Expected redirect to private network to be rejected") } if !strings.Contains(err.Error(), "refusing to access private network host") { t.Fatalf("Unexpected error for redirected private network request: %v", err) } } func TestRequestBuilder_TimeoutConfiguration(t *testing.T) { // Create a slow server that blocks until the client disconnects, so // server.Close() does not have to wait for a fixed sleep to elapse. server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-r.Context().Done() })) defer server.Close() builder := NewRequestBuilder() start := time.Now() _, err := builder.WithTimeout(100 * time.Millisecond).ExecuteRequest(server.URL) duration := time.Since(start) if err == nil { t.Error("Expected timeout error") } // Should timeout around 100ms, allow some margin if duration > 500*time.Millisecond { t.Errorf("Expected timeout around 100ms, took %v", duration) } } func configureFetcherAllowPrivateNetworksOption(t *testing.T, value string) { configureFetcherPrivateNetworkOptions(t, value, "0") } func configureFetcherPrivateNetworkOptions(t *testing.T, privateNetworks, privateFeedProxy string) { t.Helper() t.Setenv("FETCHER_ALLOW_PRIVATE_NETWORKS", privateNetworks) t.Setenv("FETCHER_ALLOW_PRIVATE_FEED_PROXY", privateFeedProxy) configParser := config.NewConfigParser() parsedOptions, err := configParser.ParseEnvironmentVariables() if err != nil { t.Fatalf("Unable to configure test options: %v", err) } previousOptions := config.Opts config.Opts = parsedOptions t.Cleanup(func() { config.Opts = previousOptions }) }