miniflux_v2/internal/http/client/client.go

// SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

package client // import "miniflux.app/v2/internal/http/client"

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io"
	"log/slog"
	"net"
	"net/http"
	"net/url"
	"time"

	"miniflux.app/v2/internal/config"
	"miniflux.app/v2/internal/errors"
)

const (
	defaultHTTPClientTimeout     = 20
	defaultHTTPClientMaxBodySize = 15 * 1024 * 1024
)

var (
	errInvalidCertificate = "Invalid SSL certificate (original error: %q)"
	errNetworkOperation   = "This website is unreachable (original error: %q)"
	errRequestTimeout     = "Website unreachable, the request timed out after %d seconds"
)

// Client builds and executes HTTP requests.
type Client struct {
	inputURL string

	requestEtagHeader          string
	requestLastModifiedHeader  string
	requestAuthorizationHeader string
	requestUsername            string
	requestPassword            string
	requestUserAgent           string
	requestCookie              string
	customHeaders              map[string]string
	useProxy                   bool
	doNotFollowRedirects       bool

	ClientTimeout               int
	ClientMaxBodySize           int64
	ClientProxyURL              string
	AllowSelfSignedCertificates bool
}

// New initializes a new HTTP client.
func New(url string) *Client {
	return &Client{
		inputURL:          url,
		ClientTimeout:     defaultHTTPClientTimeout,
		ClientMaxBodySize: defaultHTTPClientMaxBodySize,
	}
}

// NewClientWithConfig initializes a new HTTP client with application config options.
func NewClientWithConfig(url string, opts *config.Options) *Client {
	return &Client{
		inputURL:          url,
		requestUserAgent:  opts.HTTPClientUserAgent(),
		ClientTimeout:     opts.HTTPClientTimeout(),
		ClientMaxBodySize: opts.HTTPClientMaxBodySize(),
		ClientProxyURL:    opts.HTTPClientProxy(),
	}
}

// WithCredentials defines the username/password for HTTP Basic authentication.
func (c *Client) WithCredentials(username, password string) *Client {
	if username != "" && password != "" {
		c.requestUsername = username
		c.requestPassword = password
	}
	return c
}

// WithCustomHeaders defines custom HTTP headers.
func (c *Client) WithCustomHeaders(customHeaders map[string]string) *Client {
	c.customHeaders = customHeaders
	return c
}

// WithCacheHeaders defines caching headers.
func (c *Client) WithCacheHeaders(etagHeader, lastModifiedHeader string) *Client {
	c.requestEtagHeader = etagHeader
	c.requestLastModifiedHeader = lastModifiedHeader
	return c
}

// WithProxy enables proxy for the current HTTP request.
func (c *Client) WithProxy() *Client {
	c.useProxy = true
	return c
}

// WithoutRedirects disables HTTP redirects.
func (c *Client) WithoutRedirects() *Client {
	c.doNotFollowRedirects = true
	return c
}

// WithUserAgent defines the User-Agent header to use for HTTP requests.
func (c *Client) WithUserAgent(userAgent string) *Client {
	if userAgent != "" {
		c.requestUserAgent = userAgent
	}
	return c
}

// WithCookie defines the Cookies to use for HTTP requests.
func (c *Client) WithCookie(cookie string) *Client {
	if cookie != "" {
		c.requestCookie = cookie
	}
	return c
}

// Get performs a GET HTTP request.
func (c *Client) Get() (*Response, error) {
	request, err := c.buildRequest(http.MethodGet, nil)
	if err != nil {
		return nil, err
	}

	return c.executeRequest(request)
}

func (c *Client) executeRequest(request *http.Request) (*Response, error) {
	startTime := time.Now()

	slog.Debug("Executing outgoing HTTP request",
		slog.Group("request",
			slog.String("method", request.Method),
			slog.String("url", request.URL.String()),
			slog.String("user_agent", request.UserAgent()),
			slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")),
			slog.Bool("has_cookie", c.requestCookie != ""),
			slog.Bool("with_redirects", !c.doNotFollowRedirects),
			slog.Bool("with_proxy", c.useProxy),
			slog.String("proxy_url", c.ClientProxyURL),
			slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""),
		),
	)

	client := c.buildClient()
	resp, err := client.Do(request)
	if resp != nil {
		defer resp.Body.Close()
	}

	if err != nil {
		if uerr, ok := err.(*url.Error); ok {
			switch uerr.Err.(type) {
			case x509.CertificateInvalidError, x509.HostnameError:
				err = errors.NewLocalizedError(errInvalidCertificate, uerr.Err)
			case *net.OpError:
				err = errors.NewLocalizedError(errNetworkOperation, uerr.Err)
			case net.Error:
				nerr := uerr.Err.(net.Error)
				if nerr.Timeout() {
					err = errors.NewLocalizedError(errRequestTimeout, c.ClientTimeout)
				}
			}
		}

		return nil, err
	}

	if resp.ContentLength > c.ClientMaxBodySize {
		return nil, fmt.Errorf("client: response too large (%d bytes)", resp.ContentLength)
	}

	buf, err := io.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("client: error while reading body %v", err)
	}

	response := &Response{
		Body:          bytes.NewReader(buf),
		StatusCode:    resp.StatusCode,
		EffectiveURL:  resp.Request.URL.String(),
		LastModified:  resp.Header.Get("Last-Modified"),
		ETag:          resp.Header.Get("ETag"),
		Expires:       resp.Header.Get("Expires"),
		ContentType:   resp.Header.Get("Content-Type"),
		ContentLength: resp.ContentLength,
	}

	slog.Debug("Completed outgoing HTTP request",
		slog.Duration("duration", time.Since(startTime)),
		slog.Group("request",
			slog.String("method", request.Method),
			slog.String("url", request.URL.String()),
			slog.String("user_agent", request.UserAgent()),
			slog.Bool("is_authenticated", c.requestAuthorizationHeader != "" || (c.requestUsername != "" && c.requestPassword != "")),
			slog.Bool("has_cookie", c.requestCookie != ""),
			slog.Bool("with_redirects", !c.doNotFollowRedirects),
			slog.Bool("with_proxy", c.useProxy),
			slog.String("proxy_url", c.ClientProxyURL),
			slog.Bool("with_caching_headers", c.requestEtagHeader != "" || c.requestLastModifiedHeader != ""),
		),
		slog.Group("response",
			slog.Int("status_code", response.StatusCode),
			slog.String("effective_url", response.EffectiveURL),
			slog.String("content_type", response.ContentType),
			slog.Int64("content_length", response.ContentLength),
			slog.String("last_modified", response.LastModified),
			slog.String("etag", response.ETag),
			slog.String("expires", response.Expires),
		),
	)

	// Ignore caching headers for feeds that do not want any cache.
	if resp.Header.Get("Expires") == "0" {
		response.ETag = ""
		response.LastModified = ""
	}

	return response, err
}

func (c *Client) buildRequest(method string, body io.Reader) (*http.Request, error) {
	request, err := http.NewRequest(method, c.inputURL, body)
	if err != nil {
		return nil, err
	}

	request.Header = c.buildHeaders()

	if c.requestUsername != "" && c.requestPassword != "" {
		request.SetBasicAuth(c.requestUsername, c.requestPassword)
	}

	return request, nil
}

func (c *Client) buildClient() http.Client {
	client := http.Client{
		Timeout: time.Duration(c.ClientTimeout) * time.Second,
	}

	transport := &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		DialContext: (&net.Dialer{
			// Default is 30s.
			Timeout: 10 * time.Second,

			// Default is 30s.
			KeepAlive: 15 * time.Second,
		}).DialContext,

		// Default is 100.
		MaxIdleConns: 50,

		// Default is 90s.
		IdleConnTimeout: 10 * time.Second,
	}

	if c.AllowSelfSignedCertificates {
		transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true}
	}

	if c.doNotFollowRedirects {
		client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
			return http.ErrUseLastResponse
		}
	}

	if c.useProxy && c.ClientProxyURL != "" {
		proxyURL, err := url.Parse(c.ClientProxyURL)
		if err != nil {
			slog.Error("Unable to parse proxy URL",
				slog.String("proxy_url", c.ClientProxyURL),
				slog.Any("error", err),
			)
		} else {
			transport.Proxy = http.ProxyURL(proxyURL)
		}
	}

	client.Transport = transport

	return client
}

func (c *Client) buildHeaders() http.Header {
	headers := make(http.Header)
	headers.Add("Accept", "*/*")

	if c.requestUserAgent != "" {
		headers.Add("User-Agent", c.requestUserAgent)
	}

	if c.requestEtagHeader != "" {
		headers.Add("If-None-Match", c.requestEtagHeader)
	}

	if c.requestLastModifiedHeader != "" {
		headers.Add("If-Modified-Since", c.requestLastModifiedHeader)
	}

	if c.requestAuthorizationHeader != "" {
		headers.Add("Authorization", c.requestAuthorizationHeader)
	}

	if c.requestCookie != "" {
		headers.Add("Cookie", c.requestCookie)
	}

	for key, value := range c.customHeaders {
		headers.Add(key, value)
	}

	headers.Add("Connection", "close")
	return headers
}