url.go 6.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. // SPDX-FileCopyrightText: Copyright The Miniflux Authors. All rights reserved.
  2. // SPDX-License-Identifier: Apache-2.0
  3. package urllib // import "miniflux.app/v2/internal/urllib"
  4. import (
  5. "errors"
  6. "fmt"
  7. "net"
  8. "net/netip"
  9. "net/url"
  10. "strings"
  11. )
  12. var (
  13. rfc6598SharedAddressSpacePrefix = netip.MustParsePrefix("100.64.0.0/10")
  14. nat64WellKnownPrefix = netip.MustParsePrefix("64:ff9b::/96")
  15. sixToFourPrefix = netip.MustParsePrefix("2002::/16")
  16. nonPublicIPPrefixes = []netip.Prefix{
  17. netip.MustParsePrefix("168.63.129.16/32"),
  18. netip.MustParsePrefix("192.0.0.192/32"),
  19. netip.MustParsePrefix("198.18.0.0/15"),
  20. netip.MustParsePrefix("::/96"), // Deprecated IPv4-compatible IPv6 addresses (RFC 4291).
  21. netip.MustParsePrefix("fec0::/10"),
  22. }
  23. )
  24. // IsRelativePath reports whether the link is a relative path (no scheme, host, or scheme-relative // form).
  25. func IsRelativePath(link string) bool {
  26. if link == "" {
  27. return false
  28. }
  29. // Reject surrounding whitespace: browsers strip leading and trailing C0
  30. // control characters and spaces before parsing a target, while Go's url.Parse
  31. // treats spaces as path characters. Without this check, a target like
  32. // " //evil.com" is accepted here and becomes //evil.com in the browser.
  33. if link != strings.TrimSpace(link) {
  34. return false
  35. }
  36. // Reject backslashes: Go's url.Parse treats them as ordinary path
  37. // characters, but browsers normalize them to forward slashes, so a target
  38. // like "/\evil.com" would parse as relative here yet redirect to
  39. // //evil.com in the browser (open redirect).
  40. if strings.Contains(link, "\\") {
  41. return false
  42. }
  43. // Reject scheme-relative URLs, including forms with extra leading slashes
  44. // that url.Parse treats as a relative path but browsers resolve externally.
  45. if strings.HasPrefix(link, "//") {
  46. return false
  47. }
  48. if parsedURL, err := url.Parse(link); err == nil {
  49. // Only allow relative paths and ensure the URL doesn't have a host
  50. // component.
  51. if !parsedURL.IsAbs() && parsedURL.Host == "" && parsedURL.Scheme == "" {
  52. return true
  53. }
  54. }
  55. return false
  56. }
  57. // hasHTTPPrefix reports whether the URL string begins with an HTTP or HTTPS scheme.
  58. func hasHTTPPrefix(inputURL string) bool {
  59. return strings.HasPrefix(inputURL, "https://") || strings.HasPrefix(inputURL, "http://")
  60. }
  61. // hasSOCKSPrefix reports whether the URL string begins with an SOCKS5 or SOCKS5H scheme.
  62. func hasSOCKSPrefix(inputURL string) bool {
  63. return strings.HasPrefix(inputURL, "socks5://") || strings.HasPrefix(inputURL, "socks5h://")
  64. }
  65. // IsAbsoluteURL reports whether the link is absolute and starts with an HTTP or HTTPS scheme.
  66. func IsAbsoluteURL(inputURL string) bool {
  67. if !hasHTTPPrefix(inputURL) {
  68. return false
  69. }
  70. parsedURL, err := url.Parse(inputURL)
  71. if err != nil {
  72. return false
  73. }
  74. return parsedURL.IsAbs()
  75. }
  76. // IsValidProxyURL reports whether the url is absolute, has a host and starts with an HTTP, HTTPS, SOCKS5 or SOCKS5H scheme.
  77. func IsValidProxyURL(inputURL string) bool {
  78. if !hasHTTPPrefix(inputURL) && !hasSOCKSPrefix(inputURL) {
  79. return false
  80. }
  81. parsedURL, err := url.Parse(inputURL)
  82. if err != nil {
  83. return false
  84. }
  85. return parsedURL.IsAbs() && parsedURL.Host != ""
  86. }
  87. // resolveToAbsoluteURL resolves a relative URL using a base URL, parsing the base only if needed.
  88. func resolveToAbsoluteURL(parsedBaseURL *url.URL, baseURL, relativeURL string) (string, error) {
  89. // Avoid parsing the relative URL if it's already absolute
  90. if strings.HasPrefix(relativeURL, "//") {
  91. return "https:" + relativeURL, nil
  92. }
  93. if hasHTTPPrefix(relativeURL) {
  94. return relativeURL, nil
  95. }
  96. // Parse the relative URL and check if it's already absolute
  97. parsedRelativeURL, err := url.Parse(relativeURL)
  98. if err != nil {
  99. return "", fmt.Errorf("unable to parse relative URL: %w", err)
  100. }
  101. if parsedRelativeURL.IsAbs() {
  102. return relativeURL, nil
  103. }
  104. // Parse the base URL if not already parsed
  105. if parsedBaseURL == nil {
  106. parsedBaseURL, err = url.Parse(baseURL)
  107. if err != nil {
  108. return "", fmt.Errorf("unable to parse base URL: %w", err)
  109. }
  110. }
  111. return parsedBaseURL.ResolveReference(parsedRelativeURL).String(), nil
  112. }
  113. // ResolveToAbsoluteURL resolves a relative URL against a base URL and returns the absolute URL.
  114. func ResolveToAbsoluteURL(baseURL, relativeURL string) (string, error) {
  115. return resolveToAbsoluteURL(nil, baseURL, relativeURL)
  116. }
  117. // ResolveToAbsoluteURLWithParsedBaseURL resolves a relative URL using a pre-parsed base URL and returns the absolute URL.
  118. func ResolveToAbsoluteURLWithParsedBaseURL(parsedBaseURL *url.URL, relativeURL string) (string, error) {
  119. return resolveToAbsoluteURL(parsedBaseURL, "", relativeURL)
  120. }
  121. // RootURL returns the scheme and host of the given URL with a trailing slash.
  122. func RootURL(websiteURL string) string {
  123. if websiteURL == "" {
  124. return ""
  125. }
  126. if strings.HasPrefix(websiteURL, "//") {
  127. websiteURL = "https://" + websiteURL[2:]
  128. }
  129. u, err := url.Parse(websiteURL)
  130. if err != nil || u.Scheme == "" || u.Host == "" {
  131. return websiteURL
  132. }
  133. u.Fragment = ""
  134. u.RawQuery = ""
  135. u.Path = "/"
  136. u.RawPath = ""
  137. return u.Scheme + "://" + u.Host + "/"
  138. }
  139. // IsHTTPS reports whether the URL uses HTTPS.
  140. func IsHTTPS(websiteURL string) bool {
  141. parsedURL, err := url.Parse(websiteURL)
  142. if err != nil {
  143. return false
  144. }
  145. return strings.EqualFold(parsedURL.Scheme, "https")
  146. }
  147. // Domain returns the host component of the given URL.
  148. func Domain(websiteURL string) string {
  149. parsedURL, err := url.Parse(websiteURL)
  150. if err != nil {
  151. return websiteURL
  152. }
  153. return parsedURL.Host
  154. }
  155. // DomainWithoutWWW returns the host component without a leading "www." prefix when present.
  156. func DomainWithoutWWW(websiteURL string) string {
  157. return strings.TrimPrefix(Domain(websiteURL), "www.")
  158. }
  159. // JoinBaseURLAndPath joins a base URL and a path segment into a single URL string.
  160. func JoinBaseURLAndPath(baseURL, path string) (string, error) {
  161. if baseURL == "" {
  162. return "", errors.New("empty base URL")
  163. }
  164. if path == "" {
  165. return "", errors.New("empty path")
  166. }
  167. finalURL, err := url.JoinPath(baseURL, path)
  168. if err != nil {
  169. return "", fmt.Errorf("unable to join base URL %s and path %s: %w", baseURL, path, err)
  170. }
  171. return finalURL, nil
  172. }
  173. // IsNonPublicIP returns true if the given IP is not safe to access from the network.
  174. func IsNonPublicIP(ip net.IP) bool {
  175. addr, ok := netip.AddrFromSlice(ip)
  176. if !ok {
  177. return true
  178. }
  179. return isNonPublicAddr(addr.Unmap())
  180. }
  181. func isNonPublicAddr(addr netip.Addr) bool {
  182. if nat64WellKnownPrefix.Contains(addr) {
  183. addrBytes := addr.As16()
  184. return isNonPublicAddr(netip.AddrFrom4([4]byte(addrBytes[12:])))
  185. }
  186. if sixToFourPrefix.Contains(addr) {
  187. addrBytes := addr.As16()
  188. return isNonPublicAddr(netip.AddrFrom4([4]byte(addrBytes[2:6])))
  189. }
  190. if rfc6598SharedAddressSpacePrefix.Contains(addr) {
  191. return true
  192. }
  193. for _, prefix := range nonPublicIPPrefixes {
  194. if prefix.Contains(addr) {
  195. return true
  196. }
  197. }
  198. return addr.IsPrivate() ||
  199. addr.IsLoopback() ||
  200. addr.IsLinkLocalUnicast() ||
  201. addr.IsLinkLocalMulticast() ||
  202. addr.IsMulticast() ||
  203. addr.IsUnspecified()
  204. }