data_backends.py 7.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236
  1. import logging
  2. import re
  3. import tempfile
  4. from contextlib import contextmanager
  5. from pathlib import Path, PurePosixPath
  6. from urllib.parse import urlparse
  7. from django import forms
  8. from django.core.exceptions import ImproperlyConfigured
  9. from django.utils.translation import gettext as _
  10. from netbox.data_backends import DataBackend
  11. from netbox.utils import register_data_backend
  12. from utilities.constants import HTTP_PROXY_SUPPORTED_SCHEMAS, HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS
  13. from utilities.proxy import resolve_proxies
  14. from utilities.socks import ProxyPoolManager
  15. from .exceptions import SyncError
  16. __all__ = (
  17. 'GitBackend',
  18. 'LocalBackend',
  19. 'S3Backend',
  20. 'url_has_embedded_credentials',
  21. )
  22. logger = logging.getLogger('netbox.data_backends')
  23. def url_has_embedded_credentials(url):
  24. """
  25. Check if a URL contains embedded credentials (username in the URL).
  26. URLs like 'https://user@bitbucket.org/...' have embedded credentials.
  27. This is used to avoid passing explicit credentials to dulwich when the
  28. URL already contains them, which would cause authentication conflicts.
  29. """
  30. parsed = urlparse(url)
  31. return bool(parsed.username)
  32. @register_data_backend()
  33. class LocalBackend(DataBackend):
  34. name = 'local'
  35. label = _('Local')
  36. is_local = True
  37. @contextmanager
  38. def fetch(self):
  39. logger.debug("Data source type is local; skipping fetch")
  40. local_path = urlparse(self.url).path # Strip file:// scheme
  41. yield local_path
  42. @register_data_backend()
  43. class GitBackend(DataBackend):
  44. name = 'git'
  45. label = 'Git'
  46. parameters = {
  47. 'username': forms.CharField(
  48. required=False,
  49. label=_('Username'),
  50. widget=forms.TextInput(attrs={'class': 'form-control'}),
  51. help_text=_("Only used for cloning with HTTP(S)"),
  52. ),
  53. 'password': forms.CharField(
  54. required=False,
  55. label=_('Password'),
  56. widget=forms.TextInput(attrs={'class': 'form-control'}),
  57. help_text=_("Only used for cloning with HTTP(S)"),
  58. ),
  59. 'branch': forms.CharField(
  60. required=False,
  61. label=_('Branch'),
  62. widget=forms.TextInput(attrs={'class': 'form-control'})
  63. )
  64. }
  65. sensitive_parameters = ['password']
  66. def init_config(self):
  67. from dulwich.config import ConfigDict
  68. # Initialize backend config
  69. config = ConfigDict()
  70. self.socks_proxy = None
  71. # Apply HTTP proxy (if configured)
  72. proxies = resolve_proxies(url=self.url, context={'client': self}) or {}
  73. if proxy := proxies.get(self.url_scheme):
  74. if urlparse(proxy).scheme not in HTTP_PROXY_SUPPORTED_SCHEMAS:
  75. raise ImproperlyConfigured(f"Unsupported Git DataSource proxy scheme: {urlparse(proxy).scheme}")
  76. if self.url_scheme in ('http', 'https'):
  77. config.set("http", "proxy", proxy)
  78. if urlparse(proxy).scheme in HTTP_PROXY_SUPPORTED_SOCK_SCHEMAS:
  79. self.socks_proxy = proxy
  80. return config
  81. @contextmanager
  82. def fetch(self):
  83. from dulwich import porcelain
  84. local_path = tempfile.TemporaryDirectory()
  85. clone_args = {
  86. "branch": self.params.get('branch'),
  87. "config": self.config,
  88. "errstream": porcelain.NoneStream(),
  89. }
  90. # check if using socks for proxy - if so need to use custom pool_manager
  91. if self.socks_proxy:
  92. clone_args['pool_manager'] = ProxyPoolManager(self.socks_proxy)
  93. if self.url_scheme in ('http', 'https'):
  94. # Only pass explicit credentials if URL doesn't already contain embedded username
  95. # to avoid credential conflicts (see #20902)
  96. if not url_has_embedded_credentials(self.url) and self.params.get('username'):
  97. clone_args.update(
  98. {
  99. "username": self.params.get('username'),
  100. "password": self.params.get('password'),
  101. }
  102. )
  103. if self.url_scheme:
  104. clone_args["quiet"] = True
  105. clone_args["depth"] = 1
  106. logger.debug(f"Cloning git repo: {self.url}")
  107. try:
  108. porcelain.clone(self.url, local_path.name, **clone_args)
  109. except BaseException as e:
  110. raise SyncError(_("Fetching remote data failed ({name}): {error}").format(name=type(e).__name__, error=e))
  111. yield local_path.name
  112. local_path.cleanup()
  113. @register_data_backend()
  114. class S3Backend(DataBackend):
  115. name = 'amazon-s3'
  116. label = 'Amazon S3'
  117. parameters = {
  118. 'aws_access_key_id': forms.CharField(
  119. label=_('AWS access key ID'),
  120. widget=forms.TextInput(attrs={'class': 'form-control'})
  121. ),
  122. 'aws_secret_access_key': forms.CharField(
  123. label=_('AWS secret access key'),
  124. widget=forms.TextInput(attrs={'class': 'form-control'})
  125. ),
  126. }
  127. sensitive_parameters = ['aws_secret_access_key']
  128. REGION_REGEX = r's3\.([a-z0-9-]+)\.amazonaws\.com'
  129. def init_config(self):
  130. from botocore.config import Config as Boto3Config
  131. # Initialize backend config
  132. return Boto3Config(
  133. proxies=resolve_proxies(url=self.url, context={'client': self}),
  134. )
  135. @contextmanager
  136. def fetch(self):
  137. import boto3
  138. local_path = tempfile.TemporaryDirectory()
  139. local_root = Path(local_path.name).resolve()
  140. # Initialize the S3 resource and bucket
  141. aws_access_key_id = self.params.get('aws_access_key_id')
  142. aws_secret_access_key = self.params.get('aws_secret_access_key')
  143. s3 = boto3.resource(
  144. 's3',
  145. region_name=self._region_name,
  146. aws_access_key_id=aws_access_key_id,
  147. aws_secret_access_key=aws_secret_access_key,
  148. config=self.config,
  149. endpoint_url=self._endpoint_url
  150. )
  151. bucket = s3.Bucket(self._bucket_name)
  152. try:
  153. # Download all files within the specified path
  154. for obj in bucket.objects.filter(Prefix=self._remote_path):
  155. local_filename = self._resolve_local_path(local_root, obj.key)
  156. # Build local path
  157. local_filename.parent.mkdir(parents=True, exist_ok=True)
  158. bucket.download_file(obj.key, str(local_filename))
  159. yield local_path.name
  160. finally:
  161. local_path.cleanup()
  162. @staticmethod
  163. def _resolve_local_path(local_root, key):
  164. # S3 object keys are POSIX-style paths. Strip any leading separator so the key
  165. # joins onto the temp directory rather than replacing it, then ensure the
  166. # resolved destination remains within the temp directory to prevent path
  167. # traversal via crafted object keys.
  168. key_parts = PurePosixPath(key.lstrip('/')).parts
  169. local_filename = local_root.joinpath(*key_parts).resolve()
  170. if not local_filename.is_relative_to(local_root) or local_filename == local_root:
  171. raise SyncError(
  172. _("Invalid S3 object key '{key}': resolves outside of the local data directory").format(key=key)
  173. )
  174. return local_filename
  175. @property
  176. def _region_name(self):
  177. domain = urlparse(self.url).netloc
  178. if m := re.match(self.REGION_REGEX, domain):
  179. return m.group(1)
  180. return None
  181. @property
  182. def _bucket_name(self):
  183. url_path = urlparse(self.url).path.lstrip('/')
  184. return url_path.split('/')[0]
  185. @property
  186. def _endpoint_url(self):
  187. url_path = urlparse(self.url)
  188. return url_path._replace(params="", fragment="", query="", path="").geturl()
  189. @property
  190. def _remote_path(self):
  191. url_path = urlparse(self.url).path.lstrip('/')
  192. if '/' in url_path:
  193. return url_path.split('/', 1)[1]
  194. return ''