Closes #11780: Enable loading import data from remote sources

docs/release-notes/version-3.5.md

@@ -40,6 +40,7 @@ A new ASN range model has been introduced to facilitate the provisioning of new
 * [#11584](https://github.com/netbox-community/netbox/issues/11584) - Add a list view for contact assignments
 * [#11625](https://github.com/netbox-community/netbox/issues/11625) - Add HTMX support to ObjectEditView
 * [#11693](https://github.com/netbox-community/netbox/issues/11693) - Enable syncing export template content from remote sources
+* [#11780](https://github.com/netbox-community/netbox/issues/11780) - Enable loading import data from remote sources
 * [#11968](https://github.com/netbox-community/netbox/issues/11968) - Add navigation menu buttons to create device & VM components
 
 ### Other Changes

netbox/netbox/views/generic/bulk_views.py

@@ -16,10 +16,10 @@ from django_tables2.export import TableExport
 
 from extras.models import ExportTemplate
 from extras.signals import clear_webhooks
-from utilities.choices import ImportFormatChoices
 from utilities.error_handlers import handle_protectederror
 from utilities.exceptions import AbortRequest, AbortTransaction, PermissionsViolation
-from utilities.forms import BulkRenameForm, ConfirmationForm, ImportForm, restrict_form_fields
+from utilities.forms import BulkRenameForm, ConfirmationForm, restrict_form_fields
+from utilities.forms.bulk_import import ImportForm
 from utilities.htmx import is_embedded, is_htmx
 from utilities.permissions import get_permission_for_model
 from utilities.views import GetReturnURLMixin

netbox/templates/generic/bulk_import.html

@@ -15,15 +15,20 @@ Context:
 {% block tabs %}
   <ul class="nav nav-tabs px-3">
     <li class="nav-item" role="presentation">
-      <button class="nav-link active" id="data-import-tab" data-bs-toggle="tab" data-bs-target="#data-import-form" type="button" role="tab" aria-controls="data-import-form" aria-selected="true">
-        Data Import
+      <button class="nav-link active" id="import-form-tab" data-bs-toggle="tab" data-bs-target="#import-form" type="button" role="tab" aria-controls="import-form" aria-selected="true">
+        Direct Import
       </button>
     </li>
     <li class="nav-item" role="presentation">
-      <button class="nav-link" id="file-upload-tab" data-bs-toggle="tab" data-bs-target="#file-upload-form" type="button" role="tab" aria-controls="file-upload-form" aria-selected="false">
+      <button class="nav-link" id="upload-form-tab" data-bs-toggle="tab" data-bs-target="#upload-form" type="button" role="tab" aria-controls="upload-form" aria-selected="false">
         Upload File
       </button>
     </li>
+    <li class="nav-item" role="presentation">
+      <button class="nav-link" id="datafile-form-tab" data-bs-toggle="tab" data-bs-target="#datafile-form" type="button" role="tab" aria-controls="datafile-form" aria-selected="false">
+        Data File
+      </button>
+    </li>
   </ul>
 {% endblock tabs %}
 
@@ -31,45 +36,66 @@ Context:
   <div class="tab-content">
 
     {# Data Import Form #}
-    <div class="tab-pane show active" id="data-import-form" role="tabpanel" aria-labelledby="data-import-tab">
-      {% block content %}
-          <div class="row">
-              <div class="col col-md-12 col-lg-10">
-                  <form action="" method="post" enctype="multipart/form-data" class="form">
-                    {% csrf_token %}
-                    {% render_field form.data %}
-                    {% render_field form.format %}
-                    <div class="form-group">
-                      <div class="col col-md-12 text-end">
-                        <button type="submit" name="data_submit" class="btn btn-primary">Submit</button>
-                        {% if return_url %}
-                          <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
-                        {% endif %}
-                      </div>
-                    </div>
-                  </form>
+    <div class="tab-pane show active" id="import-form" role="tabpanel" aria-labelledby="import-form-tab">
+      <div class="row">
+        <div class="col col-md-12 col-lg-10 offset-lg-1">
+          <form action="" method="post" enctype="multipart/form-data" class="form">
+            {% csrf_token %}
+            <input type="hidden" name="import_method" value="direct" />
+            {% render_field form.data %}
+            {% render_field form.format %}
+            <div class="form-group">
+              <div class="col col-md-12 text-end">
+                <button type="submit" name="data_submit" class="btn btn-primary">Submit</button>
+                {% if return_url %}
+                  <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
+                {% endif %}
               </div>
-          </div>
-      {% endblock content %}
+            </div>
+          </form>
+        </div>
+      </div>
     </div>
 
     {# File Upload Form #}
-    <div class="tab-pane show" id="file-upload-form" role="tabpanel" aria-labelledby="file-upload-tab">
-              <div class="col col-md-12 col-lg-10">
-                  <form action="" method="post" enctype="multipart/form-data" class="form">
-                    {% csrf_token %}
-                    {% render_field form.data_file %}
-                    {% render_field form.format %}
-                    <div class="form-group">
-                      <div class="col col-md-12 text-end">
-                        <button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
-                        {% if return_url %}
-                          <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
-                        {% endif %}
-                      </div>
-                    </div>
-                  </form>
-              </div>
+    <div class="tab-pane show" id="upload-form" role="tabpanel" aria-labelledby="upload-form-tab">
+      <div class="col col-md-12 col-lg-10 offset-lg-1">
+        <form action="" method="post" enctype="multipart/form-data" class="form">
+          {% csrf_token %}
+          <input type="hidden" name="import_method" value="upload" />
+          {% render_field form.upload_file %}
+          {% render_field form.format %}
+          <div class="form-group">
+            <div class="col col-md-12 text-end">
+              <button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
+              {% if return_url %}
+                <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
+              {% endif %}
+            </div>
+          </div>
+        </form>
+      </div>
+    </div>
+
+    {# DataFile Form #}
+    <div class="tab-pane show" id="datafile-form" role="tabpanel" aria-labelledby="datafile-form-tab">
+      <div class="col col-md-12 col-lg-10 offset-lg-1">
+        <form action="" method="post" enctype="multipart/form-data" class="form">
+          {% csrf_token %}
+          <input type="hidden" name="import_method" value="datafile" />
+          {% render_field form.data_source %}
+          {% render_field form.data_file %}
+          {% render_field form.format %}
+          <div class="form-group">
+            <div class="col col-md-12 text-end">
+              <button type="submit" name="file_submit" class="btn btn-primary">Submit</button>
+              {% if return_url %}
+                <a href="{{ return_url }}" class="btn btn-outline-danger">Cancel</a>
+              {% endif %}
+            </div>
+          </div>
+        </form>
+      </div>
     </div>
 
     {% if fields %}

netbox/utilities/choices.py

@@ -203,6 +203,18 @@ class ButtonColorChoices(ChoiceSet):
 # Import Choices
 #
 
+class ImportMethodChoices(ChoiceSet):
+    DIRECT = 'direct'
+    UPLOAD = 'upload'
+    DATA_FILE = 'datafile'
+
+    CHOICES = [
+        (DIRECT, 'Direct'),
+        (UPLOAD, 'Upload'),
+        (DATA_FILE, 'Data file'),
+    ]
+
+
 class ImportFormatChoices(ChoiceSet):
     AUTO = 'auto'
     CSV = 'csv'

netbox/utilities/forms/bulk_import.py

@@ -0,0 +1,141 @@
+import csv
+import json
+from io import StringIO
+
+import yaml
+from django import forms
+from django.utils.translation import gettext as _
+
+from extras.forms.mixins import SyncedDataMixin
+from utilities.choices import ImportFormatChoices
+from utilities.forms.utils import parse_csv
+from ..choices import ImportMethodChoices
+from .forms import BootstrapMixin
+
+
+class ImportForm(BootstrapMixin, SyncedDataMixin, forms.Form):
+    import_method = forms.ChoiceField(
+        choices=ImportMethodChoices
+    )
+    data = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'class': 'font-monospace'}),
+        help_text=_("Enter object data in CSV, JSON or YAML format.")
+    )
+    upload_file = forms.FileField(
+        label="Data file",
+        required=False
+    )
+    format = forms.ChoiceField(
+        choices=ImportFormatChoices,
+        initial=ImportFormatChoices.AUTO
+    )
+
+    data_field = 'data'
+
+    def clean(self):
+        super().clean()
+
+        # Determine import method
+        import_method = self.cleaned_data['import_method']
+
+        # Determine whether we're reading from form data or an uploaded file
+        if self.cleaned_data['data'] and import_method != ImportMethodChoices.DIRECT:
+            raise forms.ValidationError("Form data must be empty when uploading/selecting a file.")
+        if import_method == ImportMethodChoices.UPLOAD:
+            self.upload_file = 'upload_file'
+            file = self.files.get('upload_file')
+            data = file.read().decode('utf-8-sig')
+        elif import_method == ImportMethodChoices.DATA_FILE:
+            data = self.cleaned_data['data_file'].data_as_string
+        else:
+            data = self.cleaned_data['data']
+
+        # Determine the data format
+        if self.cleaned_data['format'] == ImportFormatChoices.AUTO:
+            format = self._detect_format(data)
+        else:
+            format = self.cleaned_data['format']
+
+        # Process data according to the selected format
+        if format == ImportFormatChoices.CSV:
+            self.cleaned_data['data'] = self._clean_csv(data)
+        elif format == ImportFormatChoices.JSON:
+            self.cleaned_data['data'] = self._clean_json(data)
+        elif format == ImportFormatChoices.YAML:
+            self.cleaned_data['data'] = self._clean_yaml(data)
+        else:
+            raise forms.ValidationError(f"Unknown data format: {format}")
+
+    def _detect_format(self, data):
+        """
+        Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise
+        a ValidationError.
+        """
+        try:
+            if data[0] in ('{', '['):
+                return ImportFormatChoices.JSON
+            if data.startswith('---') or data.startswith('- '):
+                return ImportFormatChoices.YAML
+            if ',' in data.split('\n', 1)[0]:
+                return ImportFormatChoices.CSV
+        except IndexError:
+            pass
+        raise forms.ValidationError({
+            'format': _('Unable to detect data format. Please specify.')
+        })
+
+    def _clean_csv(self, data):
+        """
+        Clean CSV-formatted data. The first row will be treated as column headers.
+        """
+        stream = StringIO(data.strip())
+        reader = csv.reader(stream)
+        headers, records = parse_csv(reader)
+
+        # Set CSV headers for reference by the model form
+        self._csv_headers = headers
+
+        return records
+
+    def _clean_json(self, data):
+        """
+        Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.
+        """
+        try:
+            data = json.loads(data)
+            # Accommodate for users entering single objects
+            if type(data) is not list:
+                data = [data]
+            return data
+        except json.decoder.JSONDecodeError as err:
+            raise forms.ValidationError({
+                self.data_field: f"Invalid JSON data: {err}"
+            })
+
+    def _clean_yaml(self, data):
+        """
+        Clean YAML-formatted data. Data must be either
+          a) A single document comprising a list of dictionaries (each representing an object), or
+          b) Multiple documents, separated with the '---' token
+        """
+        records = []
+        try:
+            for data in yaml.load_all(data, Loader=yaml.SafeLoader):
+                if type(data) == list:
+                    records.extend(data)
+                elif type(data) == dict:
+                    records.append(data)
+                else:
+                    raise forms.ValidationError({
+                        self.data_field: _(
+                            "Invalid YAML data. Data must be in the form of multiple documents, or a single document "
+                            "comprising a list of dictionaries."
+                        )
+                    })
+        except yaml.error.YAMLError as err:
+            raise forms.ValidationError({
+                self.data_field: f"Invalid YAML data: {err}"
+            })
+
+        return records

netbox/utilities/forms/forms.py

@@ -1,14 +1,8 @@
-import csv
-import json
 import re
-from io import StringIO
 
-import yaml
 from django import forms
 from django.utils.translation import gettext as _
 
-from utilities.choices import ImportFormatChoices
-from utilities.forms.utils import parse_csv
 from .widgets import APISelect, APISelectMultiple, ClearableFileInput
 
 __all__ = (
@@ -18,7 +12,6 @@ __all__ = (
     'ConfirmationForm',
     'CSVModelForm',
     'FilterForm',
-    'ImportForm',
     'ReturnURLForm',
     'TableConfigForm',
 )
@@ -157,126 +150,6 @@ class CSVModelForm(forms.ModelForm):
                     del self.fields[field]
 
 
-class ImportForm(BootstrapMixin, forms.Form):
-    data = forms.CharField(
-        required=False,
-        widget=forms.Textarea(attrs={'class': 'font-monospace'}),
-        help_text=_("Enter object data in CSV, JSON or YAML format.")
-    )
-    data_file = forms.FileField(
-        label="Data file",
-        required=False
-    )
-    format = forms.ChoiceField(
-        choices=ImportFormatChoices,
-        initial=ImportFormatChoices.AUTO
-    )
-
-    data_field = 'data'
-
-    def clean(self):
-        super().clean()
-
-        # Determine whether we're reading from form data or an uploaded file
-        if self.cleaned_data['data'] and self.cleaned_data['data_file']:
-            raise forms.ValidationError("Form data must be empty when uploading a file.")
-        if 'data_file' in self.files:
-            self.data_field = 'data_file'
-            file = self.files.get('data_file')
-            data = file.read().decode('utf-8-sig')
-        else:
-            data = self.cleaned_data['data']
-
-        # Determine the data format
-        if self.cleaned_data['format'] == ImportFormatChoices.AUTO:
-            format = self._detect_format(data)
-        else:
-            format = self.cleaned_data['format']
-
-        # Process data according to the selected format
-        if format == ImportFormatChoices.CSV:
-            self.cleaned_data['data'] = self._clean_csv(data)
-        elif format == ImportFormatChoices.JSON:
-            self.cleaned_data['data'] = self._clean_json(data)
-        elif format == ImportFormatChoices.YAML:
-            self.cleaned_data['data'] = self._clean_yaml(data)
-        else:
-            raise forms.ValidationError(f"Unknown data format: {format}")
-
-    def _detect_format(self, data):
-        """
-        Attempt to automatically detect the format (CSV, JSON, or YAML) of the given data, or raise
-        a ValidationError.
-        """
-        try:
-            if data[0] in ('{', '['):
-                return ImportFormatChoices.JSON
-            if data.startswith('---') or data.startswith('- '):
-                return ImportFormatChoices.YAML
-            if ',' in data.split('\n', 1)[0]:
-                return ImportFormatChoices.CSV
-        except IndexError:
-            pass
-        raise forms.ValidationError({
-            'format': _('Unable to detect data format. Please specify.')
-        })
-
-    def _clean_csv(self, data):
-        """
-        Clean CSV-formatted data. The first row will be treated as column headers.
-        """
-        stream = StringIO(data.strip())
-        reader = csv.reader(stream)
-        headers, records = parse_csv(reader)
-
-        # Set CSV headers for reference by the model form
-        self._csv_headers = headers
-
-        return records
-
-    def _clean_json(self, data):
-        """
-        Clean JSON-formatted data. If only a single object is defined, it will be encapsulated as a list.
-        """
-        try:
-            data = json.loads(data)
-            # Accommodate for users entering single objects
-            if type(data) is not list:
-                data = [data]
-            return data
-        except json.decoder.JSONDecodeError as err:
-            raise forms.ValidationError({
-                self.data_field: f"Invalid JSON data: {err}"
-            })
-
-    def _clean_yaml(self, data):
-        """
-        Clean YAML-formatted data. Data must be either
-          a) A single document comprising a list of dictionaries (each representing an object), or
-          b) Multiple documents, separated with the '---' token
-        """
-        records = []
-        try:
-            for data in yaml.load_all(data, Loader=yaml.SafeLoader):
-                if type(data) == list:
-                    records.extend(data)
-                elif type(data) == dict:
-                    records.append(data)
-                else:
-                    raise forms.ValidationError({
-                        self.data_field: _(
-                            "Invalid YAML data. Data must be in the form of multiple documents, or a single document "
-                            "comprising a list of dictionaries."
-                        )
-                    })
-        except yaml.error.YAMLError as err:
-            raise forms.ValidationError({
-                self.data_field: f"Invalid YAML data: {err}"
-            })
-
-        return records
-
-
 class FilterForm(BootstrapMixin, forms.Form):
     """
     Base Form class for FilterSet forms.

netbox/utilities/tests/test_forms.py

@@ -3,7 +3,7 @@ from django.test import TestCase
 
 from ipam.forms import IPAddressImportForm
 from utilities.choices import ImportFormatChoices
-from utilities.forms import ImportForm
+from utilities.forms.bulk_import import ImportForm
 from utilities.forms.fields import CSVDataField
 from utilities.forms.utils import expand_alphanumeric_pattern, expand_ipaddress_pattern