Просмотр исходного кода

Closes #13381: Enable plugins to register custom data backends (#14095)

* Initial work on #13381

* Fix backend type display in table column

* Fix data source type choices during bulk edit

* Misc cleanup

* Move backend utils from core app to netbox

* Move backend type validation from serializer to model
Jeremy Stretch 2 лет назад
Родитель
Сommit
30ce9edf1c

+ 23 - 0
docs/plugins/development/data-backends.md

@@ -0,0 +1,23 @@
+# Data Backends
+
+[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
+
+```python title="data_backends.py"
+from netbox.data_backends import DataBackend
+
+class MyDataBackend(DataBackend):
+    name = 'mybackend'
+    label = 'My Backend'
+    ...
+```
+
+To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
+
+```python title="data_backends.py"
+backends = [MyDataBackend]
+```
+
+!!! tip
+    The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
+
+::: core.data_backends.DataBackend

+ 1 - 0
docs/plugins/development/index.md

@@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
 | `middleware`          | A list of middleware classes to append after NetBox's build-in middleware                                                |
 | `middleware`          | A list of middleware classes to append after NetBox's build-in middleware                                                |
 | `queues`              | A list of custom background task queues to create                                                                        |
 | `queues`              | A list of custom background task queues to create                                                                        |
 | `search_extensions`   | The dotted path to the list of search index classes (default: `search.indexes`)                                          |
 | `search_extensions`   | The dotted path to the list of search index classes (default: `search.indexes`)                                          |
+| `data_backends`       | The dotted path to the list of data source backend classes (default: `data_backends.backends`)                           |
 | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`)              |
 | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`)              |
 | `menu_items`          | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`)                      |
 | `menu_items`          | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`)                      |
 | `graphql_schema`      | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`)                                 |
 | `graphql_schema`      | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`)                                 |

+ 1 - 0
mkdocs.yml

@@ -136,6 +136,7 @@ nav:
             - Forms: 'plugins/development/forms.md'
             - Forms: 'plugins/development/forms.md'
             - Filters & Filter Sets: 'plugins/development/filtersets.md'
             - Filters & Filter Sets: 'plugins/development/filtersets.md'
             - Search: 'plugins/development/search.md'
             - Search: 'plugins/development/search.md'
+            - Data Backends: 'plugins/development/data-backends.md'
             - REST API: 'plugins/development/rest-api.md'
             - REST API: 'plugins/development/rest-api.md'
             - GraphQL API: 'plugins/development/graphql-api.md'
             - GraphQL API: 'plugins/development/graphql-api.md'
             - Background Tasks: 'plugins/development/background-tasks.md'
             - Background Tasks: 'plugins/development/background-tasks.md'

+ 2 - 1
netbox/core/api/serializers.py

@@ -4,6 +4,7 @@ from core.choices import *
 from core.models import *
 from core.models import *
 from netbox.api.fields import ChoiceField, ContentTypeField
 from netbox.api.fields import ChoiceField, ContentTypeField
 from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
 from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
+from netbox.utils import get_data_backend_choices
 from users.api.nested_serializers import NestedUserSerializer
 from users.api.nested_serializers import NestedUserSerializer
 from .nested_serializers import *
 from .nested_serializers import *
 
 
@@ -19,7 +20,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
         view_name='core-api:datasource-detail'
         view_name='core-api:datasource-detail'
     )
     )
     type = ChoiceField(
     type = ChoiceField(
-        choices=DataSourceTypeChoices
+        choices=get_data_backend_choices()
     )
     )
     status = ChoiceField(
     status = ChoiceField(
         choices=DataSourceStatusChoices,
         choices=DataSourceStatusChoices,

+ 0 - 12
netbox/core/choices.py

@@ -7,18 +7,6 @@ from utilities.choices import ChoiceSet
 # Data sources
 # Data sources
 #
 #
 
 
-class DataSourceTypeChoices(ChoiceSet):
-    LOCAL = 'local'
-    GIT = 'git'
-    AMAZON_S3 = 'amazon-s3'
-
-    CHOICES = (
-        (LOCAL, _('Local'), 'gray'),
-        (GIT, 'Git', 'blue'),
-        (AMAZON_S3, 'Amazon S3', 'blue'),
-    )
-
-
 class DataSourceStatusChoices(ChoiceSet):
 class DataSourceStatusChoices(ChoiceSet):
     NEW = 'new'
     NEW = 'new'
     QUEUED = 'queued'
     QUEUED = 'queued'

+ 13 - 46
netbox/core/data_backends.py

@@ -10,61 +10,24 @@ from django import forms
 from django.conf import settings
 from django.conf import settings
 from django.utils.translation import gettext as _
 from django.utils.translation import gettext as _
 
 
-from netbox.registry import registry
-from .choices import DataSourceTypeChoices
+from netbox.data_backends import DataBackend
+from netbox.utils import register_data_backend
 from .exceptions import SyncError
 from .exceptions import SyncError
 
 
 __all__ = (
 __all__ = (
-    'LocalBackend',
     'GitBackend',
     'GitBackend',
+    'LocalBackend',
     'S3Backend',
     'S3Backend',
 )
 )
 
 
 logger = logging.getLogger('netbox.data_backends')
 logger = logging.getLogger('netbox.data_backends')
 
 
 
 
-def register_backend(name):
-    """
-    Decorator for registering a DataBackend class.
-    """
-
-    def _wrapper(cls):
-        registry['data_backends'][name] = cls
-        return cls
-
-    return _wrapper
-
-
-class DataBackend:
-    parameters = {}
-    sensitive_parameters = []
-
-    # Prevent Django's template engine from calling the backend
-    # class when referenced via DataSource.backend_class
-    do_not_call_in_templates = True
-
-    def __init__(self, url, **kwargs):
-        self.url = url
-        self.params = kwargs
-        self.config = self.init_config()
-
-    def init_config(self):
-        """
-        Hook to initialize the instance's configuration.
-        """
-        return
-
-    @property
-    def url_scheme(self):
-        return urlparse(self.url).scheme.lower()
-
-    @contextmanager
-    def fetch(self):
-        raise NotImplemented()
-
-
-@register_backend(DataSourceTypeChoices.LOCAL)
+@register_data_backend()
 class LocalBackend(DataBackend):
 class LocalBackend(DataBackend):
+    name = 'local'
+    label = _('Local')
+    is_local = True
 
 
     @contextmanager
     @contextmanager
     def fetch(self):
     def fetch(self):
@@ -74,8 +37,10 @@ class LocalBackend(DataBackend):
         yield local_path
         yield local_path
 
 
 
 
-@register_backend(DataSourceTypeChoices.GIT)
+@register_data_backend()
 class GitBackend(DataBackend):
 class GitBackend(DataBackend):
+    name = 'git'
+    label = 'Git'
     parameters = {
     parameters = {
         'username': forms.CharField(
         'username': forms.CharField(
             required=False,
             required=False,
@@ -144,8 +109,10 @@ class GitBackend(DataBackend):
         local_path.cleanup()
         local_path.cleanup()
 
 
 
 
-@register_backend(DataSourceTypeChoices.AMAZON_S3)
+@register_data_backend()
 class S3Backend(DataBackend):
 class S3Backend(DataBackend):
+    name = 'amazon-s3'
+    label = 'Amazon S3'
     parameters = {
     parameters = {
         'aws_access_key_id': forms.CharField(
         'aws_access_key_id': forms.CharField(
             label=_('AWS access key ID'),
             label=_('AWS access key ID'),

+ 2 - 1
netbox/core/filtersets.py

@@ -4,6 +4,7 @@ from django.utils.translation import gettext as _
 import django_filters
 import django_filters
 
 
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
+from netbox.utils import get_data_backend_choices
 from .choices import *
 from .choices import *
 from .models import *
 from .models import *
 
 
@@ -16,7 +17,7 @@ __all__ = (
 
 
 class DataSourceFilterSet(NetBoxModelFilterSet):
 class DataSourceFilterSet(NetBoxModelFilterSet):
     type = django_filters.MultipleChoiceFilter(
     type = django_filters.MultipleChoiceFilter(
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         null_value=None
         null_value=None
     )
     )
     status = django_filters.MultipleChoiceFilter(
     status = django_filters.MultipleChoiceFilter(

+ 3 - 5
netbox/core/forms/bulk_edit.py

@@ -1,10 +1,9 @@
 from django import forms
 from django import forms
 from django.utils.translation import gettext_lazy as _
 from django.utils.translation import gettext_lazy as _
 
 
-from core.choices import DataSourceTypeChoices
 from core.models import *
 from core.models import *
 from netbox.forms import NetBoxModelBulkEditForm
 from netbox.forms import NetBoxModelBulkEditForm
-from utilities.forms import add_blank_choice
+from netbox.utils import get_data_backend_choices
 from utilities.forms.fields import CommentField
 from utilities.forms.fields import CommentField
 from utilities.forms.widgets import BulkEditNullBooleanSelect
 from utilities.forms.widgets import BulkEditNullBooleanSelect
 
 
@@ -16,9 +15,8 @@ __all__ = (
 class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
 class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
     type = forms.ChoiceField(
     type = forms.ChoiceField(
         label=_('Type'),
         label=_('Type'),
-        choices=add_blank_choice(DataSourceTypeChoices),
-        required=False,
-        initial=''
+        choices=get_data_backend_choices,
+        required=False
     )
     )
     enabled = forms.NullBooleanField(
     enabled = forms.NullBooleanField(
         required=False,
         required=False,

+ 2 - 1
netbox/core/forms/filtersets.py

@@ -8,6 +8,7 @@ from core.models import *
 from extras.forms.mixins import SavedFiltersMixin
 from extras.forms.mixins import SavedFiltersMixin
 from extras.utils import FeatureQuery
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelFilterSetForm
 from netbox.forms import NetBoxModelFilterSetForm
+from netbox.utils import get_data_backend_choices
 from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
 from utilities.forms import BOOLEAN_WITH_BLANK_CHOICES, FilterForm
 from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
 from utilities.forms.fields import ContentTypeChoiceField, DynamicModelMultipleChoiceField
 from utilities.forms.widgets import APISelectMultiple, DateTimePicker
 from utilities.forms.widgets import APISelectMultiple, DateTimePicker
@@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
     )
     )
     type = forms.MultipleChoiceField(
     type = forms.MultipleChoiceField(
         label=_('Type'),
         label=_('Type'),
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         required=False
         required=False
     )
     )
     status = forms.MultipleChoiceField(
     status = forms.MultipleChoiceField(

+ 12 - 7
netbox/core/forms/model_forms.py

@@ -7,6 +7,7 @@ from core.forms.mixins import SyncedDataMixin
 from core.models import *
 from core.models import *
 from netbox.forms import NetBoxModelForm
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
 from netbox.registry import registry
+from netbox.utils import get_data_backend_choices
 from utilities.forms import get_field_value
 from utilities.forms import get_field_value
 from utilities.forms.fields import CommentField
 from utilities.forms.fields import CommentField
 from utilities.forms.widgets import HTMXSelect
 from utilities.forms.widgets import HTMXSelect
@@ -18,6 +19,10 @@ __all__ = (
 
 
 
 
 class DataSourceForm(NetBoxModelForm):
 class DataSourceForm(NetBoxModelForm):
+    type = forms.ChoiceField(
+        choices=get_data_backend_choices,
+        widget=HTMXSelect()
+    )
     comments = CommentField()
     comments = CommentField()
 
 
     class Meta:
     class Meta:
@@ -26,7 +31,6 @@ class DataSourceForm(NetBoxModelForm):
             'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
             'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
         ]
         ]
         widgets = {
         widgets = {
-            'type': HTMXSelect(),
             'ignore_rules': forms.Textarea(
             'ignore_rules': forms.Textarea(
                 attrs={
                 attrs={
                     'rows': 5,
                     'rows': 5,
@@ -56,12 +60,13 @@ class DataSourceForm(NetBoxModelForm):
 
 
         # Add backend-specific form fields
         # Add backend-specific form fields
         self.backend_fields = []
         self.backend_fields = []
-        for name, form_field in backend.parameters.items():
-            field_name = f'backend_{name}'
-            self.backend_fields.append(field_name)
-            self.fields[field_name] = copy.copy(form_field)
-            if self.instance and self.instance.parameters:
-                self.fields[field_name].initial = self.instance.parameters.get(name)
+        if backend:
+            for name, form_field in backend.parameters.items():
+                field_name = f'backend_{name}'
+                self.backend_fields.append(field_name)
+                self.fields[field_name] = copy.copy(form_field)
+                if self.instance and self.instance.parameters:
+                    self.fields[field_name].initial = self.instance.parameters.get(name)
 
 
     def save(self, *args, **kwargs):
     def save(self, *args, **kwargs):
 
 

+ 18 - 0
netbox/core/migrations/0006_datasource_type_remove_choices.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.6 on 2023-10-20 17:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_job_created_auto_now'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='type',
+            field=models.CharField(max_length=50),
+        ),
+    ]

+ 11 - 10
netbox/core/models/data.py

@@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
     )
     )
     type = models.CharField(
     type = models.CharField(
         verbose_name=_('type'),
         verbose_name=_('type'),
-        max_length=50,
-        choices=DataSourceTypeChoices,
-        default=DataSourceTypeChoices.LOCAL
+        max_length=50
     )
     )
     source_url = models.CharField(
     source_url = models.CharField(
         max_length=200,
         max_length=200,
@@ -96,8 +94,9 @@ class DataSource(JobsMixin, PrimaryModel):
     def docs_url(self):
     def docs_url(self):
         return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
         return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
 
 
-    def get_type_color(self):
-        return DataSourceTypeChoices.colors.get(self.type)
+    def get_type_display(self):
+        if backend := registry['data_backends'].get(self.type):
+            return backend.label
 
 
     def get_status_color(self):
     def get_status_color(self):
         return DataSourceStatusChoices.colors.get(self.status)
         return DataSourceStatusChoices.colors.get(self.status)
@@ -110,10 +109,6 @@ class DataSource(JobsMixin, PrimaryModel):
     def backend_class(self):
     def backend_class(self):
         return registry['data_backends'].get(self.type)
         return registry['data_backends'].get(self.type)
 
 
-    @property
-    def is_local(self):
-        return self.type == DataSourceTypeChoices.LOCAL
-
     @property
     @property
     def ready_for_sync(self):
     def ready_for_sync(self):
         return self.enabled and self.status not in (
         return self.enabled and self.status not in (
@@ -123,8 +118,14 @@ class DataSource(JobsMixin, PrimaryModel):
 
 
     def clean(self):
     def clean(self):
 
 
+        # Validate data backend type
+        if self.type and self.type not in registry['data_backends']:
+            raise ValidationError({
+                'type': _("Unknown backend type: {type}".format(type=self.type))
+            })
+
         # Ensure URL scheme matches selected type
         # Ensure URL scheme matches selected type
-        if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
+        if self.backend_class.is_local and self.url_scheme not in ('file', ''):
             raise ValidationError({
             raise ValidationError({
                 'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
                 'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
             })
             })

+ 20 - 0
netbox/core/tables/columns.py

@@ -0,0 +1,20 @@
+import django_tables2 as tables
+
+from netbox.registry import registry
+
+__all__ = (
+    'BackendTypeColumn',
+)
+
+
+class BackendTypeColumn(tables.Column):
+    """
+    Display a data backend type.
+    """
+    def render(self, value):
+        if backend := registry['data_backends'].get(value):
+            return backend.label
+        return value
+
+    def value(self, value):
+        return value

+ 5 - 4
netbox/core/tables/data.py

@@ -3,6 +3,7 @@ import django_tables2 as tables
 
 
 from core.models import *
 from core.models import *
 from netbox.tables import NetBoxTable, columns
 from netbox.tables import NetBoxTable, columns
+from .columns import BackendTypeColumn
 
 
 __all__ = (
 __all__ = (
     'DataFileTable',
     'DataFileTable',
@@ -15,8 +16,8 @@ class DataSourceTable(NetBoxTable):
         verbose_name=_('Name'),
         verbose_name=_('Name'),
         linkify=True
         linkify=True
     )
     )
-    type = columns.ChoiceFieldColumn(
-        verbose_name=_('Type'),
+    type = BackendTypeColumn(
+        verbose_name=_('Type')
     )
     )
     status = columns.ChoiceFieldColumn(
     status = columns.ChoiceFieldColumn(
         verbose_name=_('Status'),
         verbose_name=_('Status'),
@@ -34,8 +35,8 @@ class DataSourceTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
     class Meta(NetBoxTable.Meta):
         model = DataSource
         model = DataSource
         fields = (
         fields = (
-            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
-            'last_updated', 'file_count',
+            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
+            'created', 'last_updated', 'file_count',
         )
         )
         default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
         default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
 
 

+ 7 - 8
netbox/core/tests/test_api.py

@@ -2,7 +2,6 @@ from django.urls import reverse
 from django.utils import timezone
 from django.utils import timezone
 
 
 from utilities.testing import APITestCase, APIViewTestCases
 from utilities.testing import APITestCase, APIViewTestCases
-from ..choices import *
 from ..models import *
 from ..models import *
 
 
 
 
@@ -26,26 +25,26 @@ class DataSourceTest(APIViewTestCases.APIViewTestCase):
     @classmethod
     @classmethod
     def setUpTestData(cls):
     def setUpTestData(cls):
         data_sources = (
         data_sources = (
-            DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
-            DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
-            DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
+            DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
+            DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
+            DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
         )
         )
         DataSource.objects.bulk_create(data_sources)
         DataSource.objects.bulk_create(data_sources)
 
 
         cls.create_data = [
         cls.create_data = [
             {
             {
                 'name': 'Data Source 4',
                 'name': 'Data Source 4',
-                'type': DataSourceTypeChoices.GIT,
+                'type': 'git',
                 'source_url': 'https://example.com/git/source4'
                 'source_url': 'https://example.com/git/source4'
             },
             },
             {
             {
                 'name': 'Data Source 5',
                 'name': 'Data Source 5',
-                'type': DataSourceTypeChoices.GIT,
+                'type': 'git',
                 'source_url': 'https://example.com/git/source5'
                 'source_url': 'https://example.com/git/source5'
             },
             },
             {
             {
                 'name': 'Data Source 6',
                 'name': 'Data Source 6',
-                'type': DataSourceTypeChoices.GIT,
+                'type': 'git',
                 'source_url': 'https://example.com/git/source6'
                 'source_url': 'https://example.com/git/source6'
             },
             },
         ]
         ]
@@ -63,7 +62,7 @@ class DataFileTest(
     def setUpTestData(cls):
     def setUpTestData(cls):
         datasource = DataSource.objects.create(
         datasource = DataSource.objects.create(
             name='Data Source 1',
             name='Data Source 1',
-            type=DataSourceTypeChoices.LOCAL,
+            type='local',
             source_url='file:///var/tmp/source1/'
             source_url='file:///var/tmp/source1/'
         )
         )
 
 

+ 7 - 7
netbox/core/tests/test_filtersets.py

@@ -18,21 +18,21 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
         data_sources = (
         data_sources = (
             DataSource(
             DataSource(
                 name='Data Source 1',
                 name='Data Source 1',
-                type=DataSourceTypeChoices.LOCAL,
+                type='local',
                 source_url='file:///var/tmp/source1/',
                 source_url='file:///var/tmp/source1/',
                 status=DataSourceStatusChoices.NEW,
                 status=DataSourceStatusChoices.NEW,
                 enabled=True
                 enabled=True
             ),
             ),
             DataSource(
             DataSource(
                 name='Data Source 2',
                 name='Data Source 2',
-                type=DataSourceTypeChoices.LOCAL,
+                type='local',
                 source_url='file:///var/tmp/source2/',
                 source_url='file:///var/tmp/source2/',
                 status=DataSourceStatusChoices.SYNCING,
                 status=DataSourceStatusChoices.SYNCING,
                 enabled=True
                 enabled=True
             ),
             ),
             DataSource(
             DataSource(
                 name='Data Source 3',
                 name='Data Source 3',
-                type=DataSourceTypeChoices.GIT,
+                type='git',
                 source_url='https://example.com/git/source3',
                 source_url='https://example.com/git/source3',
                 status=DataSourceStatusChoices.COMPLETED,
                 status=DataSourceStatusChoices.COMPLETED,
                 enabled=False
                 enabled=False
@@ -45,7 +45,7 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
 
 
     def test_type(self):
     def test_type(self):
-        params = {'type': [DataSourceTypeChoices.LOCAL]}
+        params = {'type': ['local']}
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
 
 
     def test_enabled(self):
     def test_enabled(self):
@@ -66,9 +66,9 @@ class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
     @classmethod
     @classmethod
     def setUpTestData(cls):
     def setUpTestData(cls):
         data_sources = (
         data_sources = (
-            DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
-            DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
-            DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
+            DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
+            DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
+            DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
         )
         )
         DataSource.objects.bulk_create(data_sources)
         DataSource.objects.bulk_create(data_sources)
 
 

+ 9 - 10
netbox/core/tests/test_views.py

@@ -1,7 +1,6 @@
 from django.utils import timezone
 from django.utils import timezone
 
 
 from utilities.testing import ViewTestCases, create_tags
 from utilities.testing import ViewTestCases, create_tags
-from ..choices import *
 from ..models import *
 from ..models import *
 
 
 
 
@@ -11,9 +10,9 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
     @classmethod
     @classmethod
     def setUpTestData(cls):
     def setUpTestData(cls):
         data_sources = (
         data_sources = (
-            DataSource(name='Data Source 1', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source1/'),
-            DataSource(name='Data Source 2', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source2/'),
-            DataSource(name='Data Source 3', type=DataSourceTypeChoices.LOCAL, source_url='file:///var/tmp/source3/'),
+            DataSource(name='Data Source 1', type='local', source_url='file:///var/tmp/source1/'),
+            DataSource(name='Data Source 2', type='local', source_url='file:///var/tmp/source2/'),
+            DataSource(name='Data Source 3', type='local', source_url='file:///var/tmp/source3/'),
         )
         )
         DataSource.objects.bulk_create(data_sources)
         DataSource.objects.bulk_create(data_sources)
 
 
@@ -21,7 +20,7 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
 
 
         cls.form_data = {
         cls.form_data = {
             'name': 'Data Source X',
             'name': 'Data Source X',
-            'type': DataSourceTypeChoices.GIT,
+            'type': 'git',
             'source_url': 'http:///exmaple/com/foo/bar/',
             'source_url': 'http:///exmaple/com/foo/bar/',
             'description': 'Something',
             'description': 'Something',
             'comments': 'Foo bar baz',
             'comments': 'Foo bar baz',
@@ -29,10 +28,10 @@ class DataSourceTestCase(ViewTestCases.PrimaryObjectViewTestCase):
         }
         }
 
 
         cls.csv_data = (
         cls.csv_data = (
-            f"name,type,source_url,enabled",
-            f"Data Source 4,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
-            f"Data Source 5,{DataSourceTypeChoices.LOCAL},file:///var/tmp/source4/,true",
-            f"Data Source 6,{DataSourceTypeChoices.GIT},http:///exmaple/com/foo/bar/,false",
+            "name,type,source_url,enabled",
+            "Data Source 4,local,file:///var/tmp/source4/,true",
+            "Data Source 5,local,file:///var/tmp/source4/,true",
+            "Data Source 6,git,http:///exmaple/com/foo/bar/,false",
         )
         )
 
 
         cls.csv_update_data = (
         cls.csv_update_data = (
@@ -60,7 +59,7 @@ class DataFileTestCase(
     def setUpTestData(cls):
     def setUpTestData(cls):
         datasource = DataSource.objects.create(
         datasource = DataSource.objects.create(
             name='Data Source 1',
             name='Data Source 1',
-            type=DataSourceTypeChoices.LOCAL,
+            type='local',
             source_url='file:///var/tmp/source1/'
             source_url='file:///var/tmp/source1/'
         )
         )
 
 

+ 53 - 0
netbox/netbox/data_backends.py

@@ -0,0 +1,53 @@
+from contextlib import contextmanager
+from urllib.parse import urlparse
+
+__all__ = (
+    'DataBackend',
+)
+
+
+class DataBackend:
+    """
+    A data backend represents a specific system of record for data, such as a git repository or Amazon S3 bucket.
+
+    Attributes:
+        name: The identifier under which this backend will be registered in NetBox
+        label: The human-friendly name for this backend
+        is_local: A boolean indicating whether this backend accesses local data
+        parameters: A dictionary mapping configuration form field names to their classes
+        sensitive_parameters: An iterable of field names for which the values should not be displayed to the user
+    """
+    is_local = False
+    parameters = {}
+    sensitive_parameters = []
+
+    # Prevent Django's template engine from calling the backend
+    # class when referenced via DataSource.backend_class
+    do_not_call_in_templates = True
+
+    def __init__(self, url, **kwargs):
+        self.url = url
+        self.params = kwargs
+        self.config = self.init_config()
+
+    def init_config(self):
+        """
+        A hook to initialize the instance's configuration. The data returned by this method is assigned to the
+        instance's `config` attribute upon initialization, which can be referenced by the `fetch()` method.
+        """
+        return
+
+    @property
+    def url_scheme(self):
+        return urlparse(self.url).scheme.lower()
+
+    @contextmanager
+    def fetch(self):
+        """
+        A context manager which performs the following:
+
+        1. Handles all setup and synchronization
+        2. Yields the local path at which data has been replicated
+        3. Performs any necessary cleanup
+        """
+        raise NotImplemented()

+ 8 - 0
netbox/netbox/plugins/__init__.py

@@ -8,6 +8,7 @@ from packaging import version
 
 
 from netbox.registry import registry
 from netbox.registry import registry
 from netbox.search import register_search
 from netbox.search import register_search
+from netbox.utils import register_data_backend
 from .navigation import *
 from .navigation import *
 from .registration import *
 from .registration import *
 from .templates import *
 from .templates import *
@@ -24,6 +25,7 @@ registry['plugins'].update({
 
 
 DEFAULT_RESOURCE_PATHS = {
 DEFAULT_RESOURCE_PATHS = {
     'search_indexes': 'search.indexes',
     'search_indexes': 'search.indexes',
+    'data_backends': 'data_backends.backends',
     'graphql_schema': 'graphql.schema',
     'graphql_schema': 'graphql.schema',
     'menu': 'navigation.menu',
     'menu': 'navigation.menu',
     'menu_items': 'navigation.menu_items',
     'menu_items': 'navigation.menu_items',
@@ -70,6 +72,7 @@ class PluginConfig(AppConfig):
 
 
     # Optional plugin resources
     # Optional plugin resources
     search_indexes = None
     search_indexes = None
+    data_backends = None
     graphql_schema = None
     graphql_schema = None
     menu = None
     menu = None
     menu_items = None
     menu_items = None
@@ -98,6 +101,11 @@ class PluginConfig(AppConfig):
         for idx in search_indexes:
         for idx in search_indexes:
             register_search(idx)
             register_search(idx)
 
 
+        # Register data backends (if defined)
+        data_backends = self._load_resource('data_backends') or []
+        for backend in data_backends:
+            register_data_backend()(backend)
+
         # Register template content (if defined)
         # Register template content (if defined)
         if template_extensions := self._load_resource('template_extensions'):
         if template_extensions := self._load_resource('template_extensions'):
             register_template_extensions(template_extensions)
             register_template_extensions(template_extensions)

+ 18 - 0
netbox/netbox/tests/dummy_plugin/data_backends.py

@@ -0,0 +1,18 @@
+from contextlib import contextmanager
+
+from netbox.data_backends import DataBackend
+
+
+class DummyBackend(DataBackend):
+    name = 'dummy'
+    label = 'Dummy'
+    is_local = True
+
+    @contextmanager
+    def fetch(self):
+        yield '/tmp'
+
+
+backends = (
+    DummyBackend,
+)

+ 8 - 0
netbox/netbox/tests/test_plugins.py

@@ -6,6 +6,7 @@ from django.test import Client, TestCase, override_settings
 from django.urls import reverse
 from django.urls import reverse
 
 
 from netbox.tests.dummy_plugin import config as dummy_config
 from netbox.tests.dummy_plugin import config as dummy_config
+from netbox.tests.dummy_plugin.data_backends import DummyBackend
 from netbox.plugins.navigation import PluginMenu
 from netbox.plugins.navigation import PluginMenu
 from netbox.plugins.utils import get_plugin_config
 from netbox.plugins.utils import get_plugin_config
 from netbox.graphql.schema import Query
 from netbox.graphql.schema import Query
@@ -111,6 +112,13 @@ class PluginTest(TestCase):
         """
         """
         self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
         self.assertIn('netbox.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
 
 
+    def test_data_backends(self):
+        """
+        Check registered data backends.
+        """
+        self.assertIn('dummy', registry['data_backends'])
+        self.assertIs(registry['data_backends']['dummy'], DummyBackend)
+
     def test_queues(self):
     def test_queues(self):
         """
         """
         Check that plugin queues are registered with the accurate name.
         Check that plugin queues are registered with the accurate name.

+ 26 - 0
netbox/netbox/utils.py

@@ -0,0 +1,26 @@
+from netbox.registry import registry
+
+__all__ = (
+    'get_data_backend_choices',
+    'register_data_backend',
+)
+
+
+def get_data_backend_choices():
+    return [
+        (None, '---------'),
+        *[
+            (name, cls.label) for name, cls in registry['data_backends'].items()
+        ]
+    ]
+
+
+def register_data_backend():
+    """
+    Decorator for registering a DataBackend class.
+    """
+    def _wrapper(cls):
+        registry['data_backends'][cls.name] = cls
+        return cls
+
+    return _wrapper

+ 1 - 1
netbox/templates/core/datasource.html

@@ -58,7 +58,7 @@
             <tr>
             <tr>
               <th scope="row">{% trans "URL" %}</th>
               <th scope="row">{% trans "URL" %}</th>
               <td>
               <td>
-                {% if not object.is_local %}
+                {% if not object.type.is_local %}
                   <a href="{{ object.source_url }}">{{ object.source_url }}</a>
                   <a href="{{ object.source_url }}">{{ object.source_url }}</a>
                 {% else %}
                 {% else %}
                   {{ object.source_url }}
                   {{ object.source_url }}