Closes #18287: Enable periodic synchronization for data sources (#18747)

* Add sync_interval to DataSource

* Enqueue a SyncDataSourceJob when needed after saving a DataSource

* Fix logic for clearing pending jobs on interval change

* Fix lingering background tasks after modifying DataSource

docs/models/core/datasource.md

@@ -44,6 +44,12 @@ A set of rules (one per line) identifying filenames to ignore during synchroniza
 | `*.txt`        | Ignore any files with a `.txt` extension |
 | `data???.json` | Ignore e.g. `data123.json`               |
 
+### Sync Interval
+
+!!! info "This field was introduced in NetBox v4.3."
+
+The interval at which the data source should automatically synchronize. If not set, the data source must be synchronized manually.
+
 ### Last Synced
 
 The date and time at which the source was most recently synchronized successfully.

netbox/core/api/serializers_/data.py

@@ -26,8 +26,8 @@ class DataSourceSerializer(NetBoxModelSerializer):
         model = DataSource
         fields = [
             'id', 'url', 'display_url', 'display', 'name', 'type', 'source_url', 'enabled', 'status', 'description',
-            'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated', 'last_synced',
-            'file_count',
+            'sync_interval', 'parameters', 'ignore_rules', 'comments', 'custom_fields', 'created', 'last_updated',
+            'last_synced', 'file_count',
         ]
         brief_fields = ('id', 'url', 'display', 'name', 'description')
 

netbox/core/filtersets.py

@@ -29,6 +29,10 @@ class DataSourceFilterSet(NetBoxModelFilterSet):
         choices=DataSourceStatusChoices,
         null_value=None
     )
+    sync_interval = django_filters.MultipleChoiceFilter(
+        choices=JobIntervalChoices,
+        null_value=None
+    )
 
     class Meta:
         model = DataSource

netbox/core/forms/bulk_edit.py

@@ -1,6 +1,7 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 
+from core.choices import JobIntervalChoices
 from core.models import *
 from netbox.forms import NetBoxModelBulkEditForm
 from netbox.utils import get_data_backend_choices
@@ -29,6 +30,11 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
         max_length=200,
         required=False
     )
+    sync_interval = forms.ChoiceField(
+        choices=JobIntervalChoices,
+        required=False,
+        label=_('Sync interval')
+    )
     comments = CommentField()
     parameters = forms.JSONField(
         label=_('Parameters'),
@@ -42,8 +48,8 @@ class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
 
     model = DataSource
     fieldsets = (
-        FieldSet('type', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules'),
+        FieldSet('type', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules', 'comments'),
     )
     nullable_fields = (
-        'description', 'description', 'parameters', 'comments', 'parameters', 'ignore_rules',
+        'description', 'description', 'sync_interval', 'parameters', 'parameters', 'ignore_rules' 'comments',
     )

netbox/core/forms/bulk_import.py

@@ -11,5 +11,6 @@ class DataSourceImportForm(NetBoxModelImportForm):
     class Meta:
         model = DataSource
         fields = (
-            'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'parameters', 'ignore_rules',
+            'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'parameters', 'ignore_rules',
+            'comments',
         )

netbox/core/forms/filtersets.py

@@ -27,7 +27,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
     model = DataSource
     fieldsets = (
         FieldSet('q', 'filter_id'),
-        FieldSet('type', 'status', name=_('Data Source')),
+        FieldSet('type', 'status', 'enabled', 'sync_interval', name=_('Data Source')),
     )
     type = forms.MultipleChoiceField(
         label=_('Type'),
@@ -46,6 +46,11 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
             choices=BOOLEAN_WITH_BLANK_CHOICES
         )
     )
+    sync_interval = forms.ChoiceField(
+        label=_('Sync interval'),
+        choices=JobIntervalChoices,
+        required=False
+    )
 
 
 class DataFileFilterForm(NetBoxModelFilterSetForm):

netbox/core/forms/model_forms.py

@@ -36,7 +36,7 @@ class DataSourceForm(NetBoxModelForm):
     class Meta:
         model = DataSource
         fields = [
-            'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
+            'name', 'type', 'source_url', 'enabled', 'description', 'sync_interval', 'ignore_rules', 'comments', 'tags',
         ]
         widgets = {
             'ignore_rules': forms.Textarea(
@@ -51,7 +51,10 @@ class DataSourceForm(NetBoxModelForm):
     @property
     def fieldsets(self):
         fieldsets = [
-            FieldSet('name', 'type', 'source_url', 'enabled', 'description', 'tags', 'ignore_rules', name=_('Source')),
+            FieldSet(
+                'name', 'type', 'source_url', 'description', 'tags', 'ignore_rules', name=_('Source')
+            ),
+            FieldSet('enabled', 'sync_interval', name=_('Sync')),
         ]
         if self.backend_fields:
             fieldsets.append(

netbox/core/migrations/0013_datasource_sync_interval.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.6 on 2025-02-26 19:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0012_job_object_type_optional'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='datasource',
+            name='sync_interval',
+            field=models.PositiveSmallIntegerField(blank=True, null=True),
+        ),
+    ]

netbox/core/models/data.py

@@ -59,6 +59,12 @@ class DataSource(JobsMixin, PrimaryModel):
         verbose_name=_('enabled'),
         default=True
     )
+    sync_interval = models.PositiveSmallIntegerField(
+        verbose_name=_('sync interval'),
+        choices=JobIntervalChoices,
+        blank=True,
+        null=True
+    )
     ignore_rules = models.TextField(
         verbose_name=_('ignore rules'),
         blank=True,

netbox/core/signals.py

@@ -8,16 +8,15 @@ from django.dispatch import receiver, Signal
 from django.utils.translation import gettext_lazy as _
 from django_prometheus.models import model_deletes, model_inserts, model_updates
 
-from core.choices import ObjectChangeActionChoices
+from core.choices import JobStatusChoices, ObjectChangeActionChoices
 from core.events import *
-from core.models import ObjectChange
 from extras.events import enqueue_event
 from extras.utils import run_validators
 from netbox.config import get_config
 from netbox.context import current_request, events_queue
 from netbox.models.features import ChangeLoggingMixin
 from utilities.exceptions import AbortRequest
-from .models import ConfigRevision
+from .models import ConfigRevision, DataSource, ObjectChange
 
 __all__ = (
     'clear_events',
@@ -182,6 +181,25 @@ def clear_events_queue(sender, **kwargs):
 # DataSource handlers
 #
 
+@receiver(post_save, sender=DataSource)
+def enqueue_sync_job(instance, created, **kwargs):
+    """
+    When a DataSource is saved, check its sync_interval and enqueue a sync job if appropriate.
+    """
+    from .jobs import SyncDataSourceJob
+
+    if instance.enabled and instance.sync_interval:
+        SyncDataSourceJob.enqueue_once(instance, interval=instance.sync_interval)
+    elif not created:
+        # Delete any previously scheduled recurring jobs for this DataSource
+        for job in SyncDataSourceJob.get_jobs(instance).defer('data').filter(
+            interval__isnull=False,
+            status=JobStatusChoices.STATUS_SCHEDULED
+        ):
+            # Call delete() per instance to ensure the associated background task is deleted as well
+            job.delete()
+
+
 @receiver(post_sync)
 def auto_sync(instance, **kwargs):
     """

netbox/core/tables/data.py

@@ -25,6 +25,9 @@ class DataSourceTable(NetBoxTable):
     enabled = columns.BooleanColumn(
         verbose_name=_('Enabled'),
     )
+    sync_interval = columns.ChoiceFieldColumn(
+        verbose_name=_('Sync interval'),
+    )
     tags = columns.TagColumn(
         url_name='core:datasource_list'
     )
@@ -35,10 +38,10 @@ class DataSourceTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = DataSource
         fields = (
-            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
-            'created', 'last_updated', 'file_count',
+            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'sync_interval', 'comments',
+            'parameters', 'created', 'last_updated', 'file_count',
         )
-        default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')
+        default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'sync_interval', 'file_count')
 
 
 class DataFileTable(NetBoxTable):

netbox/core/tests/test_filtersets.py

@@ -27,7 +27,8 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
                 source_url='file:///var/tmp/source1/',
                 status=DataSourceStatusChoices.NEW,
                 enabled=True,
-                description='foobar1'
+                description='foobar1',
+                sync_interval=JobIntervalChoices.INTERVAL_HOURLY
             ),
             DataSource(
                 name='Data Source 2',
@@ -35,14 +36,16 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
                 source_url='file:///var/tmp/source2/',
                 status=DataSourceStatusChoices.SYNCING,
                 enabled=True,
-                description='foobar2'
+                description='foobar2',
+                sync_interval=JobIntervalChoices.INTERVAL_DAILY
             ),
             DataSource(
                 name='Data Source 3',
                 type='git',
                 source_url='https://example.com/git/source3',
                 status=DataSourceStatusChoices.COMPLETED,
-                enabled=False
+                enabled=False,
+                sync_interval=JobIntervalChoices.INTERVAL_WEEKLY
             ),
         )
         DataSource.objects.bulk_create(data_sources)
@@ -73,6 +76,10 @@ class DataSourceTestCase(TestCase, ChangeLoggedFilterSetTests):
         params = {'status': [DataSourceStatusChoices.NEW, DataSourceStatusChoices.SYNCING]}
         self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
 
+    def test_sync_interval(self):
+        params = {'sync_interval': [JobIntervalChoices.INTERVAL_HOURLY, JobIntervalChoices.INTERVAL_DAILY]}
+        self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2)
+
 
 class DataFileTestCase(TestCase, ChangeLoggedFilterSetTests):
     queryset = DataFile.objects.all()

netbox/templates/core/datasource.html

@@ -46,6 +46,10 @@
             <th scope="row">{% trans "Status" %}</th>
             <td>{% badge object.get_status_display bg_color=object.get_status_color %}</td>
           </tr>
+          <tr>
+            <th scope="row">{% trans "Sync interval" %}</th>
+            <td>{{ object.get_sync_interval_display|placeholder }}</td>
+          </tr>
           <tr>
             <th scope="row">{% trans "Last synced" %}</th>
             <td>{{ object.last_synced|placeholder }}</td>