Adds rq retry options (#12588)

* adds rq retry options #12327

* Clean up docs; disable retries of failed jobs by default

* Pass a Retry object only if RQ_RETRY_MAX is non-zero

---------

Co-authored-by: jeremystretch <jstretch@netboxlabs.com>

docs/configuration/miscellaneous.md

@@ -204,3 +204,25 @@ This parameter defines the URL of the repository that will be checked for new Ne
 Default: `300`
 
 The maximum execution time of a background task (such as running a custom script), in seconds.
+
+---
+
+## RQ_RETRY_INTERVAL
+
+!!! note
+    This parameter was added in NetBox v3.5.
+
+Default: `60`
+
+This parameter controls how frequently a failed job is retried, up to the maximum number of times specified by `RQ_RETRY_MAX`. This must be either an integer specifying the number of seconds to wait between successive attempts, or a list of such values. For example, `[60, 300, 3600]` will retry the task after 1 minute, 5 minutes, and 1 hour.
+
+---
+
+## RQ_RETRY_MAX
+
+!!! note
+    This parameter was added in NetBox v3.5.
+
+Default: `0` (retries disabled)
+
+The maximum number of times a background task will be retried before being marked as failed.

netbox/core/models/jobs.py

@@ -16,7 +16,7 @@ from extras.utils import FeatureQuery
 from netbox.config import get_config
 from netbox.constants import RQ_QUEUE_DEFAULT
 from utilities.querysets import RestrictedQuerySet
-from utilities.rqworker import get_queue_for_model
+from utilities.rqworker import get_queue_for_model, get_rq_retry
 
 __all__ = (
     'Job',
@@ -219,5 +219,6 @@ class Job(models.Model):
                 event=event,
                 data=self.data,
                 timestamp=str(timezone.now()),
-                username=self.user.username
+                username=self.user.username,
+                retry=get_rq_retry()
             )

netbox/extras/webhooks.py

@@ -9,6 +9,7 @@ from netbox.config import get_config
 from netbox.constants import RQ_QUEUE_DEFAULT
 from netbox.registry import registry
 from utilities.api import get_serializer_for_model
+from utilities.rqworker import get_rq_retry
 from utilities.utils import serialize_object
 from .choices import *
 from .models import Webhook
@@ -116,5 +117,6 @@ def flush_webhooks(queue):
                 snapshots=data['snapshots'],
                 timestamp=str(timezone.now()),
                 username=data['username'],
-                request_id=data['request_id']
+                request_id=data['request_id'],
+                retry=get_rq_retry()
             )

netbox/netbox/settings.py

@@ -140,6 +140,8 @@ REMOTE_AUTH_STAFF_USERS = getattr(configuration, 'REMOTE_AUTH_STAFF_USERS', [])
 REMOTE_AUTH_GROUP_SEPARATOR = getattr(configuration, 'REMOTE_AUTH_GROUP_SEPARATOR', '|')
 REPORTS_ROOT = getattr(configuration, 'REPORTS_ROOT', os.path.join(BASE_DIR, 'reports')).rstrip('/')
 RQ_DEFAULT_TIMEOUT = getattr(configuration, 'RQ_DEFAULT_TIMEOUT', 300)
+RQ_RETRY_INTERVAL = getattr(configuration, 'RQ_RETRY_INTERVAL', 60)
+RQ_RETRY_MAX = getattr(configuration, 'RQ_RETRY_MAX', 0)
 SCRIPTS_ROOT = getattr(configuration, 'SCRIPTS_ROOT', os.path.join(BASE_DIR, 'scripts')).rstrip('/')
 SEARCH_BACKEND = getattr(configuration, 'SEARCH_BACKEND', 'netbox.search.backends.CachedValueSearchBackend')
 SECURE_SSL_REDIRECT = getattr(configuration, 'SECURE_SSL_REDIRECT', False)

netbox/utilities/rqworker.py

@@ -1,11 +1,12 @@
 from django_rq.queues import get_connection
-from rq import Worker
+from rq import Retry, Worker
 
 from netbox.config import get_config
 from netbox.constants import RQ_QUEUE_DEFAULT
 
 __all__ = (
     'get_queue_for_model',
+    'get_rq_retry',
     'get_workers_for_queue',
 )
 
@@ -22,3 +23,14 @@ def get_workers_for_queue(queue_name):
     Returns True if a worker process is currently servicing the specified queue.
     """
     return Worker.count(get_connection(queue_name))
+
+
+def get_rq_retry():
+    """
+    If RQ_RETRY_MAX is defined and greater than zero, instantiate and return a Retry object to be
+    used when queuing a job. Otherwise, return None.
+    """
+    retry_max = get_config().RQ_RETRY_MAX
+    retry_interval = get_config().RQ_RETRY_INTERVAL
+    if retry_max:
+        return Retry(max=retry_max, interval=retry_interval)