Merge pull request #6731 from maximumG/6651-plugins-rq-queues

Fixes #6651:  Add plugin's queueing system

contrib/netbox-rq.service

@@ -11,7 +11,7 @@ User=netbox
 Group=netbox
 WorkingDirectory=/opt/netbox
 
-ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker
+ExecStart=/opt/netbox/venv/bin/python3 /opt/netbox/netbox/manage.py rqworker high default low
 
 Restart=on-failure
 RestartSec=30

docs/plugins/development.md

@@ -383,4 +383,32 @@ class SiteAnimalCount(PluginTemplateExtension):
         })
 
 template_extensions = [SiteAnimalCount]
-```
+```
+
+## Background Tasks
+
+By default, Netbox provides 3 differents [RQ](https://python-rq.org/) queues to run background jobs : *high*, *default* and *low*.
+These 3 core queues can be used out-of-the-box by plugins to define background tasks.
+
+Plugins can also define dedicated queues. These queues can be configured under the PluginConfig class `queues` attribute. An example configuration
+is below:
+
+```python
+class MyPluginConfig(PluginConfig):
+    name = 'myplugin'
+    ...
+    queues = [
+        'queue1',
+        'queue2',
+        'queue-whatever-the-name'
+    ]
+```
+
+The PluginConfig above creates 3 queues with the following names: *myplugin.queue1*, *myplugin.queue2*, *myplugin.queue-whatever-the-name*.
+As you can see, the queue's name is always preprended with the plugin's name, to avoid any name clashes between different plugins.
+
+In case you create dedicated queues for your plugin, it is strongly advised to also create a dedicated RQ worker instance. This instance should only listen to the queues defined in your plugin - to avoid impact between your background tasks and netbox internal tasks.
+
+```
+python manage.py rqworker myplugin.queue1 myplugin.queue2 myplugin.queue-whatever-the-name
+```

netbox/extras/plugins/__init__.py

@@ -47,6 +47,9 @@ class PluginConfig(AppConfig):
     # Middleware classes provided by the plugin
     middleware = []
 
+    # Django-rq queues dedicated to the plugin
+    queues = []
+
     # Default integration paths. Plugin authors can override these to customize the paths to
     # integrated components.
     template_extensions = 'template_content.template_extensions'

netbox/extras/tests/dummy_plugin/__init__.py

@@ -12,6 +12,11 @@ class DummyPluginConfig(PluginConfig):
     middleware = [
         'extras.tests.dummy_plugin.middleware.DummyMiddleware'
     ]
+    queues = [
+        'testing-low',
+        'testing-medium',
+        'testing-high'
+    ]
 
 
 config = DummyPluginConfig

netbox/extras/tests/test_plugins.py

@@ -80,6 +80,14 @@ class PluginTest(TestCase):
         """
         self.assertIn('extras.tests.dummy_plugin.middleware.DummyMiddleware', settings.MIDDLEWARE)
 
+    def test_queues(self):
+        """
+        Check that plugin queues are registered with the accurate name.
+        """
+        self.assertIn('extras.tests.dummy_plugin.testing-low', settings.RQ_QUEUES)
+        self.assertIn('extras.tests.dummy_plugin.testing-medium', settings.RQ_QUEUES)
+        self.assertIn('extras.tests.dummy_plugin.testing-high', settings.RQ_QUEUES)
+
     def test_min_version(self):
         """
         Check enforcement of minimum NetBox version.

netbox/netbox/settings.py

@@ -546,7 +546,9 @@ else:
     }
 
 RQ_QUEUES = {
+    'high': RQ_PARAMS,
     'default': RQ_PARAMS,
+    'low': RQ_PARAMS,
 }
 
 
@@ -599,3 +601,14 @@ for plugin_name in PLUGINS:
     plugin_middleware = plugin_config.middleware
     if plugin_middleware and type(plugin_middleware) in (list, tuple):
         MIDDLEWARE.extend(plugin_middleware)
+
+    # Create RQ queues dedicated to the plugin
+    # we use the plugin name as a prefix for queue name's defined in the plugin config
+    # ex: mysuperplugin.mysuperqueue1
+    if type(plugin_config.queues) is not list:
+        raise ImproperlyConfigured(
+            "Plugin {} queues must be a list.".format(plugin_name)
+        )
+    RQ_QUEUES.update({
+        f"{plugin_name}.{queue}": RQ_PARAMS for queue in plugin_config.queues
+    })