14729 Move background tasks list from admin UI to Primary UI (#14825)

* 14729 rq table

* 14729 rq table

* 14729 rq table

* 14729 rq table

* 14729 jobs table

* 14729 jobs detail

* 14729 formatting fixup

* 14729 formatting fixup

* 14729 format datetime in tables

* 14729 display job id

* Update templates for #12128

* 14729 review fixes

* 14729 review fixes

* 14729 review fixes

* 14729 review fixes

* 14729 merge feature

* 14729 add modal

* 14729 review changes

* 14729 url fixup

* 14729 no queue param on task

* 14729 queue pages

* 14729 job status handling

* 14729 worker list

* 14729 exec detail and common view

* 14729 worker detail

* 14729 background task delete

* 14729 background task delete

* 14729 background task requeue

* 14729 background task enqueue stop

* 14729 review changes

* 14729 remove rq from admin

* 14729 add tests

* 14729 add tests

* Clean up HTML templates

* Clean up tables

* Clean up views

* Fix tests

* Clean up tests

* Move navigation menu entry for background tasks

* Remove custom deletion form

---------

Co-authored-by: Jeremy Stretch <jstretch@netboxlabs.com>

netbox/core/constants.py

@@ -0,0 +1,26 @@
+from dataclasses import dataclass
+
+from django.utils.translation import gettext_lazy as _
+from rq.job import JobStatus
+
+__all__ = (
+    'RQ_TASK_STATUSES',
+)
+
+
+@dataclass
+class Status:
+    label: str
+    color: str
+
+
+RQ_TASK_STATUSES = {
+    JobStatus.QUEUED: Status(_('Queued'), 'cyan'),
+    JobStatus.FINISHED: Status(_('Finished'), 'green'),
+    JobStatus.FAILED: Status(_('Failed'), 'red'),
+    JobStatus.STARTED: Status(_('Started'), 'blue'),
+    JobStatus.DEFERRED: Status(_('Deferred'), 'gray'),
+    JobStatus.SCHEDULED: Status(_('Scheduled'), 'purple'),
+    JobStatus.STOPPED: Status(_('Stopped'), 'orange'),
+    JobStatus.CANCELED: Status(_('Cancelled'), 'yellow'),
+}

netbox/core/tables/__init__.py

@@ -1,4 +1,5 @@
 from .config import *
 from .data import *
 from .jobs import *
+from .tasks import *
 from .plugins import *

netbox/core/tables/columns.py

@@ -1,9 +1,12 @@
 import django_tables2 as tables
+from django.utils.safestring import mark_safe
 
+from core.constants import RQ_TASK_STATUSES
 from netbox.registry import registry
 
 __all__ = (
     'BackendTypeColumn',
+    'RQJobStatusColumn',
 )
 
 
@@ -18,3 +21,16 @@ class BackendTypeColumn(tables.Column):
 
     def value(self, value):
         return value
+
+
+class RQJobStatusColumn(tables.Column):
+    """
+    Render a colored label for the status of an RQ job.
+    """
+    def render(self, value):
+        status = RQ_TASK_STATUSES.get(value)
+        return mark_safe(f'<span class="badge text-bg-{status.color}">{status.label}</span>')
+
+    def value(self, value):
+        status = RQ_TASK_STATUSES.get(value)
+        return status.label

netbox/core/tables/tasks.py

@@ -0,0 +1,134 @@
+import django_tables2 as tables
+from django.utils.translation import gettext_lazy as _
+from django_tables2.utils import A
+
+from core.tables.columns import RQJobStatusColumn
+from netbox.tables import BaseTable
+
+
+class BackgroundQueueTable(BaseTable):
+    name = tables.Column(
+        verbose_name=_("Name")
+    )
+    jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "queued"]),
+        verbose_name=_("Queued")
+    )
+    oldest_job_timestamp = tables.Column(
+        verbose_name=_("Oldest Task")
+    )
+    started_jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "started"]),
+        verbose_name=_("Active")
+    )
+    deferred_jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "deferred"]),
+        verbose_name=_("Deferred")
+    )
+    finished_jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "finished"]),
+        verbose_name=_("Finished")
+    )
+    failed_jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "failed"]),
+        verbose_name=_("Failed")
+    )
+    scheduled_jobs = tables.Column(
+        linkify=("core:background_task_list", [A("index"), "scheduled"]),
+        verbose_name=_("Scheduled")
+    )
+    workers = tables.Column(
+        linkify=("core:worker_list", [A("index")]),
+        verbose_name=_("Workers")
+    )
+    host = tables.Column(
+        accessor="connection_kwargs__host",
+        verbose_name=_("Host")
+    )
+    port = tables.Column(
+        accessor="connection_kwargs__port",
+        verbose_name=_("Port")
+    )
+    db = tables.Column(
+        accessor="connection_kwargs__db",
+        verbose_name=_("DB")
+    )
+    pid = tables.Column(
+        accessor="scheduler__pid",
+        verbose_name=_("Scheduler PID")
+    )
+
+    class Meta(BaseTable.Meta):
+        empty_text = _('No queues found')
+        fields = (
+            'name', 'jobs', 'oldest_job_timestamp', 'started_jobs', 'deferred_jobs', 'finished_jobs', 'failed_jobs',
+            'scheduled_jobs', 'workers', 'host', 'port', 'db', 'pid',
+        )
+        default_columns = (
+            'name', 'jobs', 'started_jobs', 'deferred_jobs', 'finished_jobs', 'failed_jobs', 'scheduled_jobs',
+            'workers',
+        )
+
+
+class BackgroundTaskTable(BaseTable):
+    id = tables.Column(
+        linkify=("core:background_task", [A("id")]),
+        verbose_name=_("ID")
+    )
+    created_at = tables.DateTimeColumn(
+        verbose_name=_("Created")
+    )
+    enqueued_at = tables.DateTimeColumn(
+        verbose_name=_("Enqueued")
+    )
+    ended_at = tables.DateTimeColumn(
+        verbose_name=_("Ended")
+    )
+    status = RQJobStatusColumn(
+        verbose_name=_("Status"),
+        accessor='get_status'
+    )
+    callable = tables.Column(
+        empty_values=(),
+        verbose_name=_("Callable")
+    )
+
+    class Meta(BaseTable.Meta):
+        empty_text = _('No tasks found')
+        fields = (
+            'id', 'created_at', 'enqueued_at', 'ended_at', 'status', 'callable',
+        )
+        default_columns = (
+            'id', 'created_at', 'enqueued_at', 'ended_at', 'status', 'callable',
+        )
+
+    def render_callable(self, value, record):
+        try:
+            return record.func_name
+        except Exception as e:
+            return repr(e)
+
+
+class WorkerTable(BaseTable):
+    name = tables.Column(
+        linkify=("core:worker", [A("name")]),
+        verbose_name=_("Name")
+    )
+    state = tables.Column(
+        verbose_name=_("State")
+    )
+    birth_date = tables.DateTimeColumn(
+        verbose_name=_("Birth")
+    )
+    pid = tables.Column(
+        verbose_name=_("PID")
+    )
+
+    class Meta(BaseTable.Meta):
+        empty_text = _('No workers found')
+        fields = (
+            'name', 'state', 'birth_date', 'pid',
+        )
+        default_columns = (
+            'name', 'state', 'birth_date', 'pid',
+        )

netbox/core/tests/test_views.py

@@ -1,6 +1,16 @@
+import logging
+import uuid
+from datetime import datetime
+
+from django.urls import reverse
 from django.utils import timezone
+from django_rq import get_queue
+from django_rq.settings import QUEUES_MAP
+from django_rq.workers import get_worker
+from rq.job import Job as RQ_Job, JobStatus
+from rq.registry import DeferredJobRegistry, FailedJobRegistry, FinishedJobRegistry, StartedJobRegistry
 
-from utilities.testing import ViewTestCases, create_tags
+from utilities.testing import TestCase, ViewTestCases, create_tags
 from ..models import *
 
 
@@ -87,3 +97,211 @@ class DataFileTestCase(
             ),
         )
         DataFile.objects.bulk_create(data_files)
+
+
+class BackgroundTaskTestCase(TestCase):
+    user_permissions = ()
+
+    # Dummy worker functions
+    @staticmethod
+    def dummy_job_default():
+        return "Job finished"
+
+    @staticmethod
+    def dummy_job_high():
+        return "Job finished"
+
+    @staticmethod
+    def dummy_job_failing():
+        raise Exception("Job failed")
+
+    def setUp(self):
+        super().setUp()
+        self.user.is_staff = True
+        self.user.is_active = True
+        self.user.save()
+
+        # Clear all queues prior to running each test
+        get_queue('default').connection.flushall()
+        get_queue('high').connection.flushall()
+        get_queue('low').connection.flushall()
+
+    def test_background_queue_list(self):
+        url = reverse('core:background_queue_list')
+
+        # Attempt to load view without permission
+        self.user.is_staff = False
+        self.user.save()
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 403)
+
+        # Load view with permission
+        self.user.is_staff = True
+        self.user.save()
+        response = self.client.get(url)
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('default', str(response.content))
+        self.assertIn('high', str(response.content))
+        self.assertIn('low', str(response.content))
+
+    def test_background_tasks_list_default(self):
+        queue = get_queue('default')
+        queue.enqueue(self.dummy_job_default)
+        queue_index = QUEUES_MAP['default']
+
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'queued']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content))
+
+    def test_background_tasks_list_high(self):
+        queue = get_queue('high')
+        queue.enqueue(self.dummy_job_high)
+        queue_index = QUEUES_MAP['high']
+
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'queued']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_high', str(response.content))
+
+    def test_background_tasks_list_finished(self):
+        queue = get_queue('default')
+        job = queue.enqueue(self.dummy_job_default)
+        queue_index = QUEUES_MAP['default']
+
+        registry = FinishedJobRegistry(queue.name, queue.connection)
+        registry.add(job, 2)
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'finished']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content))
+
+    def test_background_tasks_list_failed(self):
+        queue = get_queue('default')
+        job = queue.enqueue(self.dummy_job_default)
+        queue_index = QUEUES_MAP['default']
+
+        registry = FailedJobRegistry(queue.name, queue.connection)
+        registry.add(job, 2)
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'failed']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content))
+
+    def test_background_tasks_scheduled(self):
+        queue = get_queue('default')
+        queue.enqueue_at(datetime.now(), self.dummy_job_default)
+        queue_index = QUEUES_MAP['default']
+
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'scheduled']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content))
+
+    def test_background_tasks_list_deferred(self):
+        queue = get_queue('default')
+        job = queue.enqueue(self.dummy_job_default)
+        queue_index = QUEUES_MAP['default']
+
+        registry = DeferredJobRegistry(queue.name, queue.connection)
+        registry.add(job, 2)
+        response = self.client.get(reverse('core:background_task_list', args=[queue_index, 'deferred']))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('BackgroundTaskTestCase.dummy_job_default', str(response.content))
+
+    def test_background_task(self):
+        queue = get_queue('default')
+        job = queue.enqueue(self.dummy_job_default)
+
+        response = self.client.get(reverse('core:background_task', args=[job.id]))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn('Background Tasks', str(response.content))
+        self.assertIn(str(job.id), str(response.content))
+        self.assertIn('Callable', str(response.content))
+        self.assertIn('Meta', str(response.content))
+        self.assertIn('Keyword Arguments', str(response.content))
+
+    def test_background_task_delete(self):
+        queue = get_queue('default')
+        job = queue.enqueue(self.dummy_job_default)
+
+        response = self.client.post(reverse('core:background_task_delete', args=[job.id]), {'confirm': True})
+        self.assertEqual(response.status_code, 302)
+        self.assertFalse(RQ_Job.exists(job.id, connection=queue.connection))
+        self.assertNotIn(job.id, queue.job_ids)
+
+    def test_background_task_requeue(self):
+        queue = get_queue('default')
+
+        # Enqueue & run a job that will fail
+        job = queue.enqueue(self.dummy_job_failing)
+        worker = get_worker('default')
+        worker.work(burst=True)
+        self.assertTrue(job.is_failed)
+
+        # Re-enqueue the failed job and check that its status has been reset
+        response = self.client.get(reverse('core:background_task_requeue', args=[job.id]))
+        self.assertEqual(response.status_code, 302)
+        self.assertFalse(job.is_failed)
+
+    def test_background_task_enqueue(self):
+        queue = get_queue('default')
+
+        # Enqueue some jobs that each depends on its predecessor
+        job = previous_job = None
+        for _ in range(0, 3):
+            job = queue.enqueue(self.dummy_job_default, depends_on=previous_job)
+            previous_job = job
+
+        # Check that the last job to be enqueued has a status of deferred
+        self.assertIsNotNone(job)
+        self.assertEqual(job.get_status(), JobStatus.DEFERRED)
+        self.assertIsNone(job.enqueued_at)
+
+        # Force-enqueue the deferred job
+        response = self.client.get(reverse('core:background_task_enqueue', args=[job.id]))
+        self.assertEqual(response.status_code, 302)
+
+        # Check that job's status is updated correctly
+        job = queue.fetch_job(job.id)
+        self.assertEqual(job.get_status(), JobStatus.QUEUED)
+        self.assertIsNotNone(job.enqueued_at)
+
+    def test_background_task_stop(self):
+        queue = get_queue('default')
+
+        worker = get_worker('default')
+        job = queue.enqueue(self.dummy_job_default)
+        worker.prepare_job_execution(job)
+
+        self.assertEqual(job.get_status(), JobStatus.STARTED)
+
+        # Stop those jobs using the view
+        started_job_registry = StartedJobRegistry(queue.name, connection=queue.connection)
+        self.assertEqual(len(started_job_registry), 1)
+        response = self.client.get(reverse('core:background_task_stop', args=[job.id]))
+        self.assertEqual(response.status_code, 302)
+        worker.monitor_work_horse(job, queue)  # Sets the job as Failed and removes from Started
+        self.assertEqual(len(started_job_registry), 0)
+
+        canceled_job_registry = FailedJobRegistry(queue.name, connection=queue.connection)
+        self.assertEqual(len(canceled_job_registry), 1)
+        self.assertIn(job.id, canceled_job_registry)
+
+    def test_worker_list(self):
+        worker1 = get_worker('default', name=uuid.uuid4().hex)
+        worker1.register_birth()
+
+        worker2 = get_worker('high')
+        worker2.register_birth()
+
+        queue_index = QUEUES_MAP['default']
+        response = self.client.get(reverse('core:worker_list', args=[queue_index]))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(str(worker1.name), str(response.content))
+        self.assertNotIn(str(worker2.name), str(response.content))
+
+    def test_worker(self):
+        worker1 = get_worker('default', name=uuid.uuid4().hex)
+        worker1.register_birth()
+
+        response = self.client.get(reverse('core:worker', args=[worker1.name]))
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(str(worker1.name), str(response.content))
+        self.assertIn('Birth', str(response.content))
+        self.assertIn('Total working time', str(response.content))

netbox/core/urls.py

@@ -25,6 +25,17 @@ urlpatterns = (
     path('jobs/<int:pk>/', views.JobView.as_view(), name='job'),
     path('jobs/<int:pk>/delete/', views.JobDeleteView.as_view(), name='job_delete'),
 
+    # Background Tasks
+    path('background-queues/', views.BackgroundQueueListView.as_view(), name='background_queue_list'),
+    path('background-queues/<int:queue_index>/<str:status>/', views.BackgroundTaskListView.as_view(), name='background_task_list'),
+    path('background-tasks/<str:job_id>/', views.BackgroundTaskView.as_view(), name='background_task'),
+    path('background-tasks/<str:job_id>/delete/', views.BackgroundTaskDeleteView.as_view(), name='background_task_delete'),
+    path('background-tasks/<str:job_id>/requeue/', views.BackgroundTaskRequeueView.as_view(), name='background_task_requeue'),
+    path('background-tasks/<str:job_id>/enqueue/', views.BackgroundTaskEnqueueView.as_view(), name='background_task_enqueue'),
+    path('background-tasks/<str:job_id>/stop/', views.BackgroundTaskStopView.as_view(), name='background_task_stop'),
+    path('background-workers/<int:queue_index>/', views.WorkerListView.as_view(), name='worker_list'),
+    path('background-workers/<str:key>/', views.WorkerView.as_view(), name='worker'),
+
     # Config revisions
     path('config-revisions/', views.ConfigRevisionListView.as_view(), name='configrevision_list'),
     path('config-revisions/add/', views.ConfigRevisionEditView.as_view(), name='configrevision_add'),

netbox/core/views.py

@@ -3,13 +3,28 @@ from django.conf import settings
 from django.contrib import messages
 from django.contrib.auth.mixins import UserPassesTestMixin
 from django.core.cache import cache
-from django.http import HttpResponseForbidden
+from django.http import HttpResponseForbidden, Http404
 from django.shortcuts import get_object_or_404, redirect, render
+from django.urls import reverse
+from django.utils.translation import gettext_lazy as _
 from django.views.generic import View
+from django_rq.queues import get_queue_by_index, get_redis_connection
+from django_rq.settings import QUEUES_MAP, QUEUES_LIST
+from django_rq.utils import get_jobs, get_statistics, stop_jobs
+from rq import requeue_job
+from rq.exceptions import NoSuchJobError
+from rq.job import Job as RQ_Job, JobStatus as RQJobStatus
+from rq.registry import (
+    DeferredJobRegistry, FailedJobRegistry, FinishedJobRegistry, ScheduledJobRegistry, StartedJobRegistry,
+)
+from rq.worker import Worker
+from rq.worker_registration import clean_worker_registry
 
 from netbox.config import get_config, PARAMS
 from netbox.views import generic
 from netbox.views.generic.base import BaseObjectView
+from netbox.views.generic.mixins import TableMixin
+from utilities.forms import ConfirmationForm
 from utilities.utils import count_related
 from utilities.views import ContentTypePermissionRequiredMixin, register_model_view
 from . import filtersets, forms, tables
@@ -237,6 +252,276 @@ class ConfigRevisionRestoreView(ContentTypePermissionRequiredMixin, View):
         return redirect(candidate_config.get_absolute_url())
 
 
+#
+# Background Tasks (RQ)
+#
+
+class BaseRQView(UserPassesTestMixin, View):
+
+    def test_func(self):
+        return self.request.user.is_staff
+
+
+class BackgroundQueueListView(TableMixin, BaseRQView):
+    table = tables.BackgroundQueueTable
+
+    def get(self, request):
+        data = get_statistics(run_maintenance_tasks=True)["queues"]
+        table = self.get_table(data, request, bulk_actions=False)
+
+        return render(request, 'core/rq_queue_list.html', {
+            'table': table,
+        })
+
+
+class BackgroundTaskListView(TableMixin, BaseRQView):
+    table = tables.BackgroundTaskTable
+
+    def get_table_data(self, request, queue, status):
+        jobs = []
+
+        # Call get_jobs() to returned queued tasks
+        if status == RQJobStatus.QUEUED:
+            return queue.get_jobs()
+
+        # For other statuses, determine the registry to list (or raise a 404 for invalid statuses)
+        try:
+            registry_cls = {
+                RQJobStatus.STARTED: StartedJobRegistry,
+                RQJobStatus.DEFERRED: DeferredJobRegistry,
+                RQJobStatus.FINISHED: FinishedJobRegistry,
+                RQJobStatus.FAILED: FailedJobRegistry,
+                RQJobStatus.SCHEDULED: ScheduledJobRegistry,
+            }[status]
+        except KeyError:
+            raise Http404
+        registry = registry_cls(queue.name, queue.connection)
+
+        job_ids = registry.get_job_ids()
+        if status != RQJobStatus.DEFERRED:
+            jobs = get_jobs(queue, job_ids, registry)
+        else:
+            # Deferred jobs require special handling
+            for job_id in job_ids:
+                try:
+                    jobs.append(RQ_Job.fetch(job_id, connection=queue.connection, serializer=queue.serializer))
+                except NoSuchJobError:
+                    pass
+
+        if jobs and status == RQJobStatus.SCHEDULED:
+            for job in jobs:
+                job.scheduled_at = registry.get_scheduled_time(job)
+
+        return jobs
+
+    def get(self, request, queue_index, status):
+        queue = get_queue_by_index(queue_index)
+        data = self.get_table_data(request, queue, status)
+        table = self.get_table(data, request, False)
+
+        # If this is an HTMX request, return only the rendered table HTML
+        if request.htmx:
+            return render(request, 'htmx/table.html', {
+                'table': table,
+            })
+
+        return render(request, 'core/rq_task_list.html', {
+            'table': table,
+            'queue': queue,
+            'status': status,
+        })
+
+
+class BackgroundTaskView(BaseRQView):
+
+    def get(self, request, job_id):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        try:
+            job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),)
+        except NoSuchJobError:
+            raise Http404(_("Job {job_id} not found").format(job_id=job_id))
+
+        queue_index = QUEUES_MAP[job.origin]
+        queue = get_queue_by_index(queue_index)
+
+        try:
+            exc_info = job._exc_info
+        except AttributeError:
+            exc_info = None
+
+        return render(request, 'core/rq_task.html', {
+            'queue': queue,
+            'job': job,
+            'queue_index': queue_index,
+            'dependency_id': job._dependency_id,
+            'exc_info': exc_info,
+        })
+
+
+class BackgroundTaskDeleteView(BaseRQView):
+
+    def get(self, request, job_id):
+        if not request.htmx:
+            return redirect(reverse('core:background_queue_list'))
+
+        form = ConfirmationForm(initial=request.GET)
+
+        return render(request, 'htmx/delete_form.html', {
+            'object_type': 'background task',
+            'object': job_id,
+            'form': form,
+            'form_url': reverse('core:background_task_delete', kwargs={'job_id': job_id})
+        })
+
+    def post(self, request, job_id):
+        form = ConfirmationForm(request.POST)
+
+        if form.is_valid():
+            # all the RQ queues should use the same connection
+            config = QUEUES_LIST[0]
+            try:
+                job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),)
+            except NoSuchJobError:
+                raise Http404(_("Job {job_id} not found").format(job_id=job_id))
+
+            queue_index = QUEUES_MAP[job.origin]
+            queue = get_queue_by_index(queue_index)
+
+            # Remove job id from queue and delete the actual job
+            queue.connection.lrem(queue.key, 0, job.id)
+            job.delete()
+            messages.success(request, f'Deleted job {job_id}')
+        else:
+            messages.error(request, f'Error deleting job: {form.errors[0]}')
+
+        return redirect(reverse('core:background_queue_list'))
+
+
+class BackgroundTaskRequeueView(BaseRQView):
+
+    def get(self, request, job_id):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        try:
+            job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),)
+        except NoSuchJobError:
+            raise Http404(_("Job {job_id} not found").format(job_id=job_id))
+
+        queue_index = QUEUES_MAP[job.origin]
+        queue = get_queue_by_index(queue_index)
+
+        requeue_job(job_id, connection=queue.connection, serializer=queue.serializer)
+        messages.success(request, f'You have successfully requeued: {job_id}')
+        return redirect(reverse('core:background_task', args=[job_id]))
+
+
+class BackgroundTaskEnqueueView(BaseRQView):
+
+    def get(self, request, job_id):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        try:
+            job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),)
+        except NoSuchJobError:
+            raise Http404(_("Job {job_id} not found").format(job_id=job_id))
+
+        queue_index = QUEUES_MAP[job.origin]
+        queue = get_queue_by_index(queue_index)
+
+        try:
+            # _enqueue_job is new in RQ 1.14, this is used to enqueue
+            # job regardless of its dependencies
+            queue._enqueue_job(job)
+        except AttributeError:
+            queue.enqueue_job(job)
+
+        # Remove job from correct registry if needed
+        if job.get_status() == RQJobStatus.DEFERRED:
+            registry = DeferredJobRegistry(queue.name, queue.connection)
+            registry.remove(job)
+        elif job.get_status() == RQJobStatus.FINISHED:
+            registry = FinishedJobRegistry(queue.name, queue.connection)
+            registry.remove(job)
+        elif job.get_status() == RQJobStatus.SCHEDULED:
+            registry = ScheduledJobRegistry(queue.name, queue.connection)
+            registry.remove(job)
+
+        messages.success(request, f'You have successfully enqueued: {job_id}')
+        return redirect(reverse('core:background_task', args=[job_id]))
+
+
+class BackgroundTaskStopView(BaseRQView):
+
+    def get(self, request, job_id):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        try:
+            job = RQ_Job.fetch(job_id, connection=get_redis_connection(config['connection_config']),)
+        except NoSuchJobError:
+            raise Http404(_("Job {job_id} not found").format(job_id=job_id))
+
+        queue_index = QUEUES_MAP[job.origin]
+        queue = get_queue_by_index(queue_index)
+
+        stopped, _ = stop_jobs(queue, job_id)
+        if len(stopped) == 1:
+            messages.success(request, f'You have successfully stopped {job_id}')
+        else:
+            messages.error(request, f'Failed to stop {job_id}')
+
+        return redirect(reverse('core:background_task', args=[job_id]))
+
+
+class WorkerListView(TableMixin, BaseRQView):
+    table = tables.WorkerTable
+
+    def get_table_data(self, request, queue):
+        clean_worker_registry(queue)
+        all_workers = Worker.all(queue.connection)
+        workers = [worker for worker in all_workers if queue.name in worker.queue_names()]
+        return workers
+
+    def get(self, request, queue_index):
+        queue = get_queue_by_index(queue_index)
+        data = self.get_table_data(request, queue)
+
+        table = self.get_table(data, request, False)
+
+        # If this is an HTMX request, return only the rendered table HTML
+        if request.htmx:
+            if request.htmx.target != 'object_list':
+                table.embedded = True
+                # Hide selection checkboxes
+                if 'pk' in table.base_columns:
+                    table.columns.hide('pk')
+            return render(request, 'htmx/table.html', {
+                'table': table,
+                'queue': queue,
+            })
+
+        return render(request, 'core/rq_worker_list.html', {
+            'table': table,
+            'queue': queue,
+        })
+
+
+class WorkerView(BaseRQView):
+
+    def get(self, request, key):
+        # all the RQ queues should use the same connection
+        config = QUEUES_LIST[0]
+        worker = Worker.find_by_key('rq:worker:' + key, connection=get_redis_connection(config['connection_config']))
+        # Convert microseconds to milliseconds
+        worker.total_working_time = worker.total_working_time / 1000
+
+        return render(request, 'core/rq_worker.html', {
+            'worker': worker,
+            'job': worker.get_current_job(),
+            'total_working_time': worker.total_working_time * 1000,
+        })
+
+
 #
 # Plugins
 #

netbox/netbox/navigation/menu.py

@@ -451,13 +451,18 @@ ADMIN_MENU = Menu(
             ),
         ),
         MenuGroup(
-            label=_('Plugins'),
+            label=_('System'),
             items=(
                 MenuItem(
                     link='core:plugin_list',
                     link_text=_('Plugins'),
                     staff_only=True
                 ),
+                MenuItem(
+                    link='core:background_queue_list',
+                    link_text=_('Background Tasks'),
+                    staff_only=True
+                ),
             ),
         ),
     ),

netbox/netbox/urls.py

@@ -72,7 +72,6 @@ _patterns = [
     path('api/plugins/', include((plugin_api_patterns, 'plugins-api'))),
 
     # Admin
-    path('admin/background-tasks/', include('django_rq.urls')),
     path('admin/', admin_site.urls),
 ]
 

netbox/templates/admin/index.html

@@ -1,25 +0,0 @@
-{% extends "admin/index.html" %}
-{% load i18n %}
-
-{% block content_title %}{% endblock %}
-
-{% block sidebar %}
-    {{ block.super }}
-    <div class="module">
-        <table style="width: 100%">
-            <caption>{% trans "System" %}</caption>
-            <tbody>
-                <tr>
-                    <th>
-                        <a href="{% url 'rq_home' %}">{% trans "Background Tasks" %}</a>
-                    </th>
-                </tr>
-                <tr>
-                    <th>
-                        <a href="{% url 'plugins_list' %}">{% trans "Installed plugins" %}</a>
-                    </th>
-                </tr>
-            </tbody>
-        </table>
-    </div>
-{% endblock %}

netbox/templates/core/rq_queue_list.html

@@ -0,0 +1,34 @@
+{% extends 'generic/object_list.html' %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+
+{% block title %}{% trans "Background Queues" %}{% endblock %}
+
+{% block controls %}{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">
+        {% trans "Background Queues" %} {% badge table.rows|length %}
+      </a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+  <div class="row mb-3">
+    <div class="col-auto ms-auto d-print-none">
+      {# Table configuration button #}
+      <div class="table-configure input-group">
+        <button type="button" data-bs-toggle="modal" title="{% trans "Configure Table" %}" data-bs-target="#ObjectTable_config" class="btn">
+          <i class="mdi mdi-cog"></i> {% trans "Configure Table" %}
+        </button>
+      </div>
+    </div>
+  </div>
+
+  <div class="card">
+    {% render_table table %}
+  </div>
+{% endblock content %}

netbox/templates/core/rq_task.html

@@ -0,0 +1,117 @@
+{% extends 'generic/object.html' %}
+{% load i18n %}
+{% load buttons %}
+{% load helpers %}
+{% load render_table from django_tables2 %}
+
+{% block breadcrumbs %}
+  <li class="breadcrumb-item"><a href="{% url 'core:background_queue_list' %}">{% trans 'Background Tasks' %}</a></li>
+  <li class="breadcrumb-item"><a href="{% url 'core:background_task_list' queue_index=queue_index status=job.get_status %}">{{ queue.name }}</a></li>
+{% endblock breadcrumbs %}
+
+{% block title %}{% trans "Job" %} {{ job.id }}{% endblock %}
+
+{% block subtitle %}
+  <div class="text-secondary fs-5">
+    <span>{% trans "Created" %} {{ job.created_at|annotated_date }}</span>
+  </div>
+{% endblock subtitle %}
+
+{% block object_identifier %}{% endblock %}
+
+{% block controls %}
+  <div class="btn-list mb-2">
+    {% url 'core:background_task_delete' job_id=job.id as delete_url %}
+    {% include "buttons/delete.html" with url=delete_url %}
+
+    {% if job.is_started %}
+      <a href="{% url 'core:background_task_stop' job.id %}" class="btn btn-primary">
+        <i class="mdi mdi-stop-circle-outline"></i> {% trans "Stop" %}
+      </a>
+    {% endif %}
+    {% if job.is_failed %}
+      <a href="{% url 'core:background_task_requeue' job.id %}" class="btn btn-primary">
+        <i class="mdi mdi-sync"></i> {% trans "Requeue" %}
+      </a>
+    {% endif %}
+    {% if not job.is_queued and not job.is_failed %}
+      <a href="{% url 'core:background_task_enqueue' job.id %}" class="btn btn-primary">
+        <i class="mdi mdi-sync"></i> {% trans "Enqueue" %}
+      </a>
+    {% endif %}
+
+  </div>
+{% endblock controls %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">{% trans "Job" %}</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+  <div class="row">
+    <div class="col col-md-12">
+      <div class="card">
+        <h5 class="card-header">{% trans "Job" %}</h5>
+        <table class="table table-hover attr-table">
+          <tr>
+            <th scope="row">{% trans "Queue" %}</th>
+            <td>{{ job.origin|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Timeout" %}</th>
+            <td>{{ job.timeout|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Result TTL" %}</th>
+            <td>{{ job.result_ttl|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Created" %}</th>
+            <td>{{ job.created_at|annotated_date }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Queued" %}</th>
+            <td>{{ job.enqueued_at|annotated_date }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Status" %}</th>
+            <td>{{ job.get_status|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Callable" %}</th>
+            <td>{{ object.get_type_display|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Meta" %}</th>
+            <td>{{ job.meta|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Arguments" %}</th>
+            <td>{{ jobs.args|placeholder }}</td>
+          </tr>
+          <tr>
+            <th scope="row">{% trans "Keyword Arguments" %}</th>
+            {# TODO: Render as formatted JSON #}
+            <td>{{ job.kwargs }}</td>
+          </tr>
+          {% if dependency_id %}
+            <tr>
+              <th scope="row">{% trans "Depends on" %}</th>
+              <td><a href="{% url 'core:background_task' job.id %}">{{ dependency_id }}</a></td>
+            </tr>
+          {% endif %}
+          {% if exc_info %}
+            <tr>
+              <th scope="row">{% trans "Exception" %}</th>
+              <td><pre>{% if job.exc_info %}{{ job.exc_info|linebreaks }}{% endif %}</pre></td>
+            </tr>
+          {% endif %}
+        </table>
+      </div>
+    </div>
+  </div>
+{% endblock content %}

netbox/templates/core/rq_task_list.html

@@ -0,0 +1,104 @@
+{% extends 'generic/object_list.html' %}
+{% load buttons %}
+{% load helpers %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+
+{% block page-header %}
+  <div class="container-xl">
+    <div class="d-flex justify-content-between align-items-center mt-2">
+      {# Breadcrumbs #}
+      <nav class="breadcrumb-container" aria-label="breadcrumb">
+        <ol class="breadcrumb">
+          <li class="breadcrumb-item">
+            <a href="{% url 'core:background_queue_list' %}">{% trans 'Background Queues' %}</a>
+          </li>
+          <li class="breadcrumb-item">{{ queue.name }}</li>
+        </ol>
+      </nav>
+    </div>
+    <div class="row">
+      <div class="col">
+        <h2 class="page-title mt-2">{% trans 'Background Tasks' %}</h2>
+      </div>
+    </div>
+  </div>
+{% endblock page-header %}
+
+{% block title %}{{ status|capfirst }} {% trans "tasks in " %}{{ queue.name }}{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs" role="tablist">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">{% trans "Queued Jobs" %}</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+
+    {# Object list tab #}
+    <div class="tab-pane show active" id="object-list" role="tabpanel" aria-labelledby="object-list-tab">
+
+      {# Object table controls #}
+      {% include 'inc/table_controls_htmx.html' with table_modal="ObjectTable_config" %}
+
+      <form method="post" class="form form-horizontal">
+        {% csrf_token %}
+        {# "Select all" form #}
+        {% if table.paginator.num_pages > 1 %}
+          <div id="select-all-box" class="d-none card d-print-none">
+            <div class="form col-md-12">
+              <div class="card-body">
+                <div class="float-end">
+                  {% if 'bulk_edit' in actions %}
+                    {% bulk_edit_button model query_params=request.GET %}
+                  {% endif %}
+                  {% if 'bulk_delete' in actions %}
+                    {% bulk_delete_button model query_params=request.GET %}
+                  {% endif %}
+                </div>
+                <div class="form-check">
+                  <input type="checkbox" id="select-all" name="_all" class="form-check-input" />
+                  <label for="select-all" class="form-check-label">
+                    {% blocktrans trimmed with count=table.rows|length object_type_plural=table.data.verbose_name_plural %}
+                      Select <strong>all {{ count }} {{ object_type_plural }}</strong> matching query
+                    {% endblocktrans %}
+                  </label>
+                </div>
+              </div>
+            </div>
+          </div>
+        {% endif %}
+
+        <div class="form form-horizontal">
+          {% csrf_token %}
+          <input type="hidden" name="return_url" value="{% if return_url %}{{ return_url }}{% else %}{{ request.path }}{% if request.GET %}?{{ request.GET.urlencode }}{% endif %}{% endif %}" />
+
+          {# Objects table #}
+          <div class="card">
+            <div class="htmx-container table-responsive" id="object_list">
+              {% include 'htmx/table.html' %}
+            </div>
+          </div>
+          {# /Objects table #}
+
+          {# Form buttons #}
+          <div class="btn-list d-print-none mt-2">
+            {% block bulk_buttons %}
+              {% if 'bulk_edit' in actions %}
+                {% bulk_edit_button model query_params=request.GET %}
+              {% endif %}
+              {% if 'bulk_delete' in actions %}
+                {% bulk_delete_button model query_params=request.GET %}
+              {% endif %}
+            {% endblock %}
+          </div>
+          {# /Form buttons #}
+
+        </div>
+      </form>
+    </div>
+    {# /Object list tab #}
+
+{% endblock content %}

netbox/templates/core/rq_worker.html

@@ -0,0 +1,82 @@
+{% extends 'generic/object.html' %}
+{% load i18n %}
+{% load helpers %}
+{% load render_table from django_tables2 %}
+
+{% block breadcrumbs %}
+  <li class="breadcrumb-item"><a href="{% url 'core:background_queue_list' %}">{% trans 'Background Queues' %}</a></li>
+{% endblock breadcrumbs %}
+
+{% block title %}{% trans "Worker Info" %} {{ job.id }}{% endblock %}
+
+{% block subtitle %}
+  <div class="text-secondary fs-5">
+    <span>{% trans "Created" %} {{ worker.birth_date|annotated_date }}</span>
+  </div>
+{% endblock subtitle %}
+
+{% block object_identifier %}{% endblock %}
+
+{% block controls %}
+  <div class="controls">
+    <div class="control-group">
+      {% block extra_controls %}{% endblock %}
+    </div>
+  </div>
+{% endblock controls %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">{% trans "Worker" %}</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+    <div class="row">
+      <div class="col col-md-12">
+        <div class="card">
+          <h5 class="card-header">{% trans "Worker" %}</h5>
+          <table class="table table-hover attr-table">
+            <tr>
+              <th scope="row">{% trans "Name" %}</th>
+              <td>{{ worker.name|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "State" %}</th>
+              <td>{{ worker.get_state|bettertitle|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Birth" %}</th>
+              <td>{{ worker.birth_date|annotated_date }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Queues" %}</th>
+              <td>{{ worker.queue_names|join:", " }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "PID" %}</th>
+              <td>{{ worker.pid|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Curent Job" %}</th>
+              <td>{{ job.func_name|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Successful job count" %}</th>
+              <td>{{ worker.successful_job_count|placeholder }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Failed job count" %}</th>
+              <td>{{ worker.failed_job_count }}</td>
+            </tr>
+            <tr>
+              <th scope="row">{% trans "Total working time" %}</th>
+              <td>{{ total_working_time }} {% trans "seconds" %}</td>
+            </tr>
+          </table>
+        </div>
+      </div>
+    </div>
+{% endblock content %}

netbox/templates/core/rq_worker_list.html

@@ -0,0 +1,58 @@
+{% extends 'generic/object_list.html' %}
+{% load helpers %}
+{% load i18n %}
+{% load render_table from django_tables2 %}
+
+{% block page-header %}
+  <div class="container-xl">
+    <div class="d-flex justify-content-between align-items-center mt-2">
+      {# Breadcrumbs #}
+      <nav class="breadcrumb-container" aria-label="breadcrumb">
+        <ol class="breadcrumb">
+          <li class="breadcrumb-item">
+            <a href="{% url 'core:background_queue_list' %}">{% trans 'Background Workers' %}</a>
+          </li>
+          <li class="breadcrumb-item">{{ queue.name }}</li>
+        </ol>
+      </nav>
+    </div>
+    <div class="row">
+      <div class="col">
+        <h2 class="page-title mt-2">{% trans 'Background Workers' %}</h2>
+      </div>
+    </div>
+  </div>
+{% endblock page-header %}
+
+{% block title %}{{ status|capfirst }} {% trans "Workers in " %}{{ queue.name }}{% endblock %}
+
+{% block controls %}{% endblock %}
+
+{% block tabs %}
+  <ul class="nav nav-tabs">
+    <li class="nav-item" role="presentation">
+      <a class="nav-link active" role="tab">{% trans "Workers" %}</a>
+    </li>
+  </ul>
+{% endblock tabs %}
+
+{% block content %}
+  <div class="row mb-3">
+    <div class="col-auto ms-auto d-print-none">
+      {# Table configuration button #}
+      <div class="table-configure input-group">
+        <button type="button" data-bs-toggle="modal" title="{% trans "Configure Table" %}" data-bs-target="#ObjectTable_config" class="btn">
+          <i class="mdi mdi-cog"></i> {% trans "Configure Table" %}
+        </button>
+      </div>
+    </div>
+  </div>
+
+  <div class="card">
+    {% render_table table %}
+  </div>
+{% endblock content %}
+
+{% block modals %}
+  {% table_config_form table table_name="ObjectTable" %}
+{% endblock modals %}