Merge pull request #5266 from netbox-community/4559-config-context-rendering

4559 config context rendering

netbox/dcim/api/views.py

@@ -24,7 +24,7 @@ from dcim.models import (
     VirtualChassis,
 )
 from extras.api.serializers import RenderedGraphSerializer
-from extras.api.views import CustomFieldModelViewSet
+from extras.api.views import ConfigContextQuerySetMixin, CustomFieldModelViewSet
 from extras.models import Graph
 from ipam.models import Prefix, VLAN
 from utilities.api import (
@@ -336,7 +336,7 @@ class PlatformViewSet(ModelViewSet):
 # Devices
 #
 
-class DeviceViewSet(CustomFieldModelViewSet):
+class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
     queryset = Device.objects.prefetch_related(
         'device_type__manufacturer', 'device_role', 'tenant', 'platform', 'site', 'rack', 'parent_bay',
         'virtual_chassis__master', 'primary_ip4__nat_outside', 'primary_ip6__nat_outside', 'tags',

netbox/dcim/models/devices.py

@@ -15,6 +15,7 @@ from taggit.managers import TaggableManager
 from dcim.choices import *
 from dcim.constants import *
 from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, TaggedItem
+from extras.querysets import ConfigContextModelQuerySet
 from extras.utils import extras_features
 from utilities.choices import ColorChoices
 from utilities.fields import ColorField, NaturalOrderingField
@@ -594,7 +595,7 @@ class Device(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
     )
     tags = TaggableManager(through=TaggedItem)
 
-    objects = RestrictedQuerySet.as_manager()
+    objects = ConfigContextModelQuerySet.as_manager()
 
     csv_headers = [
         'name', 'device_role', 'tenant', 'manufacturer', 'device_type', 'platform', 'serial', 'asset_tag', 'status',

netbox/dcim/views.py

@@ -1163,7 +1163,7 @@ class DeviceConfigView(ObjectView):
 
 
 class DeviceConfigContextView(ObjectConfigContextView):
-    queryset = Device.objects.all()
+    queryset = Device.objects.annotate_config_context_data()
     base_template = 'dcim/device.html'
 
 

netbox/extras/api/views.py

@@ -26,6 +26,29 @@ from utilities.utils import copy_safe_request
 from . import serializers
 
 
+class ConfigContextQuerySetMixin:
+    """
+    Used by views that work with config context models (device and virtual machine).
+    Provides a get_queryset() method which deals with adding the config context
+    data annotation or not.
+    """
+
+    def get_queryset(self):
+        """
+        Build the proper queryset based on the request context
+
+        If the `brief` query param equates to True or the `exclude` query param
+        includes `config_context` as a value, return the base queryset.
+
+        Else, return the queryset annotated with config context data
+        """
+
+        request = self.get_serializer_context()['request']
+        if request.query_params.get('brief') or 'config_context' in request.query_params.get('exclude', []):
+            return self.queryset
+        return self.queryset.annotate_config_context_data()
+
+
 class ExtrasRootView(APIRootView):
     """
     Extras API root view

netbox/extras/models/models.py

@@ -542,8 +542,16 @@ class ConfigContextModel(models.Model):
 
         # Compile all config data, overwriting lower-weight values with higher-weight values where a collision occurs
         data = OrderedDict()
-        for context in ConfigContext.objects.get_for_object(self):
-            data = deepmerge(data, context.data)
+
+        if not hasattr(self, 'config_context_data'):
+            # The annotation is not available, so we fall back to manually querying for the config context objects
+            config_context_data = ConfigContext.objects.get_for_object(self, aggregate_data=True)
+        else:
+            # The attribute may exist, but the annotated value could be None if there is no config context data
+            config_context_data = self.config_context_data or []
+
+        for context in config_context_data:
+            data = deepmerge(data, context)
 
         # If the object has local config context data defined, merge it last
         if self.local_context_data:

netbox/extras/querysets.py

@@ -1,7 +1,8 @@
 from collections import OrderedDict
 
-from django.db.models import Q, QuerySet
+from django.db.models import OuterRef, Subquery, Q
 
+from utilities.query_functions import EmptyGroupByJSONBAgg, OrderableJSONBAgg
 from utilities.querysets import RestrictedQuerySet
 
 
@@ -23,9 +24,12 @@ class CustomFieldQueryset:
 
 class ConfigContextQuerySet(RestrictedQuerySet):
 
-    def get_for_object(self, obj):
+    def get_for_object(self, obj, aggregate_data=False):
         """
         Return all applicable ConfigContexts for a given object. Only active ConfigContexts will be included.
+
+        Args:
+          aggregate_data: If True, use the JSONBAgg aggregate function to return only the list of JSON data objects
         """
 
         # `device_role` for Device; `role` for VirtualMachine
@@ -45,7 +49,7 @@ class ConfigContextQuerySet(RestrictedQuerySet):
         else:
             regions = []
 
-        return self.filter(
+        queryset = self.filter(
             Q(regions__in=regions) | Q(regions=None),
             Q(sites=obj.site) | Q(sites=None),
             Q(roles=role) | Q(roles=None),
@@ -57,3 +61,72 @@ class ConfigContextQuerySet(RestrictedQuerySet):
             Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
             is_active=True,
         ).order_by('weight', 'name')
+
+        if aggregate_data:
+            return queryset.aggregate(
+                config_context_data=OrderableJSONBAgg('data', ordering=['weight', 'name'])
+            )['config_context_data']
+
+        return queryset
+
+
+class ConfigContextModelQuerySet(RestrictedQuerySet):
+    """
+    QuerySet manager used by models which support ConfigContext (device and virtual machine).
+
+    Includes a method which appends an annotation of aggregated config context JSON data objects. This is
+    implemented as a subquery which performs all the joins necessary to filter relevant config context objects.
+    This offers a substantial performance gain over ConfigContextQuerySet.get_for_object() when dealing with
+    multiple objects.
+
+    This allows the annotation to be entirely optional.
+    """
+
+    def annotate_config_context_data(self):
+        """
+        Attach the subquery annotation to the base queryset
+        """
+        from extras.models import ConfigContext
+        return self.annotate(
+            config_context_data=Subquery(
+                ConfigContext.objects.filter(
+                    self._get_config_context_filters()
+                ).annotate(
+                    _data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name'])
+                ).values("_data")
+            )
+        )
+
+    def _get_config_context_filters(self):
+        # Construct the set of Q objects for the specific object types
+        base_query = Q(
+            Q(platforms=OuterRef('platform')) | Q(platforms=None),
+            Q(tenant_groups=OuterRef('tenant__group')) | Q(tenant_groups=None),
+            Q(tenants=OuterRef('tenant')) | Q(tenants=None),
+            Q(tags=OuterRef('tags')) | Q(tags=None),
+            is_active=True,
+        )
+
+        if self.model._meta.model_name == 'device':
+            base_query.add((Q(roles=OuterRef('device_role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('site')) | Q(sites=None)), Q.AND)
+            region_field = 'site__region'
+
+        elif self.model._meta.model_name == 'virtualmachine':
+            base_query.add((Q(roles=OuterRef('role')) | Q(roles=None)), Q.AND)
+            base_query.add((Q(cluster_groups=OuterRef('cluster__group')) | Q(cluster_groups=None)), Q.AND)
+            base_query.add((Q(clusters=OuterRef('cluster')) | Q(clusters=None)), Q.AND)
+            base_query.add((Q(sites=OuterRef('cluster__site')) | Q(sites=None)), Q.AND)
+            region_field = 'cluster__site__region'
+
+        base_query.add(
+            (Q(
+                regions__tree_id=OuterRef(f'{region_field}__tree_id'),
+                regions__level__lte=OuterRef(f'{region_field}__level'),
+                regions__lft__lte=OuterRef(f'{region_field}__lft'),
+                regions__rght__gte=OuterRef(f'{region_field}__rght'),
+            ) | Q(regions=None)),
+            Q.AND
+        )
+
+        return base_query

netbox/extras/tests/test_models.py

@@ -1,9 +1,11 @@
 from django.contrib.contenttypes.models import ContentType
 from django.test import TestCase
 
-from dcim.models import Site
+from dcim.models import Device, DeviceRole, DeviceType, Manufacturer, Platform, Site, Region
 from extras.choices import TemplateLanguageChoices
-from extras.models import Graph, Tag
+from extras.models import ConfigContext, Graph, Tag
+from tenancy.models import Tenant, TenantGroup
+from virtualization.models import Cluster, ClusterGroup, ClusterType, VirtualMachine
 
 
 class GraphTest(TestCase):
@@ -53,3 +55,276 @@ class TagTest(TestCase):
         tag.save()
 
         self.assertEqual(tag.slug, 'testing-unicode-台灣')
+
+
+class ConfigContextTest(TestCase):
+    """
+    These test cases deal with the weighting, ordering, and deep merge logic of config context data.
+
+    It also ensures the various config context querysets are consistent.
+    """
+
+    def setUp(self):
+
+        manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1')
+        self.devicetype = DeviceType.objects.create(manufacturer=manufacturer, model='Device Type 1', slug='device-type-1')
+        self.devicerole = DeviceRole.objects.create(name='Device Role 1', slug='device-role-1')
+        self.region = Region.objects.create(name="Region")
+        self.site = Site.objects.create(name='Site-1', slug='site-1', region=self.region)
+        self.platform = Platform.objects.create(name="Platform")
+        self.tenantgroup = TenantGroup.objects.create(name="Tenant Group")
+        self.tenant = Tenant.objects.create(name="Tenant", group=self.tenantgroup)
+        self.tag = Tag.objects.create(name="Tag", slug="tag")
+
+        self.device = Device.objects.create(
+            name='Device 1',
+            device_type=self.devicetype,
+            device_role=self.devicerole,
+            site=self.site
+        )
+
+    def test_higher_weight_wins(self):
+
+        context1 = ConfigContext(
+            name="context 1",
+            weight=101,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2])
+
+        expected_data = {
+            "a": 123,
+            "b": 456,
+            "c": 777
+        }
+        self.assertEqual(self.device.get_config_context(), expected_data)
+
+    def test_name_ordering_after_weight(self):
+
+        context1 = ConfigContext(
+            name="context 1",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2])
+
+        expected_data = {
+            "a": 123,
+            "b": 456,
+            "c": 789
+        }
+        self.assertEqual(self.device.get_config_context(), expected_data)
+
+    def test_annotation_same_as_get_for_object(self):
+        """
+        This test incorperates features from all of the above tests cases to ensure
+        the annotate_config_context_data() and get_for_object() queryset methods are the same.
+        """
+        context1 = ConfigContext(
+            name="context 1",
+            weight=101,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 777
+            }
+        )
+        context2 = ConfigContext(
+            name="context 2",
+            weight=100,
+            data={
+                "a": 123,
+                "b": 456,
+                "c": 789
+            }
+        )
+        context3 = ConfigContext(
+            name="context 3",
+            weight=99,
+            data={
+                "d": 1
+            }
+        )
+        context4 = ConfigContext(
+            name="context 4",
+            weight=99,
+            data={
+                "d": 2
+            }
+        )
+        ConfigContext.objects.bulk_create([context1, context2, context3, context4])
+
+        annotated_queryset = Device.objects.filter(name=self.device.name).annotate_config_context_data()
+        self.assertEqual(self.device.get_config_context(), annotated_queryset[0].get_config_context())
+
+    def test_annotation_same_as_get_for_object_device_relations(self):
+
+        site_context = ConfigContext.objects.create(
+            name="site",
+            weight=100,
+            data={
+                "site": 1
+            }
+        )
+        site_context.sites.add(self.site)
+        region_context = ConfigContext.objects.create(
+            name="region",
+            weight=100,
+            data={
+                "region": 1
+            }
+        )
+        region_context.regions.add(self.region)
+        platform_context = ConfigContext.objects.create(
+            name="platform",
+            weight=100,
+            data={
+                "platform": 1
+            }
+        )
+        platform_context.platforms.add(self.platform)
+        tenant_group_context = ConfigContext.objects.create(
+            name="tenant group",
+            weight=100,
+            data={
+                "tenant_group": 1
+            }
+        )
+        tenant_group_context.tenant_groups.add(self.tenantgroup)
+        tenant_context = ConfigContext.objects.create(
+            name="tenant",
+            weight=100,
+            data={
+                "tenant": 1
+            }
+        )
+        tenant_context.tenants.add(self.tenant)
+        tag_context = ConfigContext.objects.create(
+            name="tag",
+            weight=100,
+            data={
+                "tag": 1
+            }
+        )
+        tag_context.tags.add(self.tag)
+
+        device = Device.objects.create(
+            name="Device 2",
+            site=self.site,
+            tenant=self.tenant,
+            platform=self.platform,
+            device_role=self.devicerole,
+            device_type=self.devicetype
+        )
+        device.tags.add(self.tag)
+
+        annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data()
+        self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context())
+
+    def test_annotation_same_as_get_for_object_virtualmachine_relations(self):
+
+        site_context = ConfigContext.objects.create(
+            name="site",
+            weight=100,
+            data={
+                "site": 1
+            }
+        )
+        site_context.sites.add(self.site)
+        region_context = ConfigContext.objects.create(
+            name="region",
+            weight=100,
+            data={
+                "region": 1
+            }
+        )
+        region_context.regions.add(self.region)
+        platform_context = ConfigContext.objects.create(
+            name="platform",
+            weight=100,
+            data={
+                "platform": 1
+            }
+        )
+        platform_context.platforms.add(self.platform)
+        tenant_group_context = ConfigContext.objects.create(
+            name="tenant group",
+            weight=100,
+            data={
+                "tenant_group": 1
+            }
+        )
+        tenant_group_context.tenant_groups.add(self.tenantgroup)
+        tenant_context = ConfigContext.objects.create(
+            name="tenant",
+            weight=100,
+            data={
+                "tenant": 1
+            }
+        )
+        tenant_context.tenants.add(self.tenant)
+        tag_context = ConfigContext.objects.create(
+            name="tag",
+            weight=100,
+            data={
+                "tag": 1
+            }
+        )
+        tag_context.tags.add(self.tag)
+        cluster_group = ClusterGroup.objects.create(name="Cluster Group")
+        cluster_group_context = ConfigContext.objects.create(
+            name="cluster group",
+            weight=100,
+            data={
+                "cluster_group": 1
+            }
+        )
+        cluster_group_context.cluster_groups.add(cluster_group)
+        cluster_type = ClusterType.objects.create(name="Cluster Type 1")
+        cluster = Cluster.objects.create(name="Cluster", group=cluster_group, type=cluster_type)
+        cluster_context = ConfigContext.objects.create(
+            name="cluster",
+            weight=100,
+            data={
+                "cluster": 1
+            }
+        )
+        cluster_context.clusters.add(cluster)
+
+        virtual_machine = VirtualMachine.objects.create(
+            name="VM 1",
+            cluster=cluster,
+            tenant=self.tenant,
+            platform=self.platform,
+            role=self.devicerole
+        )
+        virtual_machine.tags.add(self.tag)
+
+        annotated_queryset = VirtualMachine.objects.filter(name=virtual_machine.name).annotate_config_context_data()
+        self.assertEqual(virtual_machine.get_config_context(), annotated_queryset[0].get_config_context())

netbox/utilities/query_functions.py

@@ -1,3 +1,5 @@
+from django.contrib.postgres.aggregates import JSONBAgg
+from django.contrib.postgres.aggregates.mixins import OrderableAggMixin
 from django.db.models import F, Func
 
 
@@ -7,3 +9,21 @@ class CollateAsChar(Func):
     """
     function = 'C'
     template = '(%(expressions)s) COLLATE "%(function)s"'
+
+
+class OrderableJSONBAgg(OrderableAggMixin, JSONBAgg):
+    """
+    TODO in Django 3.2 ordering is supported natively on JSONBAgg so this is no longer needed.
+    """
+    template = '%(function)s(%(distinct)s%(expressions)s %(ordering)s)'
+
+
+class EmptyGroupByJSONBAgg(OrderableJSONBAgg):
+    """
+    JSONBAgg is a builtin aggregation function which means it includes the use of a GROUP BY clause.
+    When used as an annotation for collecting config context data objects, the GROUP BY is
+    incorrect. This subclass overrides the Django ORM aggregation control to remove the GROUP BY.
+
+    TODO in Django 3.2 ordering is supported natively on JSONBAgg so we only need to inherit from JSONBAgg.
+    """
+    contains_aggregate = False

netbox/virtualization/api/views.py

@@ -6,7 +6,7 @@ from rest_framework.routers import APIRootView
 
 from dcim.models import Device
 from extras.api.serializers import RenderedGraphSerializer
-from extras.api.views import CustomFieldModelViewSet
+from extras.api.views import ConfigContextQuerySetMixin, CustomFieldModelViewSet
 from extras.models import Graph
 from utilities.api import ModelViewSet
 from utilities.utils import get_subquery
@@ -58,7 +58,7 @@ class ClusterViewSet(CustomFieldModelViewSet):
 # Virtual machines
 #
 
-class VirtualMachineViewSet(CustomFieldModelViewSet):
+class VirtualMachineViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
     queryset = VirtualMachine.objects.prefetch_related(
         'cluster__site', 'role', 'tenant', 'platform', 'primary_ip4', 'primary_ip6', 'tags'
     )

netbox/virtualization/models.py

@@ -8,6 +8,7 @@ from taggit.managers import TaggableManager
 from dcim.choices import InterfaceModeChoices
 from dcim.models import BaseInterface, Device
 from extras.models import ChangeLoggedModel, ConfigContextModel, CustomFieldModel, ObjectChange, TaggedItem
+from extras.querysets import ConfigContextModelQuerySet
 from extras.utils import extras_features
 from utilities.fields import NaturalOrderingField
 from utilities.ordering import naturalize_interface
@@ -282,7 +283,7 @@ class VirtualMachine(ChangeLoggedModel, ConfigContextModel, CustomFieldModel):
     )
     tags = TaggableManager(through=TaggedItem)
 
-    objects = RestrictedQuerySet.as_manager()
+    objects = ConfigContextModelQuerySet.as_manager()
 
     csv_headers = [
         'name', 'status', 'role', 'cluster', 'tenant', 'platform', 'vcpus', 'memory', 'disk', 'comments',

netbox/virtualization/views.py

@@ -261,7 +261,7 @@ class VirtualMachineView(ObjectView):
 
 
 class VirtualMachineConfigContextView(ObjectConfigContextView):
-    queryset = VirtualMachine.objects.all()
+    queryset = VirtualMachine.objects.annotate_config_context_data()
     base_template = 'virtualization/virtualmachine.html'