initial work on dynamic lookup expressions

netbox/circuits/filters.py

@@ -4,7 +4,9 @@ from django.db.models import Q
 from dcim.models import Region, Site
 from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet
 from tenancy.filters import TenancyFilterSet
-from utilities.filters import NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter
+from utilities.filters import (
+    BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter
+)
 from .choices import *
 from .models import Circuit, CircuitTermination, CircuitType, Provider
 
@@ -16,7 +18,7 @@ __all__ = (
 )
 
 
-class ProviderFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class ProviderFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -65,14 +67,14 @@ class ProviderFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet):
         )
 
 
-class CircuitTypeFilterSet(NameSlugSearchFilterSet):
+class CircuitTypeFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
 
     class Meta:
         model = CircuitType
         fields = ['id', 'name', 'slug']
 
 
-class CircuitFilterSet(CustomFieldFilterSet, TenancyFilterSet, CreatedUpdatedFilterSet):
+class CircuitFilterSet(BaseFilterSet, CustomFieldFilterSet, TenancyFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -146,7 +148,7 @@ class CircuitFilterSet(CustomFieldFilterSet, TenancyFilterSet, CreatedUpdatedFil
         ).distinct()
 
 
-class CircuitTerminationFilterSet(django_filters.FilterSet):
+class CircuitTerminationFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',

netbox/dcim/filters.py

@@ -6,8 +6,8 @@ from tenancy.filters import TenancyFilterSet
 from tenancy.models import Tenant
 from utilities.constants import COLOR_CHOICES
 from utilities.filters import (
-    MultiValueCharFilter, MultiValueMACAddressFilter, MultiValueNumberFilter, NameSlugSearchFilterSet, NumericInFilter,
-    TagFilter, TreeNodeMultipleChoiceFilter,
+    BaseFilterSet, MultiValueCharFilter, MultiValueMACAddressFilter, MultiValueNumberFilter,
+    BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter, TreeNodeMultipleChoiceFilter,
 )
 from virtualization.models import Cluster
 from .choices import *
@@ -60,7 +60,7 @@ __all__ = (
 )
 
 
-class RegionFilterSet(NameSlugSearchFilterSet):
+class RegionFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
     parent_id = django_filters.ModelMultipleChoiceFilter(
         queryset=Region.objects.all(),
         label='Parent region (ID)',
@@ -77,7 +77,7 @@ class RegionFilterSet(NameSlugSearchFilterSet):
         fields = ['id', 'name', 'slug']
 
 
-class SiteFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class SiteFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -131,7 +131,7 @@ class SiteFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilter
         return queryset.filter(qs_filter)
 
 
-class RackGroupFilterSet(NameSlugSearchFilterSet):
+class RackGroupFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
     region_id = TreeNodeMultipleChoiceFilter(
         queryset=Region.objects.all(),
         field_name='site__region__in',
@@ -159,14 +159,14 @@ class RackGroupFilterSet(NameSlugSearchFilterSet):
         fields = ['id', 'name', 'slug']
 
 
-class RackRoleFilterSet(NameSlugSearchFilterSet):
+class RackRoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
 
     class Meta:
         model = RackRole
         fields = ['id', 'name', 'slug', 'color']
 
 
-class RackFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class RackFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -244,7 +244,7 @@ class RackFilterSet(TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilter
         )
 
 
-class RackReservationFilterSet(TenancyFilterSet):
+class RackReservationFilterSet(BaseFilterSet, TenancyFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -305,14 +305,14 @@ class RackReservationFilterSet(TenancyFilterSet):
         )
 
 
-class ManufacturerFilterSet(NameSlugSearchFilterSet):
+class ManufacturerFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
 
     class Meta:
         model = Manufacturer
         fields = ['id', 'name', 'slug']
 
 
-class DeviceTypeFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class DeviceTypeFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -402,7 +402,7 @@ class DeviceTypeFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet):
         return queryset.exclude(device_bay_templates__isnull=value)
 
 
-class DeviceTypeComponentFilterSet(NameSlugSearchFilterSet):
+class DeviceTypeComponentFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
     devicetype_id = django_filters.ModelMultipleChoiceFilter(
         queryset=DeviceType.objects.all(),
         field_name='device_type_id',
@@ -466,14 +466,14 @@ class DeviceBayTemplateFilterSet(DeviceTypeComponentFilterSet):
         fields = ['id', 'name']
 
 
-class DeviceRoleFilterSet(NameSlugSearchFilterSet):
+class DeviceRoleFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
 
     class Meta:
         model = DeviceRole
         fields = ['id', 'name', 'slug', 'color', 'vm_role']
 
 
-class PlatformFilterSet(NameSlugSearchFilterSet):
+class PlatformFilterSet(BaseFilterSet, NameSlugSearchFilterSet):
     manufacturer_id = django_filters.ModelMultipleChoiceFilter(
         field_name='manufacturer',
         queryset=Manufacturer.objects.all(),
@@ -491,7 +491,7 @@ class PlatformFilterSet(NameSlugSearchFilterSet):
         fields = ['id', 'name', 'slug', 'napalm_driver']
 
 
-class DeviceFilterSet(LocalConfigContextFilterSet, TenancyFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class DeviceFilterSet(BaseFilterSet, LocalConfigContextFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -690,7 +690,7 @@ class DeviceFilterSet(LocalConfigContextFilterSet, TenancyFilterSet, CustomField
         return queryset.exclude(device_bays__isnull=value)
 
 
-class DeviceComponentFilterSet(django_filters.FilterSet):
+class DeviceComponentFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',
@@ -1002,7 +1002,7 @@ class InventoryItemFilterSet(DeviceComponentFilterSet):
         return queryset.filter(qs_filter)
 
 
-class VirtualChassisFilterSet(django_filters.FilterSet):
+class VirtualChassisFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',
@@ -1056,7 +1056,7 @@ class VirtualChassisFilterSet(django_filters.FilterSet):
         return queryset.filter(qs_filter)
 
 
-class CableFilterSet(django_filters.FilterSet):
+class CableFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',
@@ -1119,7 +1119,7 @@ class CableFilterSet(django_filters.FilterSet):
         return queryset
 
 
-class ConsoleConnectionFilterSet(django_filters.FilterSet):
+class ConsoleConnectionFilterSet(BaseFilterSet):
     site = django_filters.CharFilter(
         method='filter_site',
         label='Site (slug)',
@@ -1150,7 +1150,7 @@ class ConsoleConnectionFilterSet(django_filters.FilterSet):
         )
 
 
-class PowerConnectionFilterSet(django_filters.FilterSet):
+class PowerConnectionFilterSet(BaseFilterSet):
     site = django_filters.CharFilter(
         method='filter_site',
         label='Site (slug)',
@@ -1181,7 +1181,7 @@ class PowerConnectionFilterSet(django_filters.FilterSet):
         )
 
 
-class InterfaceConnectionFilterSet(django_filters.FilterSet):
+class InterfaceConnectionFilterSet(BaseFilterSet):
     site = django_filters.CharFilter(
         method='filter_site',
         label='Site (slug)',
@@ -1215,7 +1215,7 @@ class InterfaceConnectionFilterSet(django_filters.FilterSet):
         )
 
 
-class PowerPanelFilterSet(django_filters.FilterSet):
+class PowerPanelFilterSet(BaseFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'
@@ -1264,7 +1264,7 @@ class PowerPanelFilterSet(django_filters.FilterSet):
         return queryset.filter(qs_filter)
 
 
-class PowerFeedFilterSet(CustomFieldFilterSet, CreatedUpdatedFilterSet):
+class PowerFeedFilterSet(BaseFilterSet, CustomFieldFilterSet, CreatedUpdatedFilterSet):
     id__in = NumericInFilter(
         field_name='id',
         lookup_expr='in'

netbox/extras/filters.py

@@ -4,6 +4,7 @@ from django.db.models import Q
 
 from dcim.models import DeviceRole, Platform, Region, Site
 from tenancy.models import Tenant, TenantGroup
+from utilities.filters import BaseFilterSet
 from virtualization.models import Cluster, ClusterGroup
 from .choices import *
 from .models import ConfigContext, CustomField, Graph, ExportTemplate, ObjectChange, Tag
@@ -89,21 +90,21 @@ class CustomFieldFilterSet(django_filters.FilterSet):
             self.filters['cf_{}'.format(cf.name)] = CustomFieldFilter(field_name=cf.name, custom_field=cf)
 
 
-class GraphFilterSet(django_filters.FilterSet):
+class GraphFilterSet(BaseFilterSet):
 
     class Meta:
         model = Graph
         fields = ['type', 'name', 'template_language']
 
 
-class ExportTemplateFilterSet(django_filters.FilterSet):
+class ExportTemplateFilterSet(BaseFilterSet):
 
     class Meta:
         model = ExportTemplate
         fields = ['content_type', 'name', 'template_language']
 
 
-class TagFilterSet(django_filters.FilterSet):
+class TagFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',
@@ -122,7 +123,7 @@ class TagFilterSet(django_filters.FilterSet):
         )
 
 
-class ConfigContextFilterSet(django_filters.FilterSet):
+class ConfigContextFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',
@@ -234,7 +235,7 @@ class ConfigContextFilterSet(django_filters.FilterSet):
 # Filter for Local Config Context Data
 #
 
-class LocalConfigContextFilterSet(django_filters.FilterSet):
+class LocalConfigContextFilterSet(BaseFilterSet):
     local_context_data = django_filters.BooleanFilter(
         method='_local_context_data',
         label='Has local config context data',
@@ -244,7 +245,7 @@ class LocalConfigContextFilterSet(django_filters.FilterSet):
         return queryset.exclude(local_context_data__isnull=value)
 
 
-class ObjectChangeFilterSet(django_filters.FilterSet):
+class ObjectChangeFilterSet(BaseFilterSet):
     q = django_filters.CharFilter(
         method='search',
         label='Search',

netbox/tenancy/filters.py

@@ -2,7 +2,7 @@ import django_filters
 from django.db.models import Q
 
 from extras.filters import CustomFieldFilterSet, CreatedUpdatedFilterSet
-from utilities.filters import NameSlugSearchFilterSet, NumericInFilter, TagFilter
+from utilities.filters import BaseFilterSet, NameSlugSearchFilterSet, NumericInFilter, TagFilter
 from .models import Tenant, TenantGroup
 
 

netbox/utilities/constants.py

@@ -27,3 +27,41 @@ COLOR_CHOICES = (
     ('111111', 'Black'),
     ('ffffff', 'White'),
 )
+
+
+#
+# Filter lookup expressions
+#
+
+FILTER_CHAR_BASED_LOOKUP_MAP = dict(
+    n='exact',
+    ic='icontains',
+    nic='icontains',
+    iew='iendswith',
+    niew='iendswith',
+    isw='istartswith',
+    nisw='istartswith',
+    ie='iexact',
+    nie='iexact'
+)
+
+FILTER_NUMERIC_BASED_LOOKUP_MAP = dict(
+    n='exact',
+    lte='lte',
+    lt='lt',
+    gte='gte',
+    gt='gt'
+)
+
+FILTER_LOOKUP_HELP_TEXT_MAP = dict(
+    icontains='case insensitive contains',
+    iendswith='case insensitive ends with',
+    istartswith='case insensitive starts with',
+    iexact='case insensitive exact',
+    exact='case sensitive exact',
+    lt='less than',
+    lte='less than or equal',
+    gt='greater than',
+    gte='greater than or equal',
+    n='negated'
+)

netbox/utilities/filters.py

@@ -3,7 +3,12 @@ from dcim.forms import MACAddressField
 from django import forms
 from django.conf import settings
 from django.db import models
+from django_filters.utils import get_model_field
+
 from extras.models import Tag
+from utilities.constants import (
+    FILTER_CHAR_BASED_LOOKUP_MAP, FILTER_LOOKUP_HELP_TEXT_MAP, FILTER_NUMERIC_BASED_LOOKUP_MAP
+)
 
 
 def multivalue_field_factory(field_class):
@@ -111,6 +116,92 @@ class TagFilter(django_filters.ModelMultipleChoiceFilter):
 # FilterSets
 #
 
+class BaseFilterSet(django_filters.FilterSet):
+    """
+    A base filterset which provides common functionaly to all NetBox filtersets
+    """
+    @classmethod
+    def get_filters(cls):
+        """
+        Override filter generation to support dynamic lookup expressions for certain filter types.
+
+        For specific filter types, new filters are created based on defined lookup expressions in
+        the form `<field_name>_<lookup_expr>`
+        """
+        filters = super().get_filters()
+
+        new_filters = {}
+        for existing_filter_name, existing_filter in filters.items():
+            # Loop over existing filters to extract metadata by which to create new filters
+
+            # It the filter makes use of a custom filter method or lookup expression skip it
+            # as we cannot sanely handle these cases in a generic mannor
+            if existing_filter.method is not None or existing_filter.lookup_expr != 'exact':
+                continue
+
+            # Choose the lookup expression map based on the filter type
+            if isinstance(existing_filter, (
+                django_filters.filters.CharFilter,
+                django_filters.MultipleChoiceFilter,
+                MultiValueCharFilter,
+                MultiValueMACAddressFilter,
+                TagFilter
+            )):
+                lookup_map = FILTER_CHAR_BASED_LOOKUP_MAP
+
+            elif isinstance(existing_filter, (
+                MultiValueDateFilter,
+                MultiValueDateTimeFilter,
+                MultiValueNumberFilter,
+                MultiValueTimeFilter
+            )):
+                lookup_map = FILTER_NUMERIC_BASED_LOOKUP_MAP
+
+            elif isinstance(existing_filter, (
+                django_filters.ModelChoiceFilter,
+                django_filters.ModelMultipleChoiceFilter,
+                NumericInFilter,
+                TreeNodeMultipleChoiceFilter,
+            )):
+                # These filter types support only negation
+                lookup_map = dict(
+                    n='exact'
+                )
+
+            else:
+                # Do no augment any other filter types with more lookup expressions
+                continue
+
+            # Get properties of the existing filter for later use
+            field_name = existing_filter.field_name
+            field = get_model_field(cls._meta.model, field_name)
+
+            # Create new filters for each lookup expression in the map
+            for lookup_name, lookup_expr in lookup_map.items():
+                new_filter_name = '{}_{}'.format(existing_filter_name, lookup_name)
+                
+                try:
+                    print(existing_filter_name)
+                    new_filter = cls.filter_for_field(field, field_name, lookup_expr)
+                except django_filters.exceptions.FieldLookupError:
+                    # The filter could not be created because the lookup expression is not supported
+                    continue
+
+                help_text = FILTER_LOOKUP_HELP_TEXT_MAP[lookup_expr]
+                
+                if lookup_name.startswith('n'):
+                    # This is a negation filter which requires a queryselt.exclud() clause
+                    new_filter.exclude = True
+                    help_text = 'negated {}'.format(help_text)
+
+                new_filter.extra = existing_filter.extra
+                new_filter.extra['help_text'] = '{} - {}'.format(field_name, help_text)
+                new_filters[new_filter_name] = new_filter
+
+        filters.update(new_filters)
+        return filters
+
+
 class NameSlugSearchFilterSet(django_filters.FilterSet):
     """
     A base class for adding the search method to models which only expose the `name` and `slug` fields