Merge branch 'develop' into develop-2.10

docs/release-notes/version-2.9.md

@@ -1,15 +1,21 @@
 # NetBox v2.9
 
-## v2.9.9 (FUTURE)
+## v2.9.9 (2020-11-09)
 
 ### Enhancements
 
 * [#5304](https://github.com/netbox-community/netbox/issues/5304) - Return server error messages as JSON when handling REST API requests
 * [#5310](https://github.com/netbox-community/netbox/issues/5310) - Link to rack groups within rack list table
+* [#5327](https://github.com/netbox-community/netbox/issues/5327) - Be more strict when capturing anticipated ImportError exceptions
 
 ### Bug Fixes
 
 * [#5271](https://github.com/netbox-community/netbox/issues/5271) - Fix auto-population of region field when editing a device
+* [#5314](https://github.com/netbox-community/netbox/issues/5314) - Fix config context rendering when multiple tags are assigned to an object
+* [#5316](https://github.com/netbox-community/netbox/issues/5316) - Dry running scripts should not trigger webhooks
+* [#5324](https://github.com/netbox-community/netbox/issues/5324) - Add missing template extension tags for plugins for VM interface view
+* [#5328](https://github.com/netbox-community/netbox/issues/5328) - Fix CreatedUpdatedFilterTest when running in non-UTC timezone
+* [#5331](https://github.com/netbox-community/netbox/issues/5331) - Fix filtering of sites by null region
 
 
 ---

netbox/dcim/api/views.py

@@ -391,9 +391,7 @@ class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
         if device.platform is None:
             raise ServiceUnavailable("No platform is configured for this device.")
         if not device.platform.napalm_driver:
-            raise ServiceUnavailable("No NAPALM driver is configured for this device's platform {}.".format(
-                device.platform
-            ))
+            raise ServiceUnavailable(f"No NAPALM driver is configured for this device's platform: {device.platform}.")
 
         # Check for primary IP address from NetBox object
         if device.primary_ip:
@@ -402,21 +400,25 @@ class DeviceViewSet(CustomFieldModelViewSet, ConfigContextQuerySetMixin):
             # Raise exception for no IP address and no Name if device.name does not exist
             if not device.name:
                 raise ServiceUnavailable(
-                    "This device does not have a primary IP address or device name to lookup configured.")
+                    "This device does not have a primary IP address or device name to lookup configured."
+                )
             try:
                 # Attempt to complete a DNS name resolution if no primary_ip is set
                 host = socket.gethostbyname(device.name)
             except socket.gaierror:
                 # Name lookup failure
                 raise ServiceUnavailable(
-                    f"Name lookup failure, unable to resolve IP address for {device.name}. Please set Primary IP or setup name resolution.")
+                    f"Name lookup failure, unable to resolve IP address for {device.name}. Please set Primary IP or "
+                    f"setup name resolution.")
 
         # Check that NAPALM is installed
         try:
             import napalm
             from napalm.base.exceptions import ModuleImportError
-        except ImportError:
-            raise ServiceUnavailable("NAPALM is not installed. Please see the documentation for instructions.")
+        except ModuleNotFoundError as e:
+            if getattr(e, 'name') == 'napalm':
+                raise ServiceUnavailable("NAPALM is not installed. Please see the documentation for instructions.")
+            raise e
 
         # Validate the configured driver
         try:

netbox/extras/querysets.py

@@ -42,7 +42,7 @@ class ConfigContextQuerySet(RestrictedQuerySet):
             Q(tenants=obj.tenant) | Q(tenants=None),
             Q(tags__slug__in=obj.tags.slugs()) | Q(tags=None),
             is_active=True,
-        ).order_by('weight', 'name')
+        ).order_by('weight', 'name').distinct()
 
         if aggregate_data:
             return queryset.aggregate(
@@ -77,7 +77,7 @@ class ConfigContextModelQuerySet(RestrictedQuerySet):
                     _data=EmptyGroupByJSONBAgg('data', ordering=['weight', 'name'])
                 ).values("_data")
             )
-        )
+        ).distinct()
 
     def _get_config_context_filters(self):
         # Construct the set of Q objects for the specific object types

netbox/extras/scripts.py

@@ -428,8 +428,11 @@ def run_script(data, request, commit=True, *args, **kwargs):
     # Add the current request as a property of the script
     script.request = request
 
-    with change_logging(request):
-
+    def _run_script():
+        """
+        Core script execution task. We capture this within a subfunction to allow for conditionally wrapping it with
+        the change_logging context manager (which is bypassed if commit == False).
+        """
         try:
             with transaction.atomic():
                 script.output = script.run(data=data, commit=commit)
@@ -456,6 +459,14 @@ def run_script(data, request, commit=True, *args, **kwargs):
 
         logger.info(f"Script completed in {job_result.duration}")
 
+    # Execute the script. If commit is True, wrap it with the change_logging context manager to ensure we process
+    # change logging, webhooks, etc.
+    if commit:
+        with change_logging(request):
+            _run_script()
+    else:
+        _run_script()
+
     # Delete any previous terminal state results
     JobResult.objects.filter(
         obj_type=job_result.obj_type,

netbox/extras/templatetags/custom_links.py

@@ -16,7 +16,7 @@ GROUP_BUTTON = '<div class="btn-group">\n' \
                '{} <span class="caret"></span>\n' \
                '</button>\n' \
                '<ul class="dropdown-menu pull-right">\n' \
-               '{}</ul></div>'
+               '{}</ul></div>\n'
 GROUP_LINK = '<li><a href="{}"{}>{}</a></li>\n'
 
 

netbox/extras/tests/dummy_plugin/template_content.py

@@ -13,7 +13,7 @@ class SiteContent(PluginTemplateExtension):
     def full_width_page(self):
         return "SITE CONTENT - FULL WIDTH PAGE"
 
-    def full_buttons(self):
+    def buttons(self):
         return "SITE CONTENT - BUTTONS"
 
 

netbox/extras/tests/test_api.py

@@ -4,7 +4,7 @@ from unittest import skipIf
 from django.contrib.contenttypes.models import ContentType
 from django.test import override_settings
 from django.urls import reverse
-from django.utils import timezone
+from django.utils.timezone import make_aware
 from django_rq.queues import get_connection
 from rest_framework import status
 from rq import Worker
@@ -346,8 +346,8 @@ class CreatedUpdatedFilterTest(APITestCase):
 
         # change the created and last_updated of one
         Rack.objects.filter(pk=self.rack2.pk).update(
-            last_updated=datetime.datetime(2001, 2, 3, 1, 2, 3, 4, tzinfo=timezone.utc),
-            created=datetime.datetime(2001, 2, 3)
+            last_updated=make_aware(datetime.datetime(2001, 2, 3, 1, 2, 3, 4)),
+            created=make_aware(datetime.datetime(2001, 2, 3))
         )
 
     def test_get_rack_created(self):

netbox/extras/tests/test_models.py

@@ -33,6 +33,7 @@ class ConfigContextTest(TestCase):
         self.tenantgroup = TenantGroup.objects.create(name="Tenant Group")
         self.tenant = Tenant.objects.create(name="Tenant", group=self.tenantgroup)
         self.tag = Tag.objects.create(name="Tag", slug="tag")
+        self.tag2 = Tag.objects.create(name="Tag2", slug="tag2")
 
         self.device = Device.objects.create(
             name='Device 1',
@@ -286,3 +287,37 @@ class ConfigContextTest(TestCase):
 
         annotated_queryset = VirtualMachine.objects.filter(name=virtual_machine.name).annotate_config_context_data()
         self.assertEqual(virtual_machine.get_config_context(), annotated_queryset[0].get_config_context())
+
+    def test_multiple_tags_return_distinct_objects(self):
+        """
+        Tagged items use a generic relationship, which results in duplicate rows being returned when queried.
+        This is combatted by by appending distinct() to the config context querysets. This test creates a config
+        context assigned to two tags and ensures objects related by those same two tags result in only a single
+        config context record being returned.
+
+        See https://github.com/netbox-community/netbox/issues/5314
+        """
+        tag_context = ConfigContext.objects.create(
+            name="tag",
+            weight=100,
+            data={
+                "tag": 1
+            }
+        )
+        tag_context.tags.add(self.tag)
+        tag_context.tags.add(self.tag2)
+
+        device = Device.objects.create(
+            name="Device 3",
+            site=self.site,
+            tenant=self.tenant,
+            platform=self.platform,
+            device_role=self.devicerole,
+            device_type=self.devicetype
+        )
+        device.tags.add(self.tag)
+        device.tags.add(self.tag2)
+
+        annotated_queryset = Device.objects.filter(name=device.name).annotate_config_context_data()
+        self.assertEqual(ConfigContext.objects.get_for_object(device).count(), 1)
+        self.assertEqual(device.get_config_context(), annotated_queryset[0].get_config_context())

netbox/netbox/authentication.py

@@ -137,19 +137,24 @@ class LDAPBackend:
 
     def __new__(cls, *args, **kwargs):
         try:
-            import ldap
             from django_auth_ldap.backend import LDAPBackend as LDAPBackend_, LDAPSettings
-        except ImportError:
-            raise ImproperlyConfigured(
-                "LDAP authentication has been configured, but django-auth-ldap is not installed."
-            )
+            import ldap
+        except ModuleNotFoundError as e:
+            if getattr(e, 'name') == 'django_auth_ldap':
+                raise ImproperlyConfigured(
+                    "LDAP authentication has been configured, but django-auth-ldap is not installed."
+                )
+            raise e
 
         try:
             from netbox import ldap_config
-        except ImportError:
-            raise ImproperlyConfigured(
-                "ldap_config.py does not exist"
-            )
+        except ModuleNotFoundError as e:
+            if getattr(e, 'name') == 'ldap_config':
+                raise ImproperlyConfigured(
+                    "LDAP configuration file not found: Check that ldap_config.py has been created alongside "
+                    "configuration.py."
+                )
+            raise e
 
         try:
             getattr(ldap_config, 'AUTH_LDAP_SERVER_URI')

netbox/netbox/settings.py

@@ -38,10 +38,12 @@ if platform.python_version_tuple() < ('3', '6'):
 # Import configuration parameters
 try:
     from netbox import configuration
-except ImportError:
-    raise ImproperlyConfigured(
-        "Configuration file is not present. Please define netbox/netbox/configuration.py per the documentation."
-    )
+except ModuleNotFoundError as e:
+    if getattr(e, 'name') == 'configuration':
+        raise ImproperlyConfigured(
+            "Configuration file is not present. Please define netbox/netbox/configuration.py per the documentation."
+        )
+    raise
 
 # Enforce required configuration parameters
 for parameter in ['ALLOWED_HOSTS', 'DATABASE', 'SECRET_KEY', 'REDIS']:
@@ -165,11 +167,13 @@ if STORAGE_BACKEND is not None:
 
         try:
             import storages.utils
-        except ImportError:
-            raise ImproperlyConfigured(
-                "STORAGE_BACKEND is set to {} but django-storages is not present. It can be installed by running 'pip "
-                "install django-storages'.".format(STORAGE_BACKEND)
-            )
+        except ModuleNotFoundError as e:
+            if getattr(e, 'name') == 'storages':
+                raise ImproperlyConfigured(
+                    f"STORAGE_BACKEND is set to {STORAGE_BACKEND} but django-storages is not present. It can be "
+                    f"installed by running 'pip install django-storages'."
+                )
+            raise e
 
         # Monkey-patch django-storages to fetch settings from STORAGE_CONFIG
         def _setting(name, default=None):
@@ -587,11 +591,13 @@ for plugin_name in PLUGINS:
     # Import plugin module
     try:
         plugin = importlib.import_module(plugin_name)
-    except ImportError:
-        raise ImproperlyConfigured(
-            "Unable to import plugin {}: Module not found. Check that the plugin module has been installed within the "
-            "correct Python environment.".format(plugin_name)
-        )
+    except ModuleNotFoundError as e:
+        if getattr(e, 'name') == plugin_name:
+            raise ImproperlyConfigured(
+                "Unable to import plugin {}: Module not found. Check that the plugin module has been installed within the "
+                "correct Python environment.".format(plugin_name)
+            )
+        raise e
 
     # Determine plugin config and add to INSTALLED_APPS.
     try:

netbox/templates/virtualization/vminterface.html

@@ -1,5 +1,6 @@
 {% extends 'base.html' %}
 {% load helpers %}
+{% load plugins %}
 
 {% block header %}
     <div class="row noprint">
@@ -12,6 +13,7 @@
         </div>
     </div>
     <div class="pull-right noprint">
+        {% plugin_buttons vminterface %}
         {% if perms.virtualization.change_vminterface %}
             <a href="{% url 'virtualization:vminterface_edit' pk=vminterface.pk %}" class="btn btn-warning">
                 <span class="mdi mdi-pencil" aria-hidden="true"></span> Edit
@@ -82,9 +84,11 @@
                 </tr>
             </table>
         </div>
+          {% plugin_left_page vminterface %}
     </div>
     <div class="col-md-6">
         {% include 'extras/inc/tags_panel.html' with tags=vminterface.tags.all %}
+          {% plugin_right_page vminterface %}
     </div>
 </div>
 <div class="row">
@@ -97,4 +101,9 @@
         {% include 'panel_table.html' with table=vlan_table heading="VLANs" %}
     </div>
 </div>
+    <div class="row">
+        <div class="col-md-12">
+            {% plugin_full_width_page vminterface %}
+        </div>
+    </div>
 {% endblock %}

netbox/utilities/filters.py

@@ -70,9 +70,9 @@ class TreeNodeMultipleChoiceFilter(django_filters.ModelMultipleChoiceFilter):
     Filters for a set of Models, including all descendant models within a Tree.  Example: [<Region: R1>,<Region: R2>]
     """
     def get_filter_predicate(self, v):
-        # null value filtering
+        # Null value filtering
         if v is None:
-            return {self.field_name.replace('in', 'isnull'): True}
+            return {f"{self.field_name}__isnull": True}
         return super().get_filter_predicate(v)
 
     def filter(self, qs, value):

netbox/utilities/tests/test_filters.py

@@ -23,7 +23,8 @@ class TreeNodeMultipleChoiceFilterTest(TestCase):
     class SiteFilterSet(django_filters.FilterSet):
         region = TreeNodeMultipleChoiceFilter(
             queryset=Region.objects.all(),
-            field_name='region__in',
+            field_name='region',
+            lookup_expr='in',
             to_field_name='slug',
         )