Merge branch 'develop' into develop-2.6

CHANGELOG.md

@@ -30,6 +30,31 @@ to now use "Extras | Tag."
 
 ---
 
+v2.5.9 (2019-04-01)
+
+## Enhancements
+
+* [#2933](https://github.com/digitalocean/netbox/issues/2933) - Add username to outbound webhook requests
+* [#3011](https://github.com/digitalocean/netbox/issues/3011) - Add SSL support for django-rq (requires django-rq v1.3.1+)
+* [#3025](https://github.com/digitalocean/netbox/issues/3025) - Add request ID to outbound webhook requests (for correlating all changes part of a single request)
+
+## Bug Fixes
+
+* [#2207](https://github.com/digitalocean/netbox/issues/2207) - Fixes deterministic ordering of interfaces
+* [#2577](https://github.com/digitalocean/netbox/issues/2577) - Clarification of wording in API regarding filtering
+* [#2924](https://github.com/digitalocean/netbox/issues/2924) - Add interface type for QSFP28 50GE
+* [#2936](https://github.com/digitalocean/netbox/issues/2936) - Fix device role selection showing duplicate first entry
+* [#2998](https://github.com/digitalocean/netbox/issues/2998) - Limit device query to non-racked devices if no rack selected when creating a cable
+* [#3001](https://github.com/digitalocean/netbox/issues/3001) - Fix API representation of ObjectChange `action` and add `changed_object_type`
+* [#3014](https://github.com/digitalocean/netbox/issues/3014) - Fixes VM Role filtering
+* [#3019](https://github.com/digitalocean/netbox/issues/3019) - Fix tag population when running NetBox within a path
+* [#3022](https://github.com/digitalocean/netbox/issues/3022) - Add missing cable termination types to DCIM `_choices` endpoint
+* [#3026](https://github.com/digitalocean/netbox/issues/3026) - Tweak prefix/IP filter forms to filter using VRF ID rather than route distinguisher
+* [#3027](https://github.com/digitalocean/netbox/issues/3027) - Ignore empty local context data when rendering config contexts
+* [#3032](https://github.com/digitalocean/netbox/issues/3032) - Save assigned tags when creating a new secret
+
+---
+
 v2.5.8 (2019-03-11)
 
 ## Enhancements

README.md

@@ -45,13 +45,13 @@ and run `upgrade.sh`.
 
 ## Supported SDK
 
-- [pynetbox](https://github.com/digitalocean/pynetbox) Python API client library for Netbox.
+- [pynetbox](https://github.com/digitalocean/pynetbox) - A Python API client library for Netbox
 
 ## Community SDK
 
-- [netbox-client-ruby](https://github.com/ninech/netbox-client-ruby) A ruby client library for Netbox v2.
+- [netbox-client-ruby](https://github.com/ninech/netbox-client-ruby) - A Ruby client library for Netbox
+- [powerbox](https://github.com/BatmanAMA/powerbox) - A PowerShell library for Netbox
 
 ## Ansible Inventory
 
-- [netbox-as-ansible-inventory](https://github.com/AAbouZaid/netbox-as-ansible-inventory) Ansible dynamic inventory script for Netbox.
-
+- [netbox-as-ansible-inventory](https://github.com/AAbouZaid/netbox-as-ansible-inventory) - Ansible dynamic inventory script for Netbox

docs/api/overview.md

@@ -261,7 +261,7 @@ A list of objects retrieved via the API can be filtered by passing one or more q
 GET /api/ipam/prefixes/?status=1
 ```
 
-The same filter can be incldued multiple times. These will effect a logical OR and return objects matching any of the given values. For example, the following will return all active and reserved prefixes:
+Certain filters can be included multiple times within a single request. These will effect a logical OR and return objects matching any of the given values. For example, the following will return all active and reserved prefixes:
 
 ```
 GET /api/ipam/prefixes/?status=1&status=2

docs/configuration/optional-settings.md

@@ -283,6 +283,7 @@ REDIS = {
     'PASSWORD': '',
     'DATABASE': 0,
     'DEFAULT_TIMEOUT': 300,
+    'SSL': False,
 }
 ```
 
@@ -315,3 +316,9 @@ The TCP port to use when connecting to the Redis server.
 Default: None
 
 The password to use when authenticating to the Redis server (optional).
+
+### SSL
+
+Default: False
+
+Use secure sockets layer to encrypt the connections to the Redis server.

netbox/dcim/api/serializers.py

@@ -1,3 +1,4 @@
+from django.contrib.contenttypes.models import ContentType
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
 from rest_framework.validators import UniqueTogetherValidator
@@ -506,8 +507,12 @@ class InventoryItemSerializer(TaggitSerializer, ValidatedModelSerializer):
 #
 
 class CableSerializer(ValidatedModelSerializer):
-    termination_a_type = ContentTypeField()
-    termination_b_type = ContentTypeField()
+    termination_a_type = ContentTypeField(
+        queryset=ContentType.objects.filter(model__in=CABLE_TERMINATION_TYPES)
+    )
+    termination_b_type = ContentTypeField(
+        queryset=ContentType.objects.filter(model__in=CABLE_TERMINATION_TYPES)
+    )
     termination_a = serializers.SerializerMethodField(read_only=True)
     termination_b = serializers.SerializerMethodField(read_only=True)
     status = ChoiceField(choices=CONNECTION_STATUS_CHOICES, required=False)

netbox/dcim/api/views.py

@@ -1,7 +1,7 @@
 from collections import OrderedDict
 
 from django.conf import settings
-from django.db.models import F, Q
+from django.db.models import F
 from django.http import HttpResponseForbidden
 from django.shortcuts import get_object_or_404
 from drf_yasg import openapi
@@ -35,7 +35,7 @@ from .exceptions import MissingFilterException
 
 class DCIMFieldChoicesViewSet(FieldChoicesViewSet):
     fields = (
-        (Cable, ['length_unit', 'status', 'type']),
+        (Cable, ['length_unit', 'status', 'termination_a_type', 'termination_b_type', 'type']),
         (ConsolePort, ['connection_status']),
         (Device, ['face', 'status']),
         (DeviceType, ['subdevice_role']),

netbox/dcim/constants.py

@@ -83,6 +83,7 @@ IFACE_FF_10GE_XENPAK = 1310
 IFACE_FF_10GE_X2 = 1320
 IFACE_FF_25GE_SFP28 = 1350
 IFACE_FF_40GE_QSFP_PLUS = 1400
+IFACE_FF_50GE_QSFP28 = 1420
 IFACE_FF_100GE_CFP = 1500
 IFACE_FF_100GE_CFP2 = 1510
 IFACE_FF_100GE_CFP4 = 1520
@@ -164,6 +165,7 @@ IFACE_FF_CHOICES = [
             [IFACE_FF_10GE_X2, 'X2 (10GE)'],
             [IFACE_FF_25GE_SFP28, 'SFP28 (25GE)'],
             [IFACE_FF_40GE_QSFP_PLUS, 'QSFP+ (40GE)'],
+            [IFACE_FF_50GE_QSFP28, 'QSFP28 (50GE)'],
             [IFACE_FF_100GE_CFP, 'CFP (100GE)'],
             [IFACE_FF_100GE_CFP2, 'CFP2 (100GE)'],
             [IFACE_FF_200GE_CFP2, 'CFP2 (200GE)'],

netbox/dcim/forms.py

@@ -1700,7 +1700,6 @@ class DeviceFilterForm(BootstrapMixin, CustomFieldFilterForm):
         widget=APISelectMultiple(
             api_url="/api/dcim/device-roles/",
             value_field="slug",
-            null_option=True,
         )
     )
     tenant = FilterChoiceField(

netbox/dcim/managers.py

@@ -64,11 +64,15 @@ class InterfaceManager(Manager):
 
         The original `name` field is considered in its entirety to serve as a fallback in the event interfaces do not
         match any of the prescribed fields.
+
+        The `id` field is included to enforce deterministic ordering of interfaces in similar vein of other device
+        components.
         """
 
         sql_col = '{}.name'.format(self.model._meta.db_table)
         ordering = [
-            '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name',
+            '_slot', '_subslot', '_position', '_subposition', '_type', '_id', '_channel', '_vc', 'name', 'pk'
+
         ]
 
         fields = {

netbox/extras/api/serializers.py

@@ -1,3 +1,4 @@
+from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
@@ -89,7 +90,9 @@ class TagSerializer(ValidatedModelSerializer):
 #
 
 class ImageAttachmentSerializer(ValidatedModelSerializer):
-    content_type = ContentTypeField()
+    content_type = ContentTypeField(
+        queryset=ContentType.objects.all()
+    )
     parent = serializers.SerializerMethodField(read_only=True)
 
     class Meta:
@@ -207,14 +210,25 @@ class ReportDetailSerializer(ReportSerializer):
 #
 
 class ObjectChangeSerializer(serializers.ModelSerializer):
-    user = NestedUserSerializer(read_only=True)
-    content_type = ContentTypeField(read_only=True)
-    changed_object = serializers.SerializerMethodField(read_only=True)
+    user = NestedUserSerializer(
+        read_only=True
+    )
+    action = ChoiceField(
+        choices=OBJECTCHANGE_ACTION_CHOICES,
+        read_only=True
+    )
+    changed_object_type = ContentTypeField(
+        read_only=True
+    )
+    changed_object = serializers.SerializerMethodField(
+        read_only=True
+    )
 
     class Meta:
         model = ObjectChange
         fields = [
-            'id', 'time', 'user', 'user_name', 'request_id', 'action', 'content_type', 'changed_object', 'object_data',
+            'id', 'time', 'user', 'user_name', 'request_id', 'action', 'changed_object_type', 'changed_object',
+            'object_data',
         ]
 
     @swagger_serializer_method(serializer_or_field=serializers.DictField)

netbox/extras/api/views.py

@@ -11,7 +11,7 @@ from rest_framework.viewsets import ReadOnlyModelViewSet, ViewSet
 
 from extras import filters
 from extras.models import (
-    ConfigContext, CustomField, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, TopologyMap,
+    ConfigContext, CustomFieldChoice, ExportTemplate, Graph, ImageAttachment, ObjectChange, ReportResult, TopologyMap,
     Tag,
 )
 from extras.reports import get_report, get_reports
@@ -25,8 +25,8 @@ from . import serializers
 
 class ExtrasFieldChoicesViewSet(FieldChoicesViewSet):
     fields = (
-        (CustomField, ['type']),
         (Graph, ['type']),
+        (ObjectChange, ['action']),
     )
 
 

netbox/extras/apps.py

@@ -22,6 +22,7 @@ class ExtrasConfig(AppConfig):
                     port=settings.REDIS_PORT,
                     db=settings.REDIS_DATABASE,
                     password=settings.REDIS_PASSWORD or None,
+                    ssl=settings.REDIS_SSL,
                 )
                 rs.ping()
             except redis.exceptions.ConnectionError:

netbox/extras/middleware.py

@@ -37,7 +37,7 @@ def _record_object_deleted(request, instance, **kwargs):
     if hasattr(instance, 'log_change'):
         instance.log_change(request.user, request.id, OBJECTCHANGE_ACTION_DELETE)
 
-    enqueue_webhooks(instance, OBJECTCHANGE_ACTION_DELETE)
+    enqueue_webhooks(instance, request.user, request.id, OBJECTCHANGE_ACTION_DELETE)
 
 
 class ObjectChangeMiddleware(object):
@@ -83,7 +83,7 @@ class ObjectChangeMiddleware(object):
                 obj.log_change(request.user, request.id, action)
 
             # Enqueue webhooks
-            enqueue_webhooks(obj, action)
+            enqueue_webhooks(obj, request.user, request.id, action)
 
         # Housekeeping: 1% chance of clearing out expired ObjectChanges
         if _thread_locals.changed_objects and settings.CHANGELOG_RETENTION and random.randint(1, 100) == 1:

netbox/extras/models.py

@@ -722,7 +722,7 @@ class ConfigContextModel(models.Model):
             data = deepmerge(data, context.data)
 
         # If the object has local config context data defined, merge it last
-        if self.local_context_data is not None:
+        if self.local_context_data:
             data = deepmerge(data, self.local_context_data)
 
         return data

netbox/extras/webhooks.py

@@ -9,7 +9,7 @@ from utilities.api import get_serializer_for_model
 from .constants import WEBHOOK_MODELS
 
 
-def enqueue_webhooks(instance, action):
+def enqueue_webhooks(instance, user, request_id, action):
     """
     Find Webhook(s) assigned to this instance + action and enqueue them
     to be processed
@@ -47,5 +47,7 @@ def enqueue_webhooks(instance, action):
                 serializer.data,
                 instance._meta.model_name,
                 action,
-                str(datetime.datetime.now())
+                str(datetime.datetime.now()),
+                user.username,
+                request_id
             )

netbox/extras/webhooks_worker.py

@@ -10,7 +10,7 @@ from extras.constants import WEBHOOK_CT_JSON, WEBHOOK_CT_X_WWW_FORM_ENCODED, OBJ
 
 
 @job('default')
-def process_webhook(webhook, data, model_name, event, timestamp):
+def process_webhook(webhook, data, model_name, event, timestamp, username, request_id):
     """
     Make a POST request to the defined Webhook
     """
@@ -18,6 +18,8 @@ def process_webhook(webhook, data, model_name, event, timestamp):
         'event': dict(OBJECTCHANGE_ACTION_CHOICES)[event].lower(),
         'timestamp': timestamp,
         'model': model_name,
+        'username': username,
+        'request_id': request_id,
         'data': data
     }
     headers = {

netbox/ipam/api/serializers.py

@@ -128,6 +128,7 @@ class VLANSerializer(TaggitSerializer, CustomFieldModelSerializer):
 #
 
 class PrefixSerializer(TaggitSerializer, CustomFieldModelSerializer):
+    family = ChoiceField(choices=AF_CHOICES, read_only=True)
     site = NestedSiteSerializer(required=False, allow_null=True)
     vrf = NestedVRFSerializer(required=False, allow_null=True)
     tenant = NestedTenantSerializer(required=False, allow_null=True)
@@ -189,6 +190,7 @@ class IPAddressInterfaceSerializer(WritableNestedSerializer):
 
 
 class IPAddressSerializer(TaggitSerializer, CustomFieldModelSerializer):
+    family = ChoiceField(choices=AF_CHOICES, read_only=True)
     vrf = NestedVRFSerializer(required=False, allow_null=True)
     tenant = NestedTenantSerializer(required=False, allow_null=True)
     status = ChoiceField(choices=IPADDRESS_STATUS_CHOICES, required=False)

netbox/ipam/forms.py

@@ -524,14 +524,12 @@ class PrefixFilterForm(BootstrapMixin, CustomFieldFilterForm):
         label='Mask length',
         widget=StaticSelect2()
     )
-    vrf = FilterChoiceField(
+    vrf_id = FilterChoiceField(
         queryset=VRF.objects.all(),
-        to_field_name='rd',
         label='VRF',
         null_label='-- Global --',
         widget=APISelectMultiple(
             api_url="/api/ipam/vrfs/",
-            value_field="rd",
             null_option=True,
         )
     )
@@ -973,14 +971,12 @@ class IPAddressFilterForm(BootstrapMixin, CustomFieldFilterForm):
         label='Mask length',
         widget=StaticSelect2()
     )
-    vrf = FilterChoiceField(
+    vrf_id = FilterChoiceField(
         queryset=VRF.objects.all(),
-        to_field_name='rd',
         label='VRF',
         null_label='-- Global --',
         widget=APISelectMultiple(
             api_url="/api/ipam/vrfs/",
-            value_field="rd",
             null_option=True,
         )
     )

netbox/netbox/configuration.example.py

@@ -132,6 +132,7 @@ REDIS = {
     'PASSWORD': '',
     'DATABASE': 0,
     'DEFAULT_TIMEOUT': 300,
+    'SSL': False,
 }
 
 # The file path where custom reports will be stored. A trailing slash is not needed. Note that the default value of

netbox/netbox/settings.py

@@ -131,6 +131,7 @@ REDIS_PORT = REDIS.get('PORT', 6379)
 REDIS_PASSWORD = REDIS.get('PASSWORD', '')
 REDIS_DATABASE = REDIS.get('DATABASE', 0)
 REDIS_DEFAULT_TIMEOUT = REDIS.get('DEFAULT_TIMEOUT', 300)
+REDIS_SSL = REDIS.get('SSL', False)
 
 # Email
 EMAIL_HOST = EMAIL.get('SERVER')
@@ -291,6 +292,7 @@ RQ_QUEUES = {
         'DB': REDIS_DATABASE,
         'PASSWORD': REDIS_PASSWORD,
         'DEFAULT_TIMEOUT': REDIS_DEFAULT_TIMEOUT,
+        'SSL': REDIS_SSL,
     }
 }
 

netbox/project-static/js/forms.js

@@ -155,10 +155,13 @@ $(document).ready(function() {
 
                 filter_for_elements.each(function(index, filter_for_element) {
                     var param_name = $(filter_for_element).attr(attr_name);
+                    var is_nullable = $(filter_for_element).attr("nullable");
                     var value = $(filter_for_element).val();
 
                     if (param_name && value) {
                         parameters[param_name] = value;
+                    } else if (param_name && is_nullable) {
+                        parameters[param_name] = "null";
                     }
                 });
 
@@ -247,7 +250,7 @@ $(document).ready(function() {
 
         ajax: {
             delay: 250,
-            url: "/api/extras/tags/",
+            url: netbox_api_path + "extras/tags/",
 
             data: function(params) {
                 // Paging. Note that `params.page` indexes at 1

netbox/secrets/views.py

@@ -120,6 +120,8 @@ def secret_add(request, pk):
                     secret.plaintext = str(form.cleaned_data['plaintext'])
                     secret.encrypt(master_key)
                     secret.save()
+                    form.save_m2m()
+
                     messages.success(request, "Added new secret: {}.".format(secret))
                     if '_addanother' in request.POST:
                         return redirect('dcim:device_addsecret', pk=device.pk)

netbox/utilities/api.py

@@ -21,6 +21,10 @@ class ServiceUnavailable(APIException):
     default_detail = "Service temporarily unavailable, please try again later."
 
 
+class SerializerNotFound(Exception):
+    pass
+
+
 def get_serializer_for_model(model, prefix=''):
     """
     Dynamically resolve and return the appropriate serializer for a model.
@@ -32,7 +36,9 @@ def get_serializer_for_model(model, prefix=''):
     try:
         return dynamic_import(serializer_name)
     except AttributeError:
-        return None
+        raise SerializerNotFound(
+            "Could not determine serializer for {}.{} with prefix '{}'".format(app_name, model_name, prefix)
+        )
 
 
 #
@@ -100,6 +106,10 @@ class ChoiceField(Field):
 
         return data
 
+    @property
+    def choices(self):
+        return self._choices
+
 
 class ContentTypeField(RelatedField):
     """
@@ -110,10 +120,6 @@ class ContentTypeField(RelatedField):
         "invalid": "Invalid value. Specify a content type as '<app_label>.<model_name>'.",
     }
 
-    # Can't set this as an attribute because it raises an exception when the field is read-only
-    def get_queryset(self):
-        return ContentType.objects.all()
-
     def to_internal_value(self, data):
         try:
             app_label, model = data.split('.')
@@ -234,9 +240,10 @@ class ModelViewSet(_ModelViewSet):
         # exists
         request = self.get_serializer_context()['request']
         if request.query_params.get('brief', False):
-            serializer_class = get_serializer_for_model(self.queryset.model, prefix='Nested')
-            if serializer_class is not None:
-                return serializer_class
+            try:
+                return get_serializer_for_model(self.queryset.model, prefix='Nested')
+            except SerializerNotFound:
+                pass
 
         # Fall back to the hard-coded serializer class
         return self.serializer_class
@@ -256,10 +263,14 @@ class FieldChoicesViewSet(ViewSet):
         self._fields = OrderedDict()
         for cls, field_list in self.fields:
             for field_name in field_list:
+
                 model_name = cls._meta.verbose_name.lower().replace(' ', '-')
                 key = ':'.join([model_name, field_name])
+
+                serializer = get_serializer_for_model(cls)()
                 choices = []
-                for k, v in cls._meta.get_field(field_name).choices:
+
+                for k, v in serializer.get_fields()[field_name].choices.items():
                     if type(v) in [list, tuple]:
                         for k2, v2 in v:
                             choices.append({

netbox/virtualization/forms.py

@@ -348,7 +348,7 @@ class VirtualMachineForm(BootstrapMixin, TenancyForm, CustomFieldForm):
             "role": APISelect(
                 api_url="/api/dcim/device-roles/",
                 additional_query_params={
-                    "vm_role": "true"
+                    "vm_role": "True"
                 }
             ),
             'primary_ip4': StaticSelect2(),
@@ -480,7 +480,7 @@ class VirtualMachineBulkEditForm(BootstrapMixin, AddRemoveTagsForm, CustomFieldB
         widget=APISelect(
             api_url="/api/dcim/device-roles/",
             additional_query_params={
-                "vm_role": "true"
+                "vm_role": "True"
             }
         )
     )
@@ -582,7 +582,7 @@ class VirtualMachineFilterForm(BootstrapMixin, CustomFieldFilterForm):
             value_field="slug",
             null_option=True,
             additional_query_params={
-                'vm_role': 'true'
+                'vm_role': "True"
             }
         )
     )