signals.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398
  1. import logging
  2. from threading import local
  3. from django.contrib.contenttypes.models import ContentType
  4. from django.core.exceptions import ObjectDoesNotExist, ValidationError
  5. from django.core.signals import request_finished
  6. from django.db import transaction
  7. from django.db.models import CASCADE, RESTRICT
  8. from django.db.models.fields.reverse_related import ManyToManyRel, ManyToOneRel
  9. from django.db.models.signals import m2m_changed, post_migrate, post_save, pre_delete, pre_save
  10. from django.dispatch import Signal, receiver
  11. from django.utils.translation import gettext_lazy as _
  12. from django.utils.translation import ngettext
  13. from django_prometheus.models import model_deletes, model_inserts, model_updates
  14. from rq.timeouts import JobTimeoutException
  15. from core.choices import JobStatusChoices, ObjectChangeActionChoices
  16. from core.events import *
  17. from core.exceptions import SyncError
  18. from core.models import ObjectType
  19. from extras.events import enqueue_event
  20. from extras.models import Tag
  21. from extras.utils import run_validators
  22. from netbox.config import get_config
  23. from netbox.context import current_request, events_queue
  24. from netbox.models.features import ChangeLoggingMixin, get_model_features, model_is_public
  25. from utilities.data import get_config_value_ci
  26. from utilities.exceptions import AbortRequest
  27. from .models import ConfigRevision, DataSource, ObjectChange
  28. logger = logging.getLogger('netbox.core.signals')
  29. __all__ = (
  30. 'clear_events',
  31. 'job_end',
  32. 'job_start',
  33. 'post_sync',
  34. 'pre_sync',
  35. )
  36. # Job signals
  37. job_start = Signal()
  38. job_end = Signal()
  39. # DataSource signals
  40. pre_sync = Signal()
  41. post_sync = Signal()
  42. # Event signals
  43. clear_events = Signal()
  44. #
  45. # Object types
  46. #
  47. @receiver(post_migrate)
  48. def update_object_types(sender, **kwargs):
  49. """
  50. Create or update the corresponding ObjectType for each model within the migrated app.
  51. """
  52. for model in sender.get_models():
  53. app_label, model_name = model._meta.label_lower.split('.')
  54. # Determine whether model is public and its supported features
  55. is_public = model_is_public(model)
  56. features = get_model_features(model)
  57. # Create/update the ObjectType for the model
  58. try:
  59. ot = ObjectType.objects.get_by_natural_key(app_label=app_label, model=model_name)
  60. ot.public = is_public
  61. ot.features = features
  62. ot.save()
  63. except ObjectDoesNotExist:
  64. ObjectType.objects.create(
  65. app_label=app_label,
  66. model=model_name,
  67. public=is_public,
  68. features=features,
  69. )
  70. #
  71. # Change logging & event handling
  72. #
  73. # Used to track received signals per object
  74. _signals_received = local()
  75. @receiver(pre_save)
  76. def snapshot_unsnapshotted_object(sender, instance, raw=False, using=None, **kwargs):
  77. """
  78. Generate a pre-change snapshot from the database for an existing object being saved without one.
  79. """
  80. if raw or not isinstance(instance, ChangeLoggingMixin) or current_request.get() is None:
  81. return
  82. if instance._state.adding or instance.pk is None or hasattr(instance, '_prechange_snapshot'):
  83. return
  84. instance.snapshot_from_db(using=using)
  85. @receiver((post_save, m2m_changed))
  86. def handle_changed_object(sender, instance, **kwargs):
  87. """
  88. Fires when an object is created or updated.
  89. """
  90. m2m_changed = False
  91. if not hasattr(instance, 'to_objectchange'):
  92. return
  93. # Get the current request, or bail if not set
  94. request = current_request.get()
  95. if request is None:
  96. return
  97. # Determine the type of change being made
  98. if kwargs.get('created'):
  99. event_type = OBJECT_CREATED
  100. elif 'created' in kwargs:
  101. event_type = OBJECT_UPDATED
  102. elif kwargs.get('action') in ['post_add', 'post_remove'] and kwargs['pk_set']:
  103. # m2m_changed with objects added or removed
  104. m2m_changed = True
  105. event_type = OBJECT_UPDATED
  106. elif kwargs.get('action') == 'post_clear':
  107. # Handle clearing of an M2M field
  108. if kwargs.get('model') == Tag and getattr(instance, '_prechange_snapshot', {}).get('tags'):
  109. # Handle generation of M2M changes for Tags which have a previous value (ignoring changes where the
  110. # prechange snapshot is empty)
  111. m2m_changed = True
  112. event_type = OBJECT_UPDATED
  113. else:
  114. # Other endpoints are unimpacted as they send post_add and post_remove
  115. # This will impact changes that utilize clear() however so we may want to give consideration for this branch
  116. return
  117. else:
  118. return
  119. # Create/update an ObjectChange record for this change
  120. action = {
  121. OBJECT_CREATED: ObjectChangeActionChoices.ACTION_CREATE,
  122. OBJECT_UPDATED: ObjectChangeActionChoices.ACTION_UPDATE,
  123. OBJECT_DELETED: ObjectChangeActionChoices.ACTION_DELETE,
  124. }[event_type]
  125. objectchange = instance.to_objectchange(action)
  126. object_change_id = None
  127. # If this is a many-to-many field change, check for a previous ObjectChange instance recorded
  128. # for this object by this request and update it
  129. if m2m_changed and (
  130. prev_change := ObjectChange.objects.filter(
  131. changed_object_type=ContentType.objects.get_for_model(instance),
  132. changed_object_id=instance.pk,
  133. request_id=request.id
  134. ).first()
  135. ):
  136. prev_change.postchange_data = objectchange.postchange_data
  137. prev_change.save()
  138. object_change_id = prev_change.pk
  139. elif objectchange and objectchange.has_changes:
  140. objectchange.user = request.user
  141. objectchange.request_id = request.id
  142. objectchange.save()
  143. object_change_id = objectchange.pk
  144. # Ensure that we're working with fresh M2M assignments
  145. if m2m_changed:
  146. instance.refresh_from_db()
  147. # Enqueue the object for event processing
  148. queue = events_queue.get()
  149. enqueue_event(queue, instance, request, event_type, object_change_id=object_change_id)
  150. events_queue.set(queue)
  151. # Discard a snapshot read from the database only once the event has captured it
  152. if instance.__dict__.pop('_prechange_snapshot_from_db', False):
  153. instance.__dict__.pop('_prechange_snapshot', None)
  154. # Increment metric counters
  155. if event_type == OBJECT_CREATED:
  156. model_inserts.labels(instance._meta.model_name).inc()
  157. elif event_type == OBJECT_UPDATED:
  158. model_updates.labels(instance._meta.model_name).inc()
  159. @receiver(pre_delete)
  160. def handle_deleted_object(sender, instance, **kwargs):
  161. """
  162. Fires when an object is deleted.
  163. """
  164. # Run any deletion protection rules for the object. Note that this must occur prior
  165. # to queueing any events for the object being deleted, in case a validation error is
  166. # raised, causing the deletion to fail.
  167. model_name = f'{sender._meta.app_label}.{sender._meta.model_name}'
  168. validators = get_config_value_ci(get_config().PROTECTION_RULES, model_name, default=[])
  169. try:
  170. run_validators(instance, validators)
  171. except ValidationError as e:
  172. raise AbortRequest(
  173. _("Deletion is prevented by a protection rule: {message}").format(message=e)
  174. )
  175. # Get the current request, or bail if not set
  176. request = current_request.get()
  177. if request is None:
  178. return
  179. # Check whether we've already processed a pre_delete signal for this object. (This can
  180. # happen e.g. when both a parent object and its child are deleted simultaneously, due
  181. # to cascading deletion.)
  182. if not hasattr(_signals_received, 'pre_delete'):
  183. _signals_received.pre_delete = set()
  184. signature = (ContentType.objects.get_for_model(instance), instance.pk)
  185. if signature in _signals_received.pre_delete:
  186. return
  187. _signals_received.pre_delete.add(signature)
  188. # Record an ObjectChange if applicable
  189. object_change_id = None
  190. if hasattr(instance, 'to_objectchange'):
  191. if hasattr(instance, 'snapshot') and not getattr(instance, '_prechange_snapshot', None):
  192. instance.snapshot()
  193. objectchange = instance.to_objectchange(ObjectChangeActionChoices.ACTION_DELETE)
  194. objectchange.user = request.user
  195. objectchange.request_id = request.id
  196. objectchange.save()
  197. object_change_id = objectchange.pk
  198. # Django does not automatically send an m2m_changed signal for the reverse direction of a
  199. # many-to-many relationship (see https://code.djangoproject.com/ticket/17688), so we need to
  200. # trigger one manually. We do this by checking for any reverse M2M relationships on the
  201. # instance being deleted, and explicitly call .remove() on the remote M2M field to delete
  202. # the association. This triggers an m2m_changed signal with the `post_remove` action type
  203. # for the forward direction of the relationship, ensuring that the change is recorded.
  204. # Similarly, for many-to-one relationships, we set the value on the related object to None
  205. # and save it to trigger a change record on that object.
  206. #
  207. # Skip this for private models (e.g. CablePath) whose lifecycle is an internal
  208. # implementation detail. Django's on_delete handlers (e.g. SET_NULL) already take
  209. # care of the database integrity; recording changelog entries for the related
  210. # objects would be spurious. (Ref: #21390)
  211. if not getattr(instance, '_netbox_private', False):
  212. for relation in instance._meta.related_objects:
  213. if type(relation) not in [ManyToManyRel, ManyToOneRel]:
  214. continue
  215. related_model = relation.related_model
  216. related_field_name = relation.remote_field.name
  217. if not issubclass(related_model, ChangeLoggingMixin):
  218. # We only care about triggering the m2m_changed signal for models which support
  219. # change logging
  220. continue
  221. related_object_type = ContentType.objects.get_for_model(related_model)
  222. for obj in related_model.objects.filter(**{related_field_name: instance.pk}):
  223. # Skip any related object that is itself being deleted as part of this same
  224. # operation (e.g. a sibling caught up in the same cascade). Its deletion has
  225. # already been recorded, so nulling the FK and saving here would record an
  226. # UPDATE ObjectChange *after* the object's DELETE, corrupting the changelog and
  227. # breaking branch replay. (Ref: #22270)
  228. #
  229. # Note this is order-dependent: it only fires once the related object's own
  230. # pre_delete has run (adding it to the set). If the cascade happens to delete
  231. # this instance *before* the related object, the guard won't trigger and the
  232. # related object still gets an UPDATE — but in the harmless UPDATE-then-DELETE
  233. # order, not the corrupting DELETE-then-UPDATE order. Fully suppressing it in
  234. # every ordering would require the complete deletion set, which isn't available
  235. # from a pre_delete signal.
  236. if (related_object_type, obj.pk) in _signals_received.pre_delete:
  237. continue
  238. obj.snapshot() # Ensure the change record includes the "before" state
  239. if type(relation) is ManyToManyRel:
  240. getattr(obj, related_field_name).remove(instance)
  241. elif type(relation) is ManyToOneRel and relation.null and relation.on_delete not in (CASCADE, RESTRICT):
  242. setattr(obj, related_field_name, None)
  243. obj.save()
  244. # Enqueue the object for event processing
  245. queue = events_queue.get()
  246. enqueue_event(queue, instance, request, OBJECT_DELETED, object_change_id=object_change_id)
  247. events_queue.set(queue)
  248. # Increment metric counters
  249. model_deletes.labels(instance._meta.model_name).inc()
  250. @receiver(request_finished)
  251. def clear_signal_history(sender, **kwargs):
  252. """
  253. Clear out the signals history once the request is finished.
  254. """
  255. _signals_received.pre_delete = set()
  256. @receiver(clear_events)
  257. def clear_events_queue(sender, **kwargs):
  258. """
  259. Delete any queued events (e.g. because of an aborted bulk transaction)
  260. """
  261. logger = logging.getLogger('events')
  262. logger.info(f"Clearing {len(events_queue.get())} queued events ({sender})")
  263. events_queue.set({})
  264. #
  265. # DataSource handlers
  266. #
  267. @receiver(post_save, sender=DataSource)
  268. def enqueue_sync_job(instance, created, **kwargs):
  269. """
  270. When a DataSource is saved, check its sync_interval and enqueue a sync job if appropriate.
  271. """
  272. from .jobs import SyncDataSourceJob
  273. if instance.enabled and instance.sync_interval:
  274. SyncDataSourceJob.enqueue_once(instance, interval=instance.sync_interval)
  275. elif not created:
  276. # Delete any previously scheduled recurring jobs for this DataSource
  277. for job in SyncDataSourceJob.get_jobs(instance).defer('data').filter(
  278. interval__isnull=False,
  279. status=JobStatusChoices.STATUS_SCHEDULED
  280. ):
  281. # Call delete() per instance to ensure the associated background task is deleted as well
  282. job.delete()
  283. # Keeps the aggregated error readable when a whole source fails at once
  284. _AUTO_SYNC_DETAIL_LIMIT = 10
  285. @receiver(post_sync)
  286. def auto_sync(instance, **kwargs):
  287. """
  288. Automatically synchronize any DataFiles with AutoSyncRecords after synchronizing a DataSource.
  289. """
  290. from .models import AutoSyncRecord
  291. failure_count = 0
  292. details = []
  293. first_error = None
  294. records = AutoSyncRecord.objects.filter(datafile__source=instance).order_by('pk')
  295. for autosync in records.select_related('object_type').prefetch_related('object'):
  296. # The object may be unresolvable or mid-failure, so identify by keys
  297. target = f'{autosync.object_type.app_label}.{autosync.object_type.model} ID {autosync.object_id}'
  298. if autosync.object_type.model_class() is None:
  299. # Not an orphaned row, so leave it for remove_stale_contenttypes
  300. logger.warning(f"Skipping AutoSyncRecord for uninstalled model {target}")
  301. continue
  302. try:
  303. # The try must stay outside this, so the savepoint is rolled back before the handler runs
  304. with transaction.atomic():
  305. obj = autosync.object
  306. if obj is None:
  307. # The prefetch resolves through the default manager, so recheck with the base manager
  308. try:
  309. obj = autosync.object_type.get_object_for_this_type(pk=autosync.object_id)
  310. except ObjectDoesNotExist:
  311. logger.warning(f"Deleting stale AutoSyncRecord for {target}")
  312. autosync.delete()
  313. continue
  314. obj.sync(save=True)
  315. except JobTimeoutException:
  316. # rq arms one alarm per job, so a timeout is not an ordinary per-object failure
  317. raise
  318. except Exception as e:
  319. failure_count += 1
  320. if first_error is None:
  321. first_error = e
  322. # Not capped, unlike the raised message below
  323. logger.error(f"Error auto-syncing {target}: {e}", exc_info=True)
  324. if len(details) < _AUTO_SYNC_DETAIL_LIMIT:
  325. details.append(f'- {target}: {type(e).__name__}: {e}')
  326. if first_error is not None:
  327. summary = ngettext(
  328. 'Automatic synchronization failed for {count} object:',
  329. 'Automatic synchronization failed for {count} objects:',
  330. failure_count,
  331. ).format(count=failure_count)
  332. if omitted := failure_count - len(details):
  333. details.append(ngettext(
  334. '{count} additional failure is not shown.',
  335. '{count} additional failures are not shown.',
  336. omitted,
  337. ).format(count=omitted))
  338. raise SyncError('\n'.join([summary, *details])) from first_error
  339. @receiver(post_save, sender=ConfigRevision)
  340. def update_config(sender, instance, **kwargs):
  341. """
  342. Update the cached NetBox configuration when a new ConfigRevision is created.
  343. """
  344. instance.activate()