Просмотр исходного кода

Quiet routine background job lines in the server log (#627)

* Quiet routine background job lines in the server log

One server logger (src/lib/logger.ts) with LOG_LEVEL error, warn, info
(default) or debug. Every line is [scope] message key=value on one line.
Routine per-run successes of backups and other recurring jobs move to
debug and arrive at info as an hourly roll-up; failures print at once at
error with the workshop id.

* Share the backup roll-up across module copies and keep failures at error

The backup roll-up lives on globalThis so manual and scheduled runs count
into one total. A failed licence email, seed stderr and a subscription sync
with errors now log at error; report schedules roll up hourly; LOG_LEVEL
ignores inherited property names.
Bernt Christian Egeland 1 день назад
Родитель
Сommit
92188321d1

+ 6 - 0
.env.example

@@ -23,6 +23,12 @@ NEXT_PUBLIC_APP_URL="http://localhost:3000"
 # the container instead, where a redeploy loses it.
 # DATA_ROOT="/var/lib/torqvoice"
 
+# How much the server says in its log (`docker logs`): error, warn, info or
+# debug; info unless set. At info, recurring jobs (backups, integration jobs,
+# scheduled messages) print an hourly total instead of a line per run, and
+# every failure still prints at once. Set debug to see each run.
+# LOG_LEVEL=info
+
 # Scheduled backups (Settings → Integrations). Both are optional.
 # The Docker image ships restic at /usr/local/bin/restic. An install that runs
 # outside Docker points this at its own restic binary; unset, `restic` is

+ 2 - 0
docker-compose.yml

@@ -14,6 +14,8 @@ services:
       # Optional, see .env.example: the largest backup data file a restore reads
       # into memory, in MiB (128 unless set).
       - BACKUP_MAX_DATA_FILE_MB=${BACKUP_MAX_DATA_FILE_MB:-}
+      # Optional, see .env.example: error, warn, info (the default) or debug.
+      - LOG_LEVEL=${LOG_LEVEL:-}
     volumes:
       - uploads:/app/data/uploads
       - backups:/app/data/backups

+ 22 - 0
src/__tests__/cron/tick-summary.test.ts

@@ -39,4 +39,26 @@ describe('createTickSummary', () => {
     summary.add(1)
     expect(console.warn).toHaveBeenCalledTimes(1)
   })
+
+  it('sends the total when the hour is up even if the last ticks were idle', () => {
+    const summary = createTickSummary('Scheduled messages sent')
+    summary.add(4)
+    vi.advanceTimersByTime(60 * 60 * 1000)
+    summary.add(0)
+    expect(console.warn).toHaveBeenCalledWith(
+      '[cron] Scheduled messages sent: 4 in 1 run(s) over the last 60 min'
+    )
+  })
+
+  it('says each run at LOG_LEVEL=debug', () => {
+    process.env.LOG_LEVEL = 'debug'
+    try {
+      createTickSummary('Webhook deliveries processed').add(3)
+      expect(console.warn).toHaveBeenCalledWith(
+        '[cron] Webhook deliveries processed this run count=3'
+      )
+    } finally {
+      delete process.env.LOG_LEVEL
+    }
+  })
 })

+ 124 - 4
src/__tests__/lib/backup/engine.test.ts

@@ -205,6 +205,7 @@ import {
   openPlatform,
   openTarget,
   queueBackupRun,
+  reportBackupRollup,
   runBackup,
   stagingDirFor,
   takeSafetySnapshot,
@@ -958,8 +959,9 @@ describe('runBackup, failure', () => {
       expect(view.status).toBe('failed')
       expect(view.error).toMatch(/incomplete/)
       expect(errors).toHaveBeenCalledWith(
-        expect.stringContaining('Backup is incomplete: some files could not be read'),
-        expect.objectContaining({ snapshot: SNAPSHOT_ID })
+        expect.stringContaining(
+          `Backup is incomplete: some files could not be read organizationId=${ORG} snapshot=${SNAPSHOT_ID}`
+        )
       )
       expect(callsOf('forget')).toHaveLength(0)
       expect(callsOf('snapshots')).toHaveLength(0)
@@ -1384,8 +1386,7 @@ describe('takeSafetySnapshot', () => {
       expect(safety).toEqual({ snapshotId: SNAPSHOT_ID, kind: 'platform' })
       expect(callsOf('snapshots')).toHaveLength(1)
       expect(errors).toHaveBeenCalledWith(
-        expect.stringContaining('Could not list the restore points'),
-        expect.objectContaining({ error: expect.any(String) })
+        expect.stringMatching(/Could not list the restore points .*error=/)
       )
       expect((h.platform.get(ORG) as Row).snapshots).toBeUndefined()
     } finally {
@@ -1433,3 +1434,122 @@ describe.skipIf(!realBin)('runBackup with the real restic binary', () => {
     expect(state['backup.repositorySize']).toBeGreaterThan(200_000)
   }, 120_000)
 })
+
+/**
+ * What `docker logs` shows for backups. The roll-up's clock is Date.now, so
+ * each test moves it well past anything an earlier test counted, then an hour
+ * on for the line itself.
+ */
+describe('the server log', () => {
+  const HOUR = 60 * 60 * 1000
+  let clockBase = Date.now() + 1000 * HOUR
+
+  function lines(spy: { mock: { calls: unknown[][] } }): string[] {
+    return spy.mock.calls.map((c) => c.map(String).join(' '))
+  }
+
+  /** Flushes whatever earlier tests counted, so this test starts from zero. */
+  function freshRollup(): void {
+    clockBase += 10 * HOUR
+    const now = vi.spyOn(Date, 'now').mockReturnValue(clockBase)
+    reportBackupRollup()
+    now.mockRestore()
+  }
+
+  /** The cron tick an hour and a minute after the start. */
+  function tickAnHourLater(): void {
+    const now = vi.spyOn(Date, 'now').mockReturnValue(clockBase + HOUR + 60_000)
+    reportBackupRollup()
+    now.mockRestore()
+  }
+
+  it('says nothing per successful run at the default level, then one hourly total', async () => {
+    setPlatformEnv()
+    delete process.env.LOG_LEVEL
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    try {
+      freshRollup()
+      warn.mockClear()
+      for (let i = 0; i < 2; i++) {
+        const view = await runBackup(
+          addRun({ kind: 'platform', connectionId: null, trigger: 'scheduled' })
+        )
+        expect(view.status, view.error ?? '').toBe('done')
+      }
+      expect(lines(warn)).toEqual([])
+      expect(lines(error)).toEqual([])
+
+      tickAnHourLater()
+      expect(lines(warn)).toEqual(['[backup] Backups: 2 finished, 0 failed over the last 61 min'])
+      // Nothing more until the next hour has something to say.
+      tickAnHourLater()
+      expect(warn).toHaveBeenCalledTimes(1)
+    } finally {
+      warn.mockRestore()
+      error.mockRestore()
+    }
+  })
+
+  it('prints nothing at all for an hour with no runs', () => {
+    delete process.env.LOG_LEVEL
+    freshRollup()
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      tickAnHourLater()
+      expect(warn).not.toHaveBeenCalled()
+    } finally {
+      warn.mockRestore()
+    }
+  })
+
+  it('prints a failure at once on one line with the workshop, and counts it in the total', async () => {
+    setPlatformEnv()
+    delete process.env.LOG_LEVEL
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    const error = vi.spyOn(console, 'error').mockImplementation(() => undefined)
+    try {
+      freshRollup()
+      warn.mockClear()
+      await runBackup(addRun({ kind: 'platform', connectionId: null, trigger: 'scheduled' }))
+      failRestic('backup')
+      const runId = addRun({ kind: 'platform', connectionId: null, trigger: 'scheduled' })
+      const view = await runBackup(runId)
+      expect(view.status).toBe('failed')
+
+      const failed = lines(error).filter((l) => l.startsWith('[backup] Backup run failed'))
+      expect(failed).toHaveLength(1)
+      expect(failed[0]).toContain(`runId=${runId} organizationId=${ORG} kind=platform`)
+      expect(failed[0]).toContain('trigger=scheduled error="')
+      for (const line of [...lines(warn), ...lines(error)]) expect(line).not.toMatch(/[\r\n]/)
+
+      tickAnHourLater()
+      expect(lines(warn)).toContain('[backup] Backups: 1 finished, 1 failed over the last 61 min')
+    } finally {
+      warn.mockRestore()
+      error.mockRestore()
+    }
+  })
+
+  it('shows every run on one line each at LOG_LEVEL=debug', async () => {
+    setPlatformEnv()
+    process.env.LOG_LEVEL = 'debug'
+    const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+    try {
+      freshRollup()
+      warn.mockClear()
+      await runBackup(addRun({ kind: 'platform', connectionId: null, trigger: 'scheduled' }))
+      const finished = lines(warn).filter((l) => l.includes('Backup finished'))
+      expect(finished).toEqual([
+        expect.stringMatching(
+          new RegExp(
+            `^\\[platform-backup\\] Backup finished organizationId=${ORG} snapshot=${SNAPSHOT_ID} filesNew=\\d+ dataAdded=\\d+$`
+          )
+        ),
+      ])
+      for (const line of lines(warn)) expect(line).not.toMatch(/[\r\n]/)
+    } finally {
+      warn.mockRestore()
+    }
+  })
+})

+ 1 - 1
src/__tests__/lib/backup/platform.test.ts

@@ -84,7 +84,7 @@ describe('platformBackupConfig', () => {
     expect(platformBackupConfig()).toBeNull()
     expect(platformBackupConfig()).toBeNull()
     expect(error).toHaveBeenCalledTimes(1)
-    expect(String(error.mock.calls[0][0])).toMatch(/platform backups are off/)
+    expect(String(error.mock.calls[0][0])).toMatch(/Platform backups are off/)
     expect(String(error.mock.calls[0][0])).toMatch(/ACCESS_KEY/)
     expect(String(error.mock.calls[0][0])).toMatch(/MASTER_KEY/)
     // Nothing secret in the log line.

+ 3 - 2
src/__tests__/lib/cron/appointment-notifications.test.ts

@@ -261,8 +261,9 @@ describe('a notification to the customer', () => {
 
     expect(sent).toBe(1)
     expect(console.error).toHaveBeenCalledWith(
-      expect.stringContaining('customer_sms for n-sms failed'),
-      'SMS provider down'
+      expect.stringMatching(
+        /^\[appointment-notifications\] Notification failed .*notificationId=n-sms channel=customer_sms error="SMS provider down"$/
+      )
     )
     expect(notify).toHaveBeenCalledTimes(1)
   })

+ 3 - 1
src/__tests__/lib/cron/backup-runs.test.ts

@@ -426,7 +426,9 @@ describe('processDueBackups, what is queued', () => {
     expect(await processDueBackups(now)).toBe(1)
 
     expect(queuedConnections()).toEqual(['c1', 'c2'])
-    expect(error).toHaveBeenCalledWith(expect.stringContaining('connection c1'), expect.any(Error))
+    expect(error).toHaveBeenCalledWith(
+      expect.stringMatching(/connectionId=c1 err="database went away"$/)
+    )
     error.mockRestore()
   })
 })

+ 121 - 0
src/__tests__/lib/logger.test.ts

@@ -0,0 +1,121 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+import { createLogger, createRollup, formatLine } from '@/lib/logger'
+
+const saved = process.env.LOG_LEVEL
+
+function printed(): string[] {
+  return [...vi.mocked(console.warn).mock.calls, ...vi.mocked(console.error).mock.calls].map((c) =>
+    c.map(String).join(' ')
+  )
+}
+
+beforeEach(() => {
+  delete process.env.LOG_LEVEL
+  vi.spyOn(console, 'warn').mockImplementation(() => undefined)
+  vi.spyOn(console, 'error').mockImplementation(() => undefined)
+})
+afterEach(() => {
+  vi.useRealTimers()
+  vi.restoreAllMocks()
+  if (saved === undefined) delete process.env.LOG_LEVEL
+  else process.env.LOG_LEVEL = saved
+})
+
+describe('log lines', () => {
+  it('reads [scope] message key=value, objects flattened, on one line', () => {
+    expect(
+      formatLine('platform-backup', 'Backup finished', {
+        organizationId: 'org1',
+        snapshot: '8984b7ced0f1',
+        stats: { filesNew: 0, dataAdded: 613991 },
+        at: new Date('2026-11-17T10:00:00Z'),
+        skipped: undefined,
+      })
+    ).toBe(
+      '[platform-backup] Backup finished organizationId=org1 snapshot=8984b7ced0f1 stats.filesNew=0 stats.dataAdded=613991 at=2026-11-17T10:00:00.000Z'
+    )
+  })
+
+  it('quotes values with spaces and never lets a line break through', () => {
+    const line = formatLine('backup', 'Run\nfailed', {
+      error: 'Fatal: unable to open repository\nat s3:bucket',
+      empty: '',
+      list: ['a', 'b'],
+    })
+    expect(line).toBe(
+      '[backup] Run failed error="Fatal: unable to open repository\\nat s3:bucket" empty="" list="[\\"a\\",\\"b\\"]"'
+    )
+    expect(line).not.toMatch(/[\r\n]/)
+  })
+
+  it('prints an error as its message and code; the stack only at debug, still on one line', () => {
+    const err = Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })
+    createLogger('cron').error('Backup scheduler failed', { err })
+    expect(printed()).toEqual([
+      '[cron] Backup scheduler failed err="connect ECONNREFUSED" errCode=ECONNREFUSED',
+    ])
+
+    process.env.LOG_LEVEL = 'debug'
+    createLogger('cron').error('Backup scheduler failed', { err })
+    const withStack = printed()[1]
+    expect(withStack).toContain('errStack="Error: connect ECONNREFUSED\\n')
+    expect(withStack).not.toMatch(/[\r\n]/)
+  })
+})
+
+describe('LOG_LEVEL', () => {
+  function sayEverything() {
+    const log = createLogger('cron')
+    log.error('e')
+    log.warn('w')
+    log.info('i')
+    log.debug('d')
+  }
+
+  it('is info by default: debug lines stay silent', () => {
+    sayEverything()
+    expect(printed()).toEqual(['[cron] w', '[cron] i', '[cron] e'])
+  })
+
+  it('debug brings the routine lines back', () => {
+    process.env.LOG_LEVEL = 'DEBUG'
+    sayEverything()
+    expect(printed()).toContain('[cron] d')
+  })
+
+  it('error keeps only failures, and an unknown value means info', () => {
+    process.env.LOG_LEVEL = 'error'
+    sayEverything()
+    expect(printed()).toEqual(['[cron] e'])
+    vi.mocked(console.error).mockClear()
+    for (const unknown of ['verbose', 'constructor', '__proto__']) {
+      vi.mocked(console.warn).mockClear()
+      vi.mocked(console.error).mockClear()
+      process.env.LOG_LEVEL = unknown
+      sayEverything()
+      expect(printed()).toEqual(['[cron] w', '[cron] i', '[cron] e'])
+    }
+  })
+})
+
+describe('createRollup', () => {
+  beforeEach(() => {
+    vi.useFakeTimers()
+    vi.setSystemTime(new Date('2026-11-17T10:00:00Z'))
+  })
+
+  it('prints one line an hour with every count, zeros included, and is silent on an idle hour', () => {
+    const rollup = createRollup('cron', 'Backups', { order: ['finished', 'failed'] })
+    for (let i = 0; i < 12; i++) {
+      rollup.add({ finished: 1 })
+      vi.advanceTimersByTime(5 * 60_000)
+    }
+    expect(printed()).toEqual([])
+    rollup.add({})
+    expect(printed()).toEqual(['[cron] Backups: 12 finished, 0 failed over the last 60 min'])
+
+    vi.advanceTimersByTime(60 * 60_000)
+    rollup.add({})
+    expect(printed()).toHaveLength(1)
+  })
+})

+ 2 - 1
src/instrumentation.ts

@@ -15,7 +15,8 @@ export async function register() {
       warnAboutAppUrl()
       const { markBackgroundJobsDisabled } = await import('./lib/background-jobs')
       markBackgroundJobsDisabled()
-      console.warn('[cron] DISABLE_BACKGROUND_JOBS=1, no scheduled work started')
+      const { createLogger } = await import('./lib/logger')
+      createLogger('cron').info('DISABLE_BACKGROUND_JOBS=1, no scheduled work started')
       return
     }
     const {

+ 81 - 17
src/lib/backup/engine.ts

@@ -23,6 +23,7 @@ import { getManifest } from '@/integrations/registry'
 import { db } from '@/lib/db'
 import { getFromAddress, getOrgFromAddress, sendMail, sendOrgMail } from '@/lib/email'
 import { getFeatures } from '@/lib/features'
+import { createLogger, createRollup } from '@/lib/logger'
 import { notify } from '@/lib/notify'
 import { orgUploadDir } from '@/lib/upload-root'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
@@ -76,6 +77,29 @@ import { connectMessage, ensureCacheDir, repositoryPassword, sanitizeRepository
 const PRUNE_EVERY_MS = 7 * 24 * 60 * 60 * 1000
 const CHECK_EVERY_MS = 30 * 24 * 60 * 60 * 1000
 const FAILURE_MAIL_EVERY_MS = 24 * 60 * 60 * 1000
+
+const log = createLogger('backup')
+const platformLog = createLogger('platform-backup')
+/**
+ * Finished and failed backup runs, said once an hour at `info`. Every run,
+ * whoever started it, counts here; the backup cron calls `reportBackupRollup`
+ * on each tick so the hour's line goes out on time. A failure is also logged
+ * at `error` the moment it happens.
+ */
+function newRollup() {
+  return createRollup('backup', 'Backups', { order: ['finished', 'failed'] })
+}
+// On globalThis, like the backup lock: a server action and the cron can load
+// separate copies of this module, and both must count into the same total.
+const rollupHolder = globalThis as unknown as {
+  __torqvoiceBackupRollup?: ReturnType<typeof newRollup>
+}
+rollupHolder.__torqvoiceBackupRollup ??= newRollup()
+const rollup = rollupHolder.__torqvoiceBackupRollup
+
+export function reportBackupRollup(): void {
+  rollup.add({})
+}
 /** One retry after a failed scheduled run, before the next slot. */
 export const RETRY_AFTER_MS = 30 * 60 * 1000
 
@@ -140,7 +164,17 @@ export interface OpenedTarget {
   retention: RetentionPolicy
   /** Last prune and check times, and the repository size, as the target keeps them. */
   state: { lastPruneAt: Date | null; lastCheckAt: Date | null }
-  log(level: 'info' | 'error', message: string, details?: Record<string, unknown>): Promise<void>
+  /**
+   * A line in the destination's log, or the server log for the platform. A
+   * `routine` line (a scheduled backup that went fine) is printed by the
+   * platform only at LOG_LEVEL=debug; the hourly roll-up counts it.
+   */
+  log(
+    level: 'info' | 'error',
+    message: string,
+    details?: Record<string, unknown>,
+    options?: { routine?: boolean }
+  ): Promise<void>
   /** Record a good run: when, how big, and (platform) the restore points now in the repository. */
   recordGood(data: {
     at: Date
@@ -233,10 +267,11 @@ export async function openPlatform(organizationId: string): Promise<OpenedTarget
     includeSecrets: true,
     retention: PLATFORM_RETENTION,
     state: { lastPruneAt: row.lastPruneAt, lastCheckAt: row.lastCheckAt },
-    async log(level, message, details) {
-      const line = `[platform-backup] ${organizationId}: ${message}`
-      if (level === 'error') console.error(line, details ?? '')
-      else console.warn(line, details ?? '')
+    async log(level, message, details, options) {
+      const fields = { organizationId, ...details }
+      if (level === 'error') platformLog.error(message, fields)
+      else if (options?.routine) platformLog.debug(message, fields)
+      else platformLog.info(message, fields)
     },
     async recordGood({ at, totalBytes, snapshots, prunedAt, checkedAt }) {
       await db.platformBackup.update({
@@ -475,7 +510,7 @@ export async function takeSnapshot(
     return await resticBackup(target.base, { paths, host: target.host, tags: target.tags(trigger) })
   } finally {
     await rm(dataFile, { force: true }).catch((err) =>
-      console.warn('[backup] could not remove the staging file:', err)
+      log.warn('Could not remove the staging file', { organizationId: target.organizationId, err })
     )
   }
 }
@@ -577,11 +612,16 @@ export async function runBackup(runId: string): Promise<BackupRunView> {
         `Some files could not be read, so this backup is incomplete (restic snapshot ${result.snapshotId.slice(0, 8)}). Check the server log and run the backup again.`
       )
     }
-    await target.log('info', 'Backup finished', {
-      snapshot: result.snapshotId,
-      filesNew: result.filesNew,
-      dataAdded: result.dataAdded,
-    })
+    await target.log(
+      'info',
+      'Backup finished',
+      {
+        snapshot: result.snapshotId,
+        filesNew: result.filesNew,
+        dataAdded: result.dataAdded,
+      },
+      { routine: true }
+    )
 
     // Retention after every run; the expensive prune once a week, the
     // read-back check once a month. Safety copies are kept whatever the
@@ -645,29 +685,53 @@ export async function runBackup(runId: string): Promise<BackupRunView> {
       prunedAt: prune ? now : null,
       checkedAt,
     })
-    return await finishRun(runId, {
+    const view = await finishRun(runId, {
       status: 'done',
       snapshotId: result.snapshotId,
       filesNew: result.filesNew,
       bytesAdded: BigInt(Math.round(result.dataAdded)),
       ...(totalBytes !== null ? { totalBytes: BigInt(Math.round(totalBytes)) } : {}),
     })
+    // The platform said so through target.log; a destination's line went to
+    // its own log on the card, so the server log hears it here, at debug.
+    if (target.kind !== 'platform') {
+      log.debug('Backup run finished', {
+        runId,
+        organizationId,
+        kind: run.kind,
+        connectionId: run.connectionId ?? undefined,
+        trigger: run.trigger,
+        snapshot: result.snapshotId,
+        filesNew: result.filesNew,
+        dataAdded: result.dataAdded,
+      })
+    }
+    rollup.add({ finished: 1 })
+    return view
   } catch (err) {
     const message = describeFailure(err)
-    console.error(`[backup] run ${runId} failed:`, message)
+    log.error('Backup run failed', {
+      runId,
+      organizationId,
+      kind: run.kind,
+      connectionId: run.connectionId ?? undefined,
+      trigger: run.trigger,
+      error: message,
+    })
+    rollup.add({ failed: 1 })
     const view = await finishRun(runId, { status: 'failed', error: message })
     if (run.trigger === 'scheduled') {
       await bookRetry(run, target, new Date(Date.now() + RETRY_AFTER_MS)).catch((e) =>
-        console.error('[backup] could not schedule the retry:', e)
+        log.error('Could not schedule the retry', { runId, organizationId, err: e })
       )
     }
     await reportFailure(run.kind as BackupKind, run.connectionId, organizationId, message).catch(
-      (e) => console.error('[backup] could not report the failure:', e)
+      (e) => log.error('Could not report the failure', { runId, organizationId, err: e })
     )
     return view
   } finally {
     releaseBackupLock(runId)
-    void drainBackupQueue().catch((err) => console.error('[backup] drain failed:', err))
+    void drainBackupQueue().catch((err) => log.error('Queue drain failed', { err }))
   }
 }
 
@@ -896,7 +960,7 @@ export async function mailAdmins(
       try {
         await sendMail({ to, from: await getFromAddress(), subject, html })
       } catch (fallbackErr) {
-        console.error('[backup] alert email failed:', err, fallbackErr)
+        log.error('Alert email failed', { organizationId, err, fallbackErr })
       }
     }
   }

+ 2 - 1
src/lib/backup/platform.ts

@@ -3,6 +3,7 @@ import { backupHost, backupTag } from '@/features/integrations/Lib/backup'
 import type { ResticRepository } from '@/features/integrations/Lib/backup-types'
 import type { PlanFeatures } from '@/lib/features'
 import { isCloudMode } from '@/lib/features'
+import { createLogger } from '@/lib/logger'
 
 /**
  * The backups Torqvoice runs for a workshop on the cloud instance.
@@ -77,7 +78,7 @@ export function platformBackupConfig(): PlatformBackupConfig | null {
   if (problems.length > 0) {
     if (!warned) {
       warned = true
-      console.error(`[backup] platform backups are off: ${problems.join('; ')}`)
+      createLogger('backup').error('Platform backups are off', { problems: problems.join('; ') })
     }
     return null
   }

+ 8 - 5
src/lib/backup/restore.ts

@@ -7,6 +7,7 @@ import { organizationTranslator } from '@/i18n/organization-translator'
 import { getManifest } from '@/integrations/registry'
 import { logAudit } from '@/lib/audit'
 import { db } from '@/lib/db'
+import { createLogger } from '@/lib/logger'
 import { notify } from '@/lib/notify'
 import { dataRoot } from '@/lib/upload-root'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
@@ -31,6 +32,8 @@ import { RESTORE_DISK_HEADROOM_BYTES, maxDataFileBytes } from './limits'
 import { ResticError, resticRestore, resticRestoreSize, resticSnapshots } from './restic'
 import { connectMessage } from './target'
 
+const log = createLogger('restore')
+
 /**
  * Putting a snapshot back into the workshop, from a destination it connected
  * or from the platform repository Torqvoice keeps for it.
@@ -321,13 +324,13 @@ export async function runRestore(runId: string): Promise<BackupRunView> {
           reason: run.reason,
         },
       }
-    ).catch((err) => console.error('[restore] could not write the audit row:', err))
+    ).catch((err) => log.error('Could not write the audit row', { runId, organizationId, err }))
     await tellWorkshop(organizationId, {
       name,
       snapshotId,
       bySupport: run.requestedBy === null,
       reason: run.reason,
-    }).catch((err) => console.error('[restore] could not tell the workshop:', err))
+    }).catch((err) => log.error('Could not tell the workshop', { runId, organizationId, err }))
 
     return await finishRun(runId, {
       status: 'done',
@@ -338,7 +341,7 @@ export async function runRestore(runId: string): Promise<BackupRunView> {
     })
   } catch (err) {
     const message = err instanceof ResticError ? connectMessage(err) : describe(err)
-    console.error(`[restore] run ${runId} failed (${phase}):`, message)
+    log.error('Restore run failed', { runId, organizationId, phase, error: message })
     const error =
       phase === 'safety'
         ? `Could not take a safety copy of the workshop first; nothing was changed. ${message}`
@@ -353,11 +356,11 @@ export async function runRestore(runId: string): Promise<BackupRunView> {
     if (frozen) unfreezeOrganization(organizationId)
     if (folder) {
       await rm(folder, { recursive: true, force: true }).catch((err) =>
-        console.warn('[restore] could not remove the restore folder:', err)
+        log.warn('Could not remove the restore folder', { runId, organizationId, err })
       )
     }
     releaseBackupLock(runId)
-    void drainBackupQueue().catch((err) => console.error('[restore] drain failed:', err))
+    void drainBackupQueue().catch((err) => log.error('Queue drain failed', { err }))
   }
 }
 

+ 13 - 9
src/lib/cron/appointment-notifications.ts

@@ -14,8 +14,10 @@ import { isNotificationDue } from '@/features/calendar/Lib/notificationTiming'
 import { MAX_NOTICE_MINUTES } from '@/features/calendar/Schema/bookingSchema'
 import { SETTING_KEYS } from '@/features/settings/Schema/settingsSchema'
 import { DEFAULT_TIME_FORMAT } from '@/lib/format'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
 
-const LOG_PREFIX = '[appointment-notifications]'
+const log = createLogger('appointment-notifications')
 
 type Translate = (key: string, values?: Record<string, string | number>) => string
 
@@ -246,10 +248,12 @@ export async function processDueAppointmentNotifications(now = new Date()): Prom
       }
       sent++
     } catch (error) {
-      console.error(
-        `${LOG_PREFIX} ${row.channel} for ${row.id} failed:`,
-        error instanceof Error ? error.message : error
-      )
+      log.error('Notification failed', {
+        organizationId: row.organizationId,
+        notificationId: row.id,
+        channel: row.channel,
+        error: error instanceof Error ? error.message : error,
+      })
     }
   }
   return sent
@@ -257,14 +261,14 @@ export async function processDueAppointmentNotifications(now = new Date()): Prom
 
 /** Minute by minute, so a notification goes out on the minute it was set for. */
 export function processAppointmentNotifications() {
+  const summary = createTickSummary('Appointment notifications sent')
   const job = new CronJob('* * * * *', async () => {
     try {
-      const sent = await processDueAppointmentNotifications()
-      if (sent > 0) console.warn(`${LOG_PREFIX} sent ${sent} notification(s)`)
+      summary.add(await processDueAppointmentNotifications())
     } catch (error) {
-      console.error(`${LOG_PREFIX} scan failed:`, error)
+      log.error('Scan failed', { err: error })
     }
   })
   job.start()
-  console.warn(`${LOG_PREFIX} Appointment notifications started (every minute)`)
+  log.info('Appointment notifications started (every minute)')
 }

+ 39 - 27
src/lib/cron/backup-runs.ts

@@ -15,6 +15,7 @@ import {
   drainBackupQueue,
   openPlatform,
   queueBackupRun,
+  reportBackupRollup,
   reportOverdue,
 } from '@/lib/backup/engine'
 import {
@@ -30,6 +31,7 @@ import { BACKUP_SETTING_DEFAULTS, intervalMs, isDue } from '@/lib/backup/schedul
 import { db } from '@/lib/db'
 import { sendMail } from '@/lib/email'
 import { getFeatures } from '@/lib/features'
+import { createLogger } from '@/lib/logger'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
 
 /**
@@ -57,6 +59,8 @@ const RUN_HISTORY_MS = 90 * 24 * 60 * 60 * 1000
 const KEEP_RUNS = 100
 const PRUNE_EVERY_MS = 24 * 60 * 60 * 1000
 
+const log = createLogger('cron')
+
 const PLATFORM_FREQUENCIES: BackupFrequency[] = ['hourly', 'every4h', 'daily']
 
 function stateDate(state: Record<string, unknown>, key: string): Date | null {
@@ -79,21 +83,24 @@ export async function failStaleRuns(now = new Date()): Promise<number> {
         { status: 'queued', createdAt: { lt: new Date(now.getTime() - STALE_QUEUE_MS) } },
       ],
     },
-    select: { id: true, status: true },
+    select: { id: true, status: true, organizationId: true, kind: true },
   })
   let count = 0
   for (const run of stale) {
     if (holder?.runId === run.id) continue
+    const error =
+      run.status === 'queued'
+        ? 'The run was never started; the server was probably restarted'
+        : 'The run did not finish; the server was probably restarted'
     await db.backupRun.update({
       where: { id: run.id },
-      data: {
-        status: 'failed',
-        finishedAt: now,
-        error:
-          run.status === 'queued'
-            ? 'The run was never started; the server was probably restarted'
-            : 'The run did not finish; the server was probably restarted',
-      },
+      data: { status: 'failed', finishedAt: now, error },
+    })
+    log.error('Stale backup run marked as failed', {
+      runId: run.id,
+      organizationId: run.organizationId,
+      kind: run.kind,
+      error,
     })
     count++
   }
@@ -198,7 +205,11 @@ export async function processDueBackups(now = new Date()): Promise<number> {
       }
       await alertIfOverdue(c.id, c.organizationId, settings, state, now)
     } catch (err) {
-      console.error(`[cron] backup for connection ${c.id} failed:`, err)
+      log.error('Scheduling a backup failed', {
+        organizationId: c.organizationId,
+        connectionId: c.id,
+        err,
+      })
     }
   }
   return queued
@@ -304,7 +315,7 @@ export async function processDuePlatformBackups(now = new Date()): Promise<numbe
       }
       if (row) await alertIfPlatformOverdue(org.id, row, now)
     } catch (err) {
-      console.error(`[cron] platform backup for ${org.id} failed:`, err)
+      log.error('Scheduling a platform backup failed', { organizationId: org.id, err })
     }
   }
   return queued
@@ -330,7 +341,7 @@ async function purgeRepository(organizationId: string): Promise<void> {
     where: { organizationId },
     data: { purgedAt: new Date(), snapshots: [], repositorySize: BigInt(0) },
   })
-  console.warn(`[cron] platform repository of ${organizationId} purged after the grace period`)
+  log.info('Platform repository purged after the grace period', { organizationId })
 }
 
 /**
@@ -359,9 +370,10 @@ async function alertIfPlatformOverdue(
     return
   }
   await db.platformBackup.update({ where: { organizationId }, data: { lastOverdueAlertAt: now } })
-  console.error(
-    `[cron] platform backup of ${organizationId} overdue; last good ${row.lastGoodAt?.toISOString() ?? 'never'}`
-  )
+  log.error('Platform backup overdue', {
+    organizationId,
+    lastGoodAt: row.lastGoodAt?.toISOString() ?? 'never',
+  })
   const operator = process.env.PLATFORM_BACKUP_ALERT_EMAIL?.trim()
   if (operator) {
     try {
@@ -372,7 +384,7 @@ async function alertIfPlatformOverdue(
         html: `<p>No platform backup of workshop <code>${organizationId}</code> has succeeded since ${row.lastGoodAt?.toISOString() ?? 'it was first scheduled'}.</p><p>See the Backups page on torqvoice.com admin for the failing runs.</p>`,
       })
     } catch (err) {
-      console.error('[cron] could not mail the operator:', err)
+      log.error('Could not mail the operator', { organizationId, err })
     }
   }
   await reportOverdue(null, organizationId, row.lastGoodAt)
@@ -388,31 +400,31 @@ export function processBackupRuns() {
     ticking = true
     // Restore points nobody looks at any more leave the disk now, not when the
     // next one is opened. On its own so a bad folder never holds up a backup.
-    await sweepRestorePoints().catch((err) =>
-      console.error('[cron] Restore point sweep failed:', err)
-    )
+    await sweepRestorePoints().catch((err) => log.error('Restore point sweep failed', { err }))
     // And the folder a restore that died mid-way left, for the same reason.
     await sweepAbandonedRestoreDirs()
       .then((n) => {
-        if (n > 0) console.warn(`[cron] Removed ${n} abandoned restore folder(s)`)
+        if (n > 0) log.info('Removed abandoned restore folders', { count: n })
       })
-      .catch((err) => console.error('[cron] Restore folder sweep failed:', err))
+      .catch((err) => log.error('Restore folder sweep failed', { err }))
     try {
-      const stale = await failStaleRuns()
-      if (stale > 0) console.warn(`[cron] Marked ${stale} stale backup run(s) as failed`)
+      await failStaleRuns()
+      // Routine: per tick at debug only. Each run's outcome is counted by
+      // the engine's hourly roll-up, and a failed one is logged as it fails.
       const queued = (await processDueBackups()) + (await processDuePlatformBackups())
-      if (queued > 0) console.warn(`[cron] Scheduled backups queued: ${queued}`)
+      if (queued > 0) log.debug('Scheduled backups queued', { count: queued })
       const ran = await drainBackupQueue()
-      if (ran > 0) console.warn(`[cron] Backup runs finished: ${ran}`)
+      if (ran > 0) log.debug('Backup runs finished this tick', { count: ran })
       // Old run rows go once a day, after the day's work, not on every tick.
       if (Date.now() - prunedAt > PRUNE_EVERY_MS) {
         prunedAt = Date.now()
         const pruned = await pruneOldRuns()
-        if (pruned > 0) console.warn(`[cron] Pruned ${pruned} old backup run row(s)`)
+        if (pruned > 0) log.info('Pruned old backup run rows', { count: pruned })
       }
     } catch (err) {
-      console.error('[cron] Backup scheduler failed:', err)
+      log.error('Backup scheduler failed', { err })
     } finally {
+      reportBackupRollup()
       ticking = false
     }
   })

+ 8 - 4
src/lib/cron/check-licenses.ts

@@ -3,6 +3,7 @@ import { db } from '@/lib/db'
 import { sendOrgMail, getOrgFromAddress } from '@/lib/email'
 import { notify } from '@/lib/notify'
 import { revalidateLicense } from '@/lib/license/revalidate'
+import { createLogger } from '@/lib/logger'
 import {
   LICENSE_TOKEN_MAX_AGE_DAYS,
   LICENSE_TOKEN_WARN_AGE_DAYS,
@@ -76,7 +77,7 @@ async function emailOwner(organizationId: string, subject: string, html: string)
     const from = await getOrgFromAddress(organizationId)
     await sendOrgMail(organizationId, { from, to: owner.user.email, subject, html })
   } catch (error) {
-    console.warn(`[cron] Failed to send license email for org ${organizationId}:`, error)
+    createLogger('cron').error('Could not send the licence email', { organizationId, err: error })
   }
 }
 
@@ -160,7 +161,10 @@ async function revalidateAll(orgs: KeyedOrg[]) {
     try {
       await revalidateOrganizationLicense(org.organizationId, org.key)
     } catch (error) {
-      console.error(`[cron] Failed to revalidate license for org ${org.organizationId}:`, error)
+      createLogger('cron').error('Licence revalidation failed', {
+        organizationId: org.organizationId,
+        err: error,
+      })
     }
   }
 }
@@ -191,7 +195,7 @@ export async function refreshLicensesMissingTokens() {
     if (needing.length === 0) return
     await revalidateAll(needing)
   } catch (error) {
-    console.error('[license] Boot-time licence refresh failed:', error)
+    createLogger('license').error('Boot-time licence refresh failed', { err: error })
   }
 }
 
@@ -201,7 +205,7 @@ export function checkLicenses() {
     try {
       await revalidateAll(await orgsWithKeys())
     } catch (error) {
-      console.error('[cron] License revalidation failed:', error)
+      createLogger('cron').error('Licence revalidation failed', { err: error })
     }
   })
 

+ 6 - 4
src/lib/cron/cleanup-audit-logs.ts

@@ -1,5 +1,6 @@
 import { CronJob } from 'cron'
 import { db } from '@/lib/db'
+import { createLogger } from '@/lib/logger'
 
 /**
  * Audit log retention cleanup — runs daily at 03:00 UTC.
@@ -21,12 +22,13 @@ export function cleanupAuditLogs() {
       })
 
       if (result.count > 0) {
-        console.warn(
-          `[cron] Audit log cleanup: deleted ${result.count} logs older than ${retentionDays} days`
-        )
+        createLogger('cron').info('Audit log cleanup', {
+          deleted: result.count,
+          olderThanDays: retentionDays,
+        })
       }
     } catch (error) {
-      console.error('[cron] Audit log cleanup failed:', error)
+      createLogger('cron').error('Audit log cleanup failed', { err: error })
     }
   })
 

+ 6 - 4
src/lib/cron/cleanup-portal-sessions.ts

@@ -1,5 +1,6 @@
 import { CronJob } from 'cron'
 import { db } from '@/lib/db'
+import { createLogger } from '@/lib/logger'
 
 /** Deletes expired portal sessions and used magic links daily at 02:00 UTC */
 export function cleanupPortalSessions() {
@@ -19,12 +20,13 @@ export function cleanupPortalSessions() {
       ])
 
       if (deletedSessions.count > 0 || deletedLinks.count > 0) {
-        console.warn(
-          `[cron] Cleanup: ${deletedSessions.count} sessions, ${deletedLinks.count} magic links`
-        )
+        createLogger('cron').info('Portal session cleanup', {
+          sessions: deletedSessions.count,
+          magicLinks: deletedLinks.count,
+        })
       }
     } catch (error) {
-      console.error('[cron] Portal session cleanup failed:', error)
+      createLogger('cron').error('Portal session cleanup failed', { err: error })
     }
   })
 

+ 14 - 13
src/lib/cron/demo-reset.ts

@@ -2,6 +2,7 @@ import { spawn } from 'node:child_process'
 import { CronJob } from 'cron'
 import { db } from '@/lib/db'
 import { isDemoMode } from '@/lib/demo'
+import { createLogger } from '@/lib/logger'
 
 /**
  * Auto-reset the demo org's data every 3 hours so visitors get a clean slate.
@@ -14,20 +15,20 @@ import { isDemoMode } from '@/lib/demo'
 
 const DEMO_ORG_ID = 'cmmh0vczm0000oiyan37vuyqf'
 const SEED_SCRIPT = 'prisma/seed_dummy_data.ts'
-const LOG_PREFIX = '[demo-reset]'
+const log = createLogger('demo-reset')
 
 let seedInFlight = false
 
 function runSeed(): Promise<void> {
   return new Promise((resolve) => {
     if (seedInFlight) {
-      console.warn(`${LOG_PREFIX} seed already running, skipping this tick`)
+      log.warn('Seed already running, skipping this tick')
       resolve()
       return
     }
     seedInFlight = true
     const startedAt = Date.now()
-    console.warn(`${LOG_PREFIX} starting seed: npx tsx ${SEED_SCRIPT}`)
+    log.info('Starting seed', { script: SEED_SCRIPT })
 
     const child = spawn('npx', ['tsx', SEED_SCRIPT], {
       cwd: process.cwd(),
@@ -37,26 +38,26 @@ function runSeed(): Promise<void> {
 
     child.stdout?.on('data', (buf: Buffer) => {
       const line = buf.toString().trimEnd()
-      if (line) console.warn(`${LOG_PREFIX} ${line}`)
+      if (line) log.debug(line)
     })
     child.stderr?.on('data', (buf: Buffer) => {
       const line = buf.toString().trimEnd()
-      if (line) console.error(`${LOG_PREFIX} ${line}`)
+      if (line) log.error(line)
     })
 
     child.on('close', (code) => {
       seedInFlight = false
       const ms = Date.now() - startedAt
       if (code === 0) {
-        console.warn(`${LOG_PREFIX} seed completed in ${ms}ms`)
+        log.info('Seed completed', { ms })
       } else {
-        console.error(`${LOG_PREFIX} seed exited with code ${code} after ${ms}ms`)
+        log.error('Seed failed', { exitCode: code, ms })
       }
       resolve()
     })
     child.on('error', (err) => {
       seedInFlight = false
-      console.error(`${LOG_PREFIX} failed to spawn seed process:`, err)
+      log.error('Could not spawn the seed process', { err })
       resolve()
     })
   })
@@ -66,13 +67,13 @@ async function seedIfEmpty() {
   try {
     const count = await db.customer.count({ where: { organizationId: DEMO_ORG_ID } })
     if (count === 0) {
-      console.warn(`${LOG_PREFIX} demo org empty on boot — seeding now`)
+      log.info('Demo org empty on boot, seeding now')
       await runSeed()
     } else {
-      console.warn(`${LOG_PREFIX} demo org has ${count} customers — skipping boot seed`)
+      log.info('Demo org already has data, skipping the boot seed', { customers: count })
     }
   } catch (err) {
-    console.error(`${LOG_PREFIX} boot-seed check failed:`, err)
+    log.error('Boot-seed check failed', { err })
   }
 }
 
@@ -90,9 +91,9 @@ export function startDemoResetCron() {
     try {
       await runSeed()
     } catch (err) {
-      console.error(`${LOG_PREFIX} scheduled reset failed:`, err)
+      log.error('Scheduled reset failed', { err })
     }
   })
   job.start()
-  console.warn(`${LOG_PREFIX} scheduled: every 3 hours (cron: 0 */3 * * *)`)
+  log.info('Scheduled every 3 hours', { cron: '0 */3 * * *' })
 }

+ 8 - 5
src/lib/cron/integration-jobs.ts

@@ -1,4 +1,5 @@
 import { CronJob } from 'cron'
+import { createLogger } from '@/lib/logger'
 import { createTickSummary } from './tick-summary'
 import {
   cleanupIntegrationHistory,
@@ -7,6 +8,8 @@ import {
   scheduleDueSyncs,
 } from '@/features/integrations/Lib/jobs'
 
+const log = createLogger('cron')
+
 /**
  * Integration job runner, every minute: put back anything a crashed worker
  * left running, queue the timed syncs that are due, then run due jobs.
@@ -16,15 +19,15 @@ export function processIntegrationJobs() {
   const job = new CronJob('* * * * *', async () => {
     try {
       const recovered = await recoverStuckJobs()
-      if (recovered > 0) console.warn(`[cron] Recovered ${recovered} stuck integration jobs`)
+      if (recovered > 0) log.warn('Recovered stuck integration jobs', { count: recovered })
       await scheduleDueSyncs()
       summary.add(await runDueJobs(50))
     } catch (err) {
-      console.error('[cron] Integration job runner failed:', err)
+      log.error('Integration job runner failed', { err })
     }
   })
   job.start()
-  console.warn('[cron] Integration job runner started (every minute)')
+  log.info('Integration job runner started (every minute)')
 }
 
 /** Daily at 03:40 UTC: logs older than 30 days, finished jobs older than 7. */
@@ -33,10 +36,10 @@ export function cleanupIntegrationLogs() {
     try {
       const result = await cleanupIntegrationHistory()
       if (result.jobs || result.logs) {
-        console.warn(`[cron] Integration cleanup: ${result.jobs} jobs, ${result.logs} log lines`)
+        log.info('Integration cleanup', { jobs: result.jobs, logLines: result.logs })
       }
     } catch (err) {
-      console.error('[cron] Integration cleanup failed:', err)
+      log.error('Integration cleanup failed', { err })
     }
   })
   job.start()

+ 17 - 13
src/lib/cron/low-stock-alerts.ts

@@ -3,6 +3,8 @@ import { db } from '@/lib/db'
 import { notify } from '@/lib/notify'
 import { sendOrgMail, getOrgFromAddress } from '@/lib/email'
 import { isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
 import { SETTING_KEYS } from '@/features/settings/Schema/settingsSchema'
 import {
   buildAlertSummary,
@@ -12,7 +14,7 @@ import {
   type LowStockCandidate,
 } from '@/features/inventory/Lib/lowStockAlerts'
 
-const LOG_PREFIX = '[low-stock]'
+const log = createLogger('low-stock')
 const DEFAULT_EMAIL_MIN_INTERVAL_HOURS = 24
 
 interface OrgAlertSettings {
@@ -76,10 +78,10 @@ async function sendDigest(
   try {
     fromAddress = await getOrgFromAddress(organizationId)
   } catch (error) {
-    console.error(
-      `${LOG_PREFIX} email is misconfigured (could not resolve sender); skipping digest:`,
-      error
-    )
+    log.error('Email is misconfigured (could not resolve the sender); skipping the digest', {
+      organizationId,
+      err: error,
+    })
     return false
   }
 
@@ -106,7 +108,7 @@ async function sendDigest(
       })
       sent++
     } catch (error) {
-      console.error(`${LOG_PREFIX} email to ${recipient.email} failed:`, error)
+      log.error('Digest email failed', { organizationId, err: error })
     }
   }
 
@@ -138,7 +140,7 @@ async function sendDigest(
     })
   } catch (error) {
     // Losing the timestamp only costs an extra digest later; never fail here.
-    console.error(`${LOG_PREFIX} could not record last-email time:`, error)
+    log.error('Could not record the last-email time', { organizationId, err: error })
   }
 
   return true
@@ -247,7 +249,7 @@ export async function processOrgLowStock(organizationId: string, now = new Date(
     try {
       emailed = await sendDigest(organizationId, newlyLow, now, settings.defaultThreshold)
     } catch (error) {
-      console.error(`${LOG_PREFIX} digest email failed:`, error)
+      log.error('Digest email failed', { organizationId, err: error })
     }
   }
 
@@ -267,6 +269,7 @@ export async function processOrgLowStock(organizationId: string, now = new Date(
 
 /** Hourly low-stock scan. Organizations opt in via settings. */
 export function checkLowStock() {
+  const summary = createTickSummary('Low-stock parts alerted')
   const job = new CronJob('15 * * * *', async () => {
     try {
       // Only organizations that switched the feature on.
@@ -284,15 +287,16 @@ export function checkLowStock() {
           const result = await processOrgLowStock(row.organizationId)
           if (!result.skipped) alerted += result.alerted
         } catch (error) {
-          console.error(`${LOG_PREFIX} org ${row.organizationId} failed:`, error)
+          log.error('Low-stock scan failed for a workshop', {
+            organizationId: row.organizationId,
+            err: error,
+          })
         }
       }
 
-      if (alerted > 0) {
-        console.warn(`${LOG_PREFIX} alerted on ${alerted} newly low part(s)`)
-      }
+      summary.add(alerted)
     } catch (error) {
-      console.error(`${LOG_PREFIX} scan failed:`, error)
+      log.error('Scan failed', { err: error })
     }
   })
 

+ 12 - 5
src/lib/cron/recurring-invoices.ts

@@ -13,6 +13,10 @@ import { lineTotal } from '@/features/inventory/Lib/partPricing'
 import { shiftWorkshopTime } from '@/lib/workshop-datetime'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
 import { isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
+
+const log = createLogger('cron')
 
 /**
  * One period on from `current`, at the same time on the workshop's clock.
@@ -49,6 +53,7 @@ async function generateInvoiceNumber(organizationId: string, timeZone: string):
 
 /** Generates invoices from recurring templates every hour */
 export function processRecurringInvoices() {
+  const summary = createTickSummary('Recurring invoices processed')
   const job = new CronJob('0 * * * *', async () => {
     try {
       const now = new Date()
@@ -167,15 +172,17 @@ export function processRecurringInvoices() {
 
           processed++
         } catch (error) {
-          console.error(`[cron] Failed to process recurring invoice ${ri.id}:`, error)
+          log.error('Recurring invoice failed', {
+            organizationId: ri.vehicle.organizationId,
+            recurringInvoiceId: ri.id,
+            err: error,
+          })
         }
       }
 
-      if (processed > 0) {
-        console.warn(`[cron] Processed ${processed} recurring invoices`)
-      }
+      summary.add(processed)
     } catch (error) {
-      console.error('[cron] Recurring invoice processing failed:', error)
+      log.error('Recurring invoice processing failed', { err: error })
     }
   })
 

+ 9 - 9
src/lib/cron/reminder-alerts.ts

@@ -3,8 +3,10 @@ import { db } from '@/lib/db'
 import { notify } from '@/lib/notify'
 import { sendOrgMail, getOrgFromAddress } from '@/lib/email'
 import { frozenOrganizationIds, isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
 
-const LOG_PREFIX = '[reminder-alerts]'
+const log = createLogger('reminder-alerts')
 
 type DueReminder = {
   id: string
@@ -61,7 +63,7 @@ async function sendReminderEmail(organizationId: string, reminders: DueReminder[
   try {
     fromAddress = await getOrgFromAddress(organizationId)
   } catch (error) {
-    console.error(`${LOG_PREFIX} could not resolve sender; skipping email:`, error)
+    log.error('Could not resolve the sender; skipping the email', { organizationId, err: error })
     return
   }
 
@@ -179,7 +181,7 @@ export async function processDueReminders(now = new Date()) {
       try {
         await sendReminderEmail(organizationId, emailReminders)
       } catch (error) {
-        console.error(`${LOG_PREFIX} email failed for org ${organizationId}:`, error)
+        log.error('Email failed', { organizationId, err: error })
       }
     }
 
@@ -196,16 +198,14 @@ export async function processDueReminders(now = new Date()) {
 
 /** Hourly scan for reminders that have come due. */
 export function checkDueReminders() {
+  const summary = createTickSummary('Due reminders notified')
   const job = new CronJob('*/10 * * * *', async () => {
     try {
-      const notified = await processDueReminders()
-      if (notified > 0) {
-        console.warn(`${LOG_PREFIX} notified on ${notified} due reminder(s)`)
-      }
+      summary.add(await processDueReminders())
     } catch (error) {
-      console.error(`${LOG_PREFIX} scan failed:`, error)
+      log.error('Scan failed', { err: error })
     }
   })
   job.start()
-  console.warn(`${LOG_PREFIX} Due-reminder processor started (hourly)`)
+  log.info('Due-reminder processor started (every 10 minutes)')
 }

+ 24 - 6
src/lib/cron/report-schedules.ts

@@ -21,6 +21,10 @@ import { shiftWorkshopTime, workshopMonthKey } from '@/lib/workshop-datetime'
 import { isShopFeeLine } from '@/features/settings/Lib/shopFee'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
 import { isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
+
+const log = createLogger('cron')
 
 // --------------- date helpers ---------------
 // All in the workshop's zone: the server runs in UTC and its clock says
@@ -778,7 +782,11 @@ export async function processOneSchedule(
         ],
       })
     } catch (emailErr) {
-      console.error(`[cron] Failed to email report to ${recipient.email}:`, emailErr)
+      log.error('Could not email a scheduled report', {
+        organizationId: schedule.organizationId,
+        scheduleId: schedule.id,
+        err: emailErr,
+      })
     }
   }
 
@@ -796,15 +804,19 @@ export async function processOneSchedule(
     },
   })
 
-  console.warn(
-    `[cron] Report schedule ${schedule.id} processed, sent to ${recipients.length} recipients`
-  )
+  log.debug('Report schedule processed', {
+    organizationId: schedule.organizationId,
+    scheduleId: schedule.id,
+    recipients: recipients.length,
+  })
 }
 
 // --------------- main cron ---------------
 
 export function processReportSchedules() {
+  const summary = createTickSummary('Report schedules processed')
   const job = new CronJob('0 * * * *', async () => {
+    let processed = 0
     const dueSchedules = await db.reportSchedule.findMany({
       where: { isActive: true, nextRunDate: { lte: new Date() } },
     })
@@ -821,12 +833,18 @@ export function processReportSchedules() {
           zones.set(schedule.organizationId, timeZone)
         }
         await processOneSchedule(schedule, timeZone)
+        processed++
       } catch (err) {
-        console.error(`[cron] Failed to process report schedule ${schedule.id}:`, err)
+        log.error('Report schedule failed', {
+          organizationId: schedule.organizationId,
+          scheduleId: schedule.id,
+          err,
+        })
       }
     }
+    summary.add(processed)
   })
 
   job.start()
-  console.warn('[cron] Report schedule processor started (hourly)')
+  log.info('Report schedule processor started (hourly)')
 }

+ 16 - 9
src/lib/cron/scheduled-messages.ts

@@ -6,8 +6,10 @@ import {
 } from '@/features/scheduled-messages/Lib/dispatchScheduledMessage'
 import { workshopTimeZone } from '@/lib/workshop-timezone'
 import { frozenOrganizationIds, isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
+import { createTickSummary } from './tick-summary'
 
-const LOG_PREFIX = '[scheduled-messages]'
+const log = createLogger('scheduled-messages')
 
 /** Nothing older than this is sent on a late start; it is marked failed instead. */
 const MAX_LATENESS_MS = 24 * 60 * 60 * 1000
@@ -74,7 +76,10 @@ export async function processDueMessages(now = new Date()): Promise<number> {
           errorMessage: 'Missed its send window and was skipped',
         },
       })
-      console.warn(`${LOG_PREFIX} skipped stale message ${message.id}`)
+      log.warn('Skipped a stale message', {
+        organizationId: message.organizationId,
+        messageId: message.id,
+      })
       continue
     }
 
@@ -104,7 +109,11 @@ export async function processDueMessages(now = new Date()): Promise<number> {
           errorMessage,
         },
       })
-      console.error(`${LOG_PREFIX} send failed for ${message.id}:`, errorMessage)
+      log.error('Send failed', {
+        organizationId: message.organizationId,
+        messageId: message.id,
+        error: errorMessage,
+      })
     }
   }
 
@@ -113,16 +122,14 @@ export async function processDueMessages(now = new Date()): Promise<number> {
 
 /** Minute-by-minute scan, so a message goes out on the minute it was set for. */
 export function processScheduledMessages() {
+  const summary = createTickSummary('Scheduled messages sent')
   const job = new CronJob('* * * * *', async () => {
     try {
-      const sent = await processDueMessages()
-      if (sent > 0) {
-        console.warn(`${LOG_PREFIX} sent ${sent} scheduled message(s)`)
-      }
+      summary.add(await processDueMessages())
     } catch (error) {
-      console.error(`${LOG_PREFIX} scan failed:`, error)
+      log.error('Scan failed', { err: error })
     }
   })
   job.start()
-  console.warn(`${LOG_PREFIX} Scheduled-message processor started (every minute)`)
+  log.info('Scheduled-message processor started (every minute)')
 }

+ 12 - 9
src/lib/cron/sweep-orphan-files.ts

@@ -1,5 +1,8 @@
 import { CronJob } from 'cron'
 import { purgeTrash, sweepOrphanFiles, TRASH_DAYS } from '@/lib/files/manager'
+import { createLogger } from '@/lib/logger'
+
+const log = createLogger('cron')
 
 /**
  * Uploaded files no row points at, once a day at 03:30 UTC: moved to the
@@ -18,22 +21,22 @@ export function sweepOrphanFilesDaily() {
   const job = new CronJob('30 3 * * *', async () => {
     try {
       const { checked, removed, refused, skipped, deferred } = await sweepOrphanFiles()
-      if (refused) console.error(`[cron] File sweep refused: ${refused}`)
+      if (refused) log.error('File sweep refused', { reason: refused })
       else if (removed > 0 || deferred > 0) {
-        console.warn(
-          `[cron] File sweep: ${removed} unused files of ${checked} moved to the trash, ${deferred} left for the next run`
-        )
+        log.info('File sweep moved unused files to the trash', {
+          removed,
+          checked,
+          deferred,
+        })
       }
       if (skipped.length > 0) {
-        console.error(`[cron] File sweep left ${skipped.length} workshop(s) alone, see above`)
+        log.error('File sweep left workshops alone, see above', { count: skipped.length })
       }
       const purged = await purgeTrash()
       if (purged > 0)
-        console.warn(
-          `[cron] File sweep: ${purged} trash day(s) older than ${TRASH_DAYS} days deleted`
-        )
+        log.info('File sweep deleted old trash days', { days: purged, olderThanDays: TRASH_DAYS })
     } catch (error) {
-      console.error('[cron] File sweep failed:', error)
+      log.error('File sweep failed', { err: error })
     }
   })
   job.start()

+ 5 - 6
src/lib/cron/sync-subscriptions.ts

@@ -1,6 +1,7 @@
 import { CronJob } from 'cron'
 import { isCloudMode } from '@/lib/features'
 import { billingRequest, isTorqvoiceComBillingConfigured } from '@/lib/torqvoice-com'
+import { createLogger } from '@/lib/logger'
 
 /**
  * Cloud mode: once a day, asks torqvoice.com to compare every Stripe-backed
@@ -17,13 +18,11 @@ export function syncSubscriptions() {
         'sync',
         {}
       )
-      if (result.synced > 0 || result.errors > 0) {
-        console.warn(
-          `[cron] Subscription sync: ${result.checked} checked, ${result.synced} synced, ${result.errors} errors`
-        )
-      }
+      const fields = { checked: result.checked, synced: result.synced, errors: result.errors }
+      if (result.errors > 0) createLogger('cron').error('Subscription sync had errors', fields)
+      else if (result.synced > 0) createLogger('cron').info('Subscription sync', fields)
     } catch (error) {
-      console.error('[cron] Subscription sync failed:', error)
+      createLogger('cron').error('Subscription sync failed', { err: error })
     }
   })
 

+ 17 - 6
src/lib/cron/tick-summary.ts

@@ -1,23 +1,34 @@
+import { createLogger } from '@/lib/logger'
+
 /**
  * Every-minute crons used to print one line per tick that did work, which on
  * a busy install is a line a minute of "processed: 1". Routine work is already
  * recorded per item (integration logs, delivery rows), so the console only
- * needs an hourly total. Failures and recoveries still log immediately.
+ * needs an hourly total, at `info`; each tick's count is `debug`. Failures
+ * and recoveries still log immediately.
+ *
+ * Call `add` on every tick, idle ones too, so the total goes out when the
+ * hour is up rather than at the next tick that happens to do work.
  */
 const SUMMARY_MS = 60 * 60 * 1000
 
 export function createTickSummary(label: string, intervalMs = SUMMARY_MS) {
+  const log = createLogger('cron')
   let total = 0
   let ticks = 0
   let since = Date.now()
   return {
     add(count: number) {
-      if (count <= 0) return
-      total += count
-      ticks++
+      if (count > 0) {
+        total += count
+        ticks++
+        log.debug(`${label} this run`, { count })
+      }
       if (Date.now() - since < intervalMs) return
-      const minutes = Math.round((Date.now() - since) / 60_000)
-      console.warn(`[cron] ${label}: ${total} in ${ticks} run(s) over the last ${minutes} min`)
+      if (total > 0) {
+        const minutes = Math.round((Date.now() - since) / 60_000)
+        log.info(`${label}: ${total} in ${ticks} run(s) over the last ${minutes} min`)
+      }
       total = 0
       ticks = 0
       since = Date.now()

+ 8 - 9
src/lib/cron/webhook-deliveries.ts

@@ -1,8 +1,11 @@
 import { CronJob } from 'cron'
 import { db } from '@/lib/db'
 import { processDueDeliveries, recoverStuckDeliveries } from '@/features/webhooks/Lib/deliver'
+import { createLogger } from '@/lib/logger'
 import { createTickSummary } from './tick-summary'
 
+const log = createLogger('cron')
+
 /**
  * Webhook delivery retry cron — runs every minute. First sweeps any
  * "inflight" rows that look stuck (a worker crashed mid-delivery), then
@@ -13,16 +16,14 @@ export function processWebhookDeliveries() {
   const job = new CronJob('* * * * *', async () => {
     try {
       const recovered = await recoverStuckDeliveries()
-      if (recovered > 0) {
-        console.warn(`[cron] Recovered ${recovered} stuck webhook deliveries`)
-      }
+      if (recovered > 0) log.warn('Recovered stuck webhook deliveries', { count: recovered })
       summary.add(await processDueDeliveries(100))
     } catch (err) {
-      console.error('[cron] Webhook delivery processor failed:', err)
+      log.error('Webhook delivery processor failed', { err })
     }
   })
   job.start()
-  console.warn('[cron] Webhook delivery processor started (every minute)')
+  log.info('Webhook delivery processor started (every minute)')
 }
 
 /**
@@ -40,12 +41,10 @@ export function cleanupWebhookDeliveries() {
         where: { createdAt: { lt: cutoff } },
       })
       if (result.count > 0) {
-        console.warn(
-          `[cron] Webhook delivery cleanup: deleted ${result.count} rows older than ${days} days`
-        )
+        log.info('Webhook delivery cleanup', { deleted: result.count, olderThanDays: days })
       }
     } catch (err) {
-      console.error('[cron] Webhook delivery cleanup failed:', err)
+      log.error('Webhook delivery cleanup failed', { err })
     }
   })
   job.start()

+ 3 - 2
src/lib/cron/workshop-holidays.ts

@@ -4,6 +4,7 @@ import { isRestoring } from '@/lib/backup/freeze'
 import { SETTING_KEYS } from '@/features/settings/Schema/settingsSchema'
 import { syncPublicHolidays } from '@/features/workshop-calendar/Lib/holidaySync.server'
 import { refreshLinkedFeeds } from '@/features/workshop-calendar/Lib/feedSync.server'
+import { createLogger } from '@/lib/logger'
 
 /**
  * Once a day, fetch each workshop's public holidays again (so next year's
@@ -31,7 +32,7 @@ export async function syncWorkshopHolidaysOnce(now = new Date()): Promise<number
       await refreshLinkedFeeds(organizationId, { now })
       synced++
     } catch (error) {
-      console.error(`[cron] Holiday sync failed for ${organizationId}:`, error)
+      createLogger('cron').error('Holiday sync failed', { organizationId, err: error })
     }
   }
   return synced
@@ -43,7 +44,7 @@ export function syncWorkshopHolidays() {
     try {
       await syncWorkshopHolidaysOnce()
     } catch (error) {
-      console.error('[cron] Holiday sync failed:', error)
+      createLogger('cron').error('Holiday sync failed', { err: error })
     }
   })
   job.start()

+ 21 - 16
src/lib/files/manager.ts

@@ -17,6 +17,7 @@ import path from 'node:path'
 import { UPLOAD_CATEGORIES } from '@/lib/upload-url'
 import { uploadsRoot, uploadsRoots } from '@/lib/upload-root'
 import { isRestoring } from '@/lib/backup/freeze'
+import { createLogger } from '@/lib/logger'
 import {
   allReferencedSuffixes,
   existingOrganizationIds,
@@ -24,6 +25,8 @@ import {
   referencedSuffixes,
 } from './references'
 
+const log = createLogger('files')
+
 /**
  * The one place uploaded files are deleted from disk.
  *
@@ -200,7 +203,7 @@ export async function releaseFiles(
     inUse = await referencedSuffixes([...candidates.keys()])
   } catch (err) {
     // Unable to tell whether they are in use: keep every one.
-    console.error(`[files] could not check references (${reason}); nothing deleted:`, err)
+    log.error('Could not check references; nothing deleted', { organizationId, reason, err })
     result.skipped.push(...[...candidates.values()].map((c) => c.url))
     return result
   }
@@ -217,18 +220,19 @@ export async function releaseFiles(
       try {
         if (await trashFile(root, target, file, now)) removed = true
       } catch (err) {
-        console.error(`[files] could not delete ${file.suffix} (${reason}):`, err)
+        log.error('Could not delete a file', { organizationId, file: file.suffix, reason, err })
       }
     }
     if (removed) result.removed.push(url)
   }
 
   if (result.removed.length > 0) {
-    console.warn(
-      `[files] ${reason}: moved ${result.removed.length} unused file(s) of ${organizationId} to the trash: ${result.removed
-        .map((url) => parseStoredFileUrl(url)?.suffix)
-        .join(', ')}`
-    )
+    log.info('Moved unused files to the trash', {
+      organizationId,
+      reason,
+      count: result.removed.length,
+      files: result.removed.map((url) => parseStoredFileUrl(url)?.suffix).join(','),
+    })
   }
   return result
 }
@@ -254,7 +258,7 @@ export async function discardUnsavedUpload(
   })
   if (!target) return
   await deleteFile(target).catch((err) =>
-    console.error(`[files] could not discard /${folder}/${name}:`, err)
+    log.error('Could not discard a file', { file: `/${folder}/${name}`, err })
   )
 }
 
@@ -327,7 +331,7 @@ export async function removeOrganizationFiles(
       if (!(await taken(dir))) continue
       await moveInto(dir, path.join(path.resolve(root), TRASH, day, organizationId))
     } catch (err) {
-      console.error(`[files] could not move the uploads of ${organizationId} to the trash:`, err)
+      log.error('Could not move the uploads to the trash', { organizationId, err })
     }
   }
 }
@@ -356,7 +360,7 @@ export async function releaseInstallFile(
   try {
     return await trashFile(root, target, file, now)
   } catch (err) {
-    console.error(`[files] could not release install file ${file.suffix}:`, err)
+    log.error('Could not release an install file', { file: file.suffix, err })
     return false
   }
 }
@@ -454,7 +458,7 @@ export async function sweepOrphanFiles({
   const result: SweepResult = { checked: 0, removed: 0, skipped: [], deferred: 0 }
   const refuse = (reason: string, err?: unknown) => {
     result.refused = reason
-    console.error(`[files] sweep refused, nothing moved: ${reason}`, ...(err ? [err] : []))
+    log.error('Sweep refused, nothing moved', { reason, err })
     return result
   }
   if (!(await hasAnyOrganization())) return refuse('the database has no workshops')
@@ -519,7 +523,7 @@ export async function sweepOrphanFiles({
     if (!trustDatabase && checked >= SWEEP_MIN_FOR_SHARE && share > SWEEP_MAX_SHARE) {
       const reason = `${theirs.length} of ${checked} files look unused, more than a sweep should find; check the database, then run with ${SWEEP_TRUST_DATABASE}=1 if they really are`
       result.skipped.push({ organizationId, reason })
-      console.error(`[files] sweep left ${organizationId} alone: ${reason}`)
+      log.error('Sweep left a workshop alone', { organizationId, reason })
       continue
     }
     unused.push(...theirs)
@@ -542,13 +546,14 @@ export async function sweepOrphanFiles({
     if (stillUsed.has(file.suffix)) continue
     if (await trashFile(root, target, file, now).catch(() => false)) {
       result.removed += 1
-      console.warn(
-        `[files] sweep: moved unused ${file.suffix} of ${file.organizationId} to the trash`
-      )
+      log.debug('Sweep moved an unused file to the trash', {
+        organizationId: file.organizationId,
+        file: file.suffix,
+      })
     }
   }
   if (result.deferred > 0) {
-    console.warn(`[files] sweep: ${result.deferred} more unused file(s) left for the next run`)
+    log.info('Sweep left unused files for the next run', { count: result.deferred })
   }
   return result
 }

+ 158 - 0
src/lib/logger.ts

@@ -0,0 +1,158 @@
+/**
+ * The server's log lines, as `docker logs` shows them.
+ *
+ * Every line is one line: `[scope] message key=value ...`, no timestamp
+ * (Docker adds one), objects flattened, so a `grep` for a workshop id or a
+ * scope finds the whole event. `LOG_LEVEL` picks how much is said: `error`,
+ * `warn`, `info` (the default) or `debug`. Routine successes of recurring
+ * jobs are `debug`; at `info` they arrive as an hourly roll-up instead (see
+ * `createRollup`). Failures are `error` and always print.
+ *
+ * Never pass customer data, secrets or tokens as fields; ids are fine.
+ */
+
+export type LogLevel = 'error' | 'warn' | 'info' | 'debug'
+
+const RANK: Record<LogLevel, number> = { error: 0, warn: 1, info: 2, debug: 3 }
+
+export type LogFields = Record<string, unknown>
+
+/** The level set by LOG_LEVEL, read on every call so a test or a restart changes it. */
+export function logLevel(): LogLevel {
+  const raw = process.env.LOG_LEVEL?.trim().toLowerCase()
+  return raw && Object.hasOwn(RANK, raw) ? (raw as LogLevel) : 'info'
+}
+
+export function logEnabled(level: LogLevel): boolean {
+  return RANK[level] <= RANK[logLevel()]
+}
+
+const PLAIN = /^[^\s"=\\]+$/
+
+function scalar(value: unknown): string {
+  if (value === null || value === undefined) return String(value)
+  if (value instanceof Date) {
+    return Number.isNaN(value.getTime()) ? 'invalid-date' : value.toISOString()
+  }
+  if (typeof value === 'bigint') return value.toString()
+  if (typeof value === 'number' || typeof value === 'boolean') return String(value)
+  const text = typeof value === 'string' ? value : safeJson(value)
+  // Quoted and escaped whenever it has a space, a quote or a line break, so
+  // a message from a library can never start a second line.
+  return text !== '' && PLAIN.test(text) ? text : JSON.stringify(text)
+}
+
+function safeJson(value: unknown): string {
+  try {
+    return JSON.stringify(value, (_k, v) => (typeof v === 'bigint' ? v.toString() : v)) ?? ''
+  } catch {
+    return String(value)
+  }
+}
+
+/** One error as fields: its message, its code when it has one, and the stack at debug. */
+function errorFields(prefix: string, err: unknown): LogFields {
+  if (!(err instanceof Error)) return { [prefix]: err }
+  const out: LogFields = { [prefix]: err.message || err.name }
+  const code = (err as { code?: unknown }).code
+  if (typeof code === 'string' || typeof code === 'number') out[`${prefix}Code`] = code
+  if (logEnabled('debug') && err.stack) out[`${prefix}Stack`] = err.stack
+  return out
+}
+
+function flatten(fields: LogFields, prefix = '', out: string[] = [], depth = 0): string[] {
+  for (const [key, value] of Object.entries(fields)) {
+    if (value === undefined) continue
+    const name = prefix ? `${prefix}.${key}` : key
+    if (value instanceof Error) {
+      for (const [k, v] of Object.entries(errorFields(name, value))) out.push(`${k}=${scalar(v)}`)
+    } else if (
+      value !== null &&
+      typeof value === 'object' &&
+      !(value instanceof Date) &&
+      !Array.isArray(value) &&
+      depth < 2
+    ) {
+      flatten(value as LogFields, name, out, depth + 1)
+    } else {
+      out.push(`${name}=${scalar(value)}`)
+    }
+  }
+  return out
+}
+
+/** The line as it is printed. Exported for tests and for anything that must match it. */
+export function formatLine(scope: string, message: string, fields?: LogFields): string {
+  const parts = [`[${scope}]`, oneLine(message)]
+  if (fields) parts.push(...flatten(fields))
+  return parts.join(' ')
+}
+
+function oneLine(text: string): string {
+  return text.replace(/\r?\n\s*/g, ' ')
+}
+
+function write(level: LogLevel, scope: string, message: string, fields?: LogFields) {
+  if (!logEnabled(level)) return
+  const line = formatLine(scope, message, fields)
+  // Biome allows warn and error only; both reach `docker logs`.
+  if (level === 'error') console.error(line)
+  else console.warn(line)
+}
+
+export interface Logger {
+  error(message: string, fields?: LogFields): void
+  warn(message: string, fields?: LogFields): void
+  info(message: string, fields?: LogFields): void
+  debug(message: string, fields?: LogFields): void
+}
+
+export function createLogger(scope: string): Logger {
+  return {
+    error: (message, fields) => write('error', scope, message, fields),
+    warn: (message, fields) => write('warn', scope, message, fields),
+    info: (message, fields) => write('info', scope, message, fields),
+    debug: (message, fields) => write('debug', scope, message, fields),
+  }
+}
+
+const HOUR_MS = 60 * 60 * 1000
+
+/**
+ * A periodic total for a recurring job, at `info`: the job reports every
+ * tick (zeros included, so the total goes out on time), and once an interval
+ * has passed one line says what happened, e.g. `[cron] Backups: 14 finished,
+ * 0 failed over the last 60 min`. Nothing at all when every count was zero.
+ * The per-tick detail belongs at `debug`; failures are logged where they
+ * happen, never only here.
+ */
+export function createRollup(
+  scope: string,
+  label: string,
+  options: { intervalMs?: number; order?: string[] } = {}
+) {
+  const intervalMs = options.intervalMs ?? HOUR_MS
+  let counts: Record<string, number> = {}
+  let since = Date.now()
+  const log = createLogger(scope)
+  return {
+    add(tick: Record<string, number>) {
+      for (const [key, n] of Object.entries(tick)) {
+        if (n > 0) counts[key] = (counts[key] ?? 0) + n
+      }
+      if (Date.now() - since < intervalMs) return
+      const total = Object.values(counts).reduce((a, b) => a + b, 0)
+      if (total > 0) {
+        const keys = [
+          ...(options.order ?? []),
+          ...Object.keys(counts).filter((k) => !options.order?.includes(k)),
+        ]
+        const minutes = Math.round((Date.now() - since) / 60_000)
+        const what = keys.map((k) => `${counts[k] ?? 0} ${k}`).join(', ')
+        log.info(`${label}: ${what} over the last ${minutes} min`)
+      }
+      counts = {}
+      since = Date.now()
+    },
+  }
+}