restore.test.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787
  1. // @vitest-environment node
  2. import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
  3. import { tmpdir } from 'node:os'
  4. import path from 'node:path'
  5. import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
  6. /**
  7. * Restoring a snapshot from a backup destination, with restic and the
  8. * import mocked: restic "writes" the snapshot into the target folder the
  9. * engine asks for, and the import records what it was given. What is
  10. * checked is the glue: the run row, the lock, where the folder goes, how
  11. * the data file and the uploads folder are found, and that the folder is
  12. * removed however the run ends.
  13. *
  14. * The safety copy is the engine's real takeSafetySnapshot, not a stub: the
  15. * data writer is a fake and restic's init, unlock and backup are mocked, so
  16. * what is checked is that an actual snapshot of the workshop (tagged
  17. * pre-restore, with what the destination is set to keep) is asked of the
  18. * repository the restore reads from, before anything in the workshop is
  19. * replaced. A self-hosted install
  20. * has no platform repository, so that is where it goes here.
  21. */
  22. type Row = Record<string, any>
  23. const h = vi.hoisted(() => ({
  24. runs: new Map<string, Row>(),
  25. connections: new Map<string, Row>(),
  26. seq: 0,
  27. /** What the fake restic restore writes under the target, as relative paths. */
  28. restoredTree: {} as Record<string, string>,
  29. /** The folder restic was asked to restore into, kept to look at later. */
  30. restoreTarget: null as string | null,
  31. /** Whether the restored files still existed when the import ran. */
  32. seenAtImport: null as { dataFile: boolean; filesDir: boolean | null; restoring: boolean } | null,
  33. importResult: { redactedSettings: ['ai.apiKey'] } as Row,
  34. importError: null as Error | null,
  35. log: vi.fn(async () => undefined),
  36. /** Where the fake destination keeps its repository: a folder under the test root. */
  37. repoDir: '',
  38. /** What happened, in order: the safety copy, restic's restore, the import. */
  39. events: [] as string[],
  40. /** Thrown by the fake restic backup, so the safety copy fails. */
  41. safetyError: null as Error | null,
  42. /** Every run's safetySnapshotId at the moment restic restored. */
  43. safetyAtRestore: null as (string | null)[] | null,
  44. owner: { userId: 'owner1' } as Row | null,
  45. /** The destination's backup settings. */
  46. settings: {} as Row,
  47. }))
  48. function merge(row: Row, data: Row): Row {
  49. const out = { ...row }
  50. for (const [k, v] of Object.entries(data)) if (v !== undefined) out[k] = v
  51. return out
  52. }
  53. vi.mock('@/lib/db', () => {
  54. const db: Record<string, any> = {
  55. backupRun: {
  56. create: vi.fn(async ({ data }: { data: Row }) => {
  57. const id = `run${++h.seq}`
  58. h.runs.set(id, {
  59. id,
  60. startedAt: null,
  61. finishedAt: null,
  62. snapshotId: null,
  63. filesNew: null,
  64. bytesAdded: null,
  65. totalBytes: null,
  66. error: null,
  67. createdAt: new Date(),
  68. ...data,
  69. })
  70. return { id }
  71. }),
  72. update: vi.fn(async ({ where, data }: { where: { id: string }; data: Row }) => {
  73. const row = merge(h.runs.get(where.id) ?? {}, data)
  74. h.runs.set(where.id, row)
  75. return row
  76. }),
  77. findUnique: vi.fn(
  78. async ({ where }: { where: { id: string } }) => h.runs.get(where.id) ?? null
  79. ),
  80. // The busy check (organisation, kind, connection, open statuses) and
  81. // the queue's next run (status 'queued', oldest first).
  82. findFirst: vi.fn(async ({ where, orderBy }: { where: Row; orderBy?: Row }) => {
  83. const statuses: string[] =
  84. typeof where.status === 'string' ? [where.status] : where.status.in
  85. const rows = [...h.runs.values()].filter(
  86. (r) =>
  87. statuses.includes(r.status) &&
  88. (where.organizationId === undefined || r.organizationId === where.organizationId) &&
  89. (where.kind === undefined || r.kind === where.kind) &&
  90. (where.connectionId === undefined || r.connectionId === where.connectionId) &&
  91. (where.trigger === undefined ||
  92. (typeof where.trigger === 'string'
  93. ? r.trigger === where.trigger
  94. : where.trigger.in.includes(r.trigger)))
  95. )
  96. if (orderBy) rows.sort((a, b) => a.createdAt.getTime() - b.createdAt.getTime())
  97. return rows[0] ?? null
  98. }),
  99. },
  100. integrationConnection: {
  101. findUnique: vi.fn(async ({ where }: { where: { id: string } }) => {
  102. const row = h.connections.get(where.id)
  103. return row ? { ...row } : null
  104. }),
  105. },
  106. organizationMember: {
  107. findFirst: vi.fn(async () => h.owner),
  108. findMany: vi.fn(async () => [{ user: { email: 'owner@workshop.test' } }]),
  109. },
  110. }
  111. // The queue's transaction: the advisory lock is a no-op here, and the
  112. // transaction client is the same fake.
  113. db.$transaction = async (fn: (tx: Record<string, any>) => unknown) => fn(db)
  114. db.$executeRaw = vi.fn(async () => 0)
  115. return { db }
  116. })
  117. vi.mock('@/features/integrations/Lib/connections', () => ({
  118. loadConnection: vi.fn(async (id: string) => {
  119. const row = h.connections.get(id)
  120. if (!row) throw new Error('Integration connection not found')
  121. return {
  122. ctx: {
  123. connection: {
  124. id,
  125. organizationId: row.organizationId,
  126. connectorId: row.connectorId,
  127. settings: { ...h.settings },
  128. state: {},
  129. },
  130. credentials: { password: 'correct horse battery' },
  131. log: h.log,
  132. saveState: vi.fn(async () => undefined),
  133. },
  134. server: {
  135. backupTarget: { repository: async () => ({ repository: h.repoDir, env: {} }) },
  136. },
  137. status: row.status,
  138. }
  139. }),
  140. setConnectionStatus: vi.fn(async () => undefined),
  141. }))
  142. vi.mock('@/integrations/registry', () => ({
  143. getManifest: (id: string) => ({ id, name: 'Test storage', capabilities: ['backup.target'] }),
  144. listManifests: () => [],
  145. }))
  146. vi.mock('@/lib/audit', () => ({ logAudit: vi.fn(async () => undefined) }))
  147. vi.mock('@/lib/backup/build-data', () => ({
  148. writeBackupData: vi.fn(
  149. async (organizationId: string, _options: Row, out: NodeJS.WritableStream) => {
  150. out.write(JSON.stringify({ organizationId }))
  151. return { datasets: ['customers'] }
  152. }
  153. ),
  154. }))
  155. vi.mock('@/lib/features', () => ({
  156. isCloudMode: () => false,
  157. getFeatures: vi.fn(async () => ({ platformBackups: false })),
  158. }))
  159. vi.mock('@/i18n/organization-translator', () => ({
  160. organizationTranslator: async () =>
  161. Object.assign((key: string) => key, { has: () => false, locale: 'en' }),
  162. }))
  163. vi.mock('@/lib/workshop-timezone', () => ({ workshopTimeZone: async () => 'UTC' }))
  164. vi.mock('@/lib/notify', () => ({ notify: vi.fn(async () => undefined) }))
  165. vi.mock('@/lib/email', () => ({
  166. getOrgFromAddress: vi.fn(async () => 'Workshop <post@workshop.test>'),
  167. sendOrgMail: vi.fn(async () => undefined),
  168. getFromAddress: vi.fn(async () => 'Torqvoice <noreply@torqvoice.test>'),
  169. sendMail: vi.fn(async () => undefined),
  170. }))
  171. vi.mock('@/lib/backup/restic', async (importOriginal) => {
  172. const actual = await importOriginal<typeof import('@/lib/backup/restic')>()
  173. return {
  174. ...actual,
  175. resticInit: vi.fn(async () => ({ created: false })),
  176. resticUnlock: vi.fn(async () => undefined),
  177. resticBackup: vi.fn(async () => {
  178. h.events.push('safety')
  179. if (h.safetyError) throw h.safetyError
  180. return {
  181. snapshotId: SAFETY,
  182. filesNew: 1,
  183. filesChanged: 0,
  184. dataAdded: 100,
  185. totalBytesProcessed: 100,
  186. totalFilesProcessed: 1,
  187. partial: false,
  188. }
  189. }),
  190. resticSnapshots: vi.fn(async () => []),
  191. resticRestoreSize: vi.fn(async () => 0),
  192. resticRestore: vi.fn(async (_opts: unknown, input: { snapshotId: string; target: string }) => {
  193. h.events.push('restore')
  194. h.safetyAtRestore = [...h.runs.values()].map((r) => r.safetySnapshotId ?? null)
  195. h.restoreTarget = input.target
  196. let bytes = 0
  197. for (const [rel, body] of Object.entries(h.restoredTree)) {
  198. const file = path.join(input.target, rel)
  199. mkdirSync(path.dirname(file), { recursive: true })
  200. writeFileSync(file, body)
  201. bytes += body.length
  202. }
  203. return { filesRestored: Object.keys(h.restoredTree).length, bytesRestored: bytes }
  204. }),
  205. }
  206. })
  207. // A data file the server would hold is tiny here, so the cap can be tripped
  208. // without writing a hundred megabytes.
  209. vi.mock('@/lib/backup/limits', async (importOriginal) => ({
  210. ...(await importOriginal<typeof import('@/lib/backup/limits')>()),
  211. maxDataFileBytes: () => 1000,
  212. }))
  213. vi.mock('@/lib/backup/import', () => ({
  214. restoreFromDisk: vi.fn(
  215. async (_actor: unknown, input: { dataFile: string; filesDir: string | null }) => {
  216. const { isRestoring } = await import('@/lib/backup/freeze')
  217. h.events.push('import')
  218. h.seenAtImport = {
  219. dataFile: existsSync(input.dataFile),
  220. filesDir: input.filesDir === null ? null : existsSync(input.filesDir),
  221. restoring: isRestoring('org1'),
  222. }
  223. if (h.importError) throw h.importError
  224. return h.importResult
  225. }
  226. ),
  227. }))
  228. import { logAudit } from '@/lib/audit'
  229. import { writeBackupData } from '@/lib/backup/build-data'
  230. import { BackupBusyError } from '@/lib/backup/engine'
  231. import { isRestoring } from '@/lib/backup/freeze'
  232. import { restoreFromDisk } from '@/lib/backup/import'
  233. import {
  234. resticBackup,
  235. resticRestore,
  236. resticRestoreSize,
  237. resticSnapshots,
  238. } from '@/lib/backup/restic'
  239. import { db } from '@/lib/db'
  240. import { notify } from '@/lib/notify'
  241. import {
  242. listSnapshots,
  243. locateRestored,
  244. queueRestore,
  245. restoreDirFor,
  246. runRestore,
  247. sweepAbandonedRestoreDirs,
  248. } from '@/lib/backup/restore'
  249. const ORG = 'org1'
  250. const CONN = 'conn1'
  251. const SNAPSHOT = '28891c8f448a3c36c2271f6dc4e03b08052265e671b6a8c438c9358a8a1e602b'
  252. const OLDER = '9d6cb16e448a3c36c2271f6dc4e03b08052265e671b6a8c438c9358a8a1e602b'
  253. const SAFETY = '5a1e7c3b448a3c36c2271f6dc4e03b08052265e671b6a8c438c9358a8a1e602b'
  254. const SAFETY_FAILED = 'Could not take a safety copy of the workshop first; nothing was changed.'
  255. const root = mkdtempSync(path.join(tmpdir(), 'backup-restore-test-'))
  256. const dataDir = path.join(root, 'data')
  257. // The engine stages each workshop's data file at a fixed path under
  258. // os.tmpdir() and deletes it after the run. engine.test.ts stages the same
  259. // workshop, and Vitest runs the two files at the same time, so each points
  260. // os.tmpdir() at its own folder; with a shared one, a run in one file deletes
  261. // the data file the other is about to read.
  262. const outerTmpdir = process.env.TMPDIR
  263. process.env.TMPDIR = root
  264. const savedEnv = { ...process.env }
  265. /** A snapshot of the staged data file and the uploads folder, as the engine takes them. */
  266. const FULL_TREE = {
  267. 'tmp/torqvoice-backup-staging/org1/data.json': '{"version":2,"data":{}}',
  268. 'srv/old-server/data/uploads/org1/logos/logo.png': 'png bytes',
  269. 'srv/old-server/data/uploads/org1/service-files/a.pdf': 'pdf bytes',
  270. }
  271. function addConnection(): void {
  272. h.connections.set(CONN, {
  273. id: CONN,
  274. organizationId: ORG,
  275. connectorId: 'local-folder',
  276. status: 'active',
  277. })
  278. }
  279. function addRun(over: Row = {}): string {
  280. const id = `run${++h.seq}`
  281. h.runs.set(id, {
  282. id,
  283. organizationId: ORG,
  284. kind: 'destination',
  285. connectionId: CONN,
  286. trigger: 'restore',
  287. status: 'queued',
  288. requestedBy: 'user1',
  289. reason: null,
  290. startedAt: null,
  291. finishedAt: null,
  292. snapshotId: SNAPSHOT,
  293. safetySnapshotId: null,
  294. filesNew: null,
  295. bytesAdded: null,
  296. totalBytes: null,
  297. error: null,
  298. createdAt: new Date(),
  299. ...over,
  300. })
  301. return id
  302. }
  303. beforeEach(() => {
  304. vi.clearAllMocks()
  305. h.runs.clear()
  306. h.connections.clear()
  307. h.seq = 0
  308. h.restoredTree = { ...FULL_TREE }
  309. h.restoreTarget = null
  310. h.seenAtImport = null
  311. h.importResult = { redactedSettings: ['ai.apiKey'] }
  312. h.importError = null
  313. h.events = []
  314. h.safetyError = null
  315. h.safetyAtRestore = null
  316. h.owner = { userId: 'owner1' }
  317. h.settings = {}
  318. h.repoDir = path.join(root, 'repo')
  319. rmSync(dataDir, { recursive: true, force: true })
  320. mkdirSync(dataDir, { recursive: true })
  321. process.env.DATA_ROOT = dataDir
  322. ;(globalThis as Row).__torqvoiceBackupLock = null
  323. ;(globalThis as Row).__torqvoiceBackupDraining = false
  324. ;(globalThis as Row).__torqvoiceRestoring?.clear()
  325. addConnection()
  326. })
  327. afterEach(() => {
  328. for (const key of Object.keys(process.env)) {
  329. if (!(key in savedEnv)) delete process.env[key]
  330. }
  331. Object.assign(process.env, savedEnv)
  332. })
  333. afterAll(() => {
  334. if (outerTmpdir === undefined) delete process.env.TMPDIR
  335. else process.env.TMPDIR = outerTmpdir
  336. rmSync(root, { recursive: true, force: true })
  337. })
  338. describe('listSnapshots', () => {
  339. it('lists the snapshots newest first, with size and whether files were included', async () => {
  340. vi.mocked(resticSnapshots).mockResolvedValueOnce([
  341. {
  342. id: OLDER,
  343. shortId: '9d6cb16e',
  344. time: '2026-10-28T02:00:00+01:00',
  345. hostname: 'torqvoice-oldorg',
  346. tags: ['org:oldorg', 'scheduled'],
  347. paths: ['/tmp/torqvoice-backup-staging/oldorg'],
  348. },
  349. {
  350. id: 'probe000',
  351. shortId: 'probe000',
  352. time: '2026-10-30T00:00:00+01:00',
  353. hostname: 'torqvoice-probe-org1',
  354. tags: ['torqvoice-probe'],
  355. paths: ['/tmp/torqvoice-backup-probe-x'],
  356. },
  357. {
  358. id: SNAPSHOT,
  359. shortId: '28891c8f',
  360. time: '2026-10-29T09:09:10.44+01:00',
  361. hostname: 'torqvoice-org1',
  362. tags: ['org:org1', 'manual'],
  363. paths: ['/tmp/torqvoice-backup-staging/org1', '/srv/data/uploads/org1'],
  364. summary: {
  365. filesNew: 2,
  366. filesChanged: 0,
  367. dataAdded: 3851,
  368. totalFilesProcessed: 2,
  369. totalBytesProcessed: 4096,
  370. },
  371. },
  372. ])
  373. const list = await listSnapshots(CONN)
  374. expect(list).toEqual([
  375. {
  376. id: SNAPSHOT,
  377. shortId: '28891c8f',
  378. time: '2026-10-29T09:09:10.44+01:00',
  379. bytes: 4096,
  380. filesNew: 2,
  381. dataAdded: 3851,
  382. files: true,
  383. trigger: 'manual',
  384. thisWorkshop: true,
  385. },
  386. // An earlier install's snapshot in the same repository restores too;
  387. // the connection check's probe does not show.
  388. {
  389. id: OLDER,
  390. shortId: '9d6cb16e',
  391. time: '2026-10-28T02:00:00+01:00',
  392. bytes: null,
  393. filesNew: null,
  394. dataAdded: null,
  395. files: false,
  396. trigger: 'scheduled',
  397. thisWorkshop: false,
  398. },
  399. ])
  400. expect(vi.mocked(resticSnapshots).mock.calls[0][1]).toEqual({ tags: [] })
  401. })
  402. })
  403. describe('queueRestore', () => {
  404. it('makes a restore run that names the snapshot', async () => {
  405. const { runId } = await queueRestore(CONN, SNAPSHOT)
  406. expect(h.runs.get(runId)).toMatchObject({
  407. organizationId: ORG,
  408. kind: 'destination',
  409. connectionId: CONN,
  410. trigger: 'restore',
  411. status: 'queued',
  412. snapshotId: SNAPSHOT,
  413. requestedBy: null,
  414. })
  415. })
  416. it('takes a run target and keeps who asked and why', async () => {
  417. const { runId } = await queueRestore({ kind: 'destination', connectionId: CONN }, SNAPSHOT, {
  418. requestedBy: 'user1',
  419. reason: 'Put back yesterday',
  420. })
  421. expect(h.runs.get(runId)).toMatchObject({
  422. kind: 'destination',
  423. connectionId: CONN,
  424. trigger: 'restore',
  425. snapshotId: SNAPSHOT,
  426. requestedBy: 'user1',
  427. reason: 'Put back yesterday',
  428. })
  429. })
  430. it('refuses while a backup or restore is queued or running, and refuses a bad id', async () => {
  431. addRun({ trigger: 'manual', status: 'running' })
  432. await expect(queueRestore(CONN, SNAPSHOT)).rejects.toBeInstanceOf(BackupBusyError)
  433. await expect(queueRestore(CONN, 'latest')).rejects.toThrow(/snapshot id/)
  434. })
  435. })
  436. describe('sweepAbandonedRestoreDirs', () => {
  437. it('removes the folders no running restore owns, and keeps the one holding the lock', async () => {
  438. const live = restoreDirFor(ORG, 'runlive')
  439. const dead = restoreDirFor(ORG, 'rundead')
  440. const stranger = path.join(dataDir, 'restore-tmp', 'not a run')
  441. for (const dir of [live, dead, stranger]) mkdirSync(dir, { recursive: true })
  442. ;(globalThis as Row).__torqvoiceBackupLock = { runId: 'runlive', since: Date.now() }
  443. await expect(sweepAbandonedRestoreDirs()).resolves.toBe(1)
  444. expect(existsSync(live)).toBe(true)
  445. expect(existsSync(dead)).toBe(false)
  446. expect(existsSync(stranger)).toBe(true)
  447. })
  448. it('is quiet when the folder does not exist yet', async () => {
  449. await expect(sweepAbandonedRestoreDirs()).resolves.toBe(0)
  450. })
  451. })
  452. describe('runRestore', () => {
  453. it('restores into a folder under the data root, imports from it, and removes it', async () => {
  454. const runId = addRun()
  455. const view = await runRestore(runId)
  456. expect(view.status).toBe('done')
  457. expect(view.snapshotId).toBe(SNAPSHOT)
  458. expect(view.filesNew).toBe(3)
  459. expect(view.bytesAdded).toBe(
  460. Object.values(FULL_TREE).reduce((sum, body) => sum + body.length, 0)
  461. )
  462. expect(view.error).toBeNull()
  463. // Where restic was pointed: a folder of this run's own under the data root.
  464. expect(h.restoreTarget).toBe(restoreDirFor(ORG, runId))
  465. expect(h.restoreTarget?.startsWith(path.join(dataDir, 'restore-tmp'))).toBe(true)
  466. expect(vi.mocked(resticRestore).mock.calls[0][1]).toEqual({
  467. snapshotId: SNAPSHOT,
  468. target: h.restoreTarget,
  469. })
  470. // The data file and the uploads folder, found wherever the old server kept them.
  471. const target = h.restoreTarget as string
  472. expect(vi.mocked(restoreFromDisk).mock.calls[0][0]).toEqual({
  473. organizationId: ORG,
  474. userId: 'user1',
  475. })
  476. expect(vi.mocked(restoreFromDisk).mock.calls[0][1]).toEqual({
  477. dataFile: path.join(target, 'tmp/torqvoice-backup-staging/org1/data.json'),
  478. filesDir: path.join(target, 'srv/old-server/data/uploads/org1'),
  479. })
  480. expect(h.seenAtImport).toEqual({ dataFile: true, filesDir: true, restoring: true })
  481. // Gone afterwards, and the lock is free again.
  482. expect(existsSync(target)).toBe(false)
  483. expect((globalThis as Row).__torqvoiceBackupLock).toBeNull()
  484. // The audit row is written after the import, which replaced the old log.
  485. expect(logAudit).toHaveBeenCalledTimes(1)
  486. expect(vi.mocked(logAudit).mock.calls[0][0]).toEqual({ userId: 'user1', organizationId: ORG })
  487. expect(vi.mocked(logAudit).mock.calls[0][1]).toMatchObject({
  488. action: 'integration.backup_restore',
  489. entity: 'BackupRun',
  490. entityId: runId,
  491. details: { key: 'backup_restored', params: { name: 'Test storage', snapshot: '28891c8f' } },
  492. metadata: { redactedSettings: ['ai.apiKey'] },
  493. })
  494. expect(h.log).toHaveBeenCalledWith('info', 'Backup restored', expect.any(Object))
  495. })
  496. it('restores a data-only snapshot with no uploads folder', async () => {
  497. h.restoredTree = {
  498. 'tmp/torqvoice-backup-staging/org1/data.json': '{"version":2,"data":{}}',
  499. }
  500. const runId = addRun()
  501. const view = await runRestore(runId)
  502. expect(view.status).toBe('done')
  503. expect(vi.mocked(restoreFromDisk).mock.calls[0][1].filesDir).toBeNull()
  504. })
  505. it('fails the run, keeps the message and removes the folder when the import throws', async () => {
  506. h.importError = new Error('Could not restore customers: boom')
  507. const runId = addRun()
  508. const view = await runRestore(runId)
  509. expect(view.status).toBe('failed')
  510. expect(view.error).toBe(
  511. `Could not restore customers: boom The safety copy taken first (${SAFETY.slice(0, 8)}) is among the restore points; restore it to undo.`
  512. )
  513. expect(existsSync(h.restoreTarget as string)).toBe(false)
  514. expect((globalThis as Row).__torqvoiceBackupLock).toBeNull()
  515. expect(logAudit).not.toHaveBeenCalled()
  516. })
  517. it('fails when the snapshot holds no data.json', async () => {
  518. h.restoredTree = { 'srv/data/uploads/org1/logos/logo.png': 'png' }
  519. const runId = addRun()
  520. const view = await runRestore(runId)
  521. expect(view.status).toBe('failed')
  522. expect(view.error).toBe('The snapshot holds no data.json Nothing in the workshop was changed.')
  523. expect(restoreFromDisk).not.toHaveBeenCalled()
  524. expect(existsSync(h.restoreTarget as string)).toBe(false)
  525. })
  526. it('refuses a snapshot the data disk cannot take, before restic writes a byte of it', async () => {
  527. vi.mocked(resticRestoreSize).mockResolvedValueOnce(Number.MAX_SAFE_INTEGER)
  528. const runId = addRun()
  529. const view = await runRestore(runId)
  530. expect(view.status).toBe('failed')
  531. expect(view.error).toMatch(/free; a restore keeps/)
  532. expect(view.error).toMatch(/Nothing in the workshop was changed\.$/)
  533. expect(vi.mocked(resticRestoreSize).mock.calls[0][1]).toBe(SNAPSHOT)
  534. expect(resticRestore).not.toHaveBeenCalled()
  535. expect(restoreFromDisk).not.toHaveBeenCalled()
  536. expect(isRestoring(ORG)).toBe(false)
  537. })
  538. it('refuses a data file larger than the server reads into memory, and removes the folder', async () => {
  539. h.restoredTree['tmp/torqvoice-backup-staging/org1/data.json'] =
  540. `{"version":2,"padding":"${'x'.repeat(1200)}","data":{}}`
  541. const runId = addRun()
  542. const view = await runRestore(runId)
  543. expect(view.status).toBe('failed')
  544. expect(view.error).toMatch(/more than this server reads into memory/)
  545. expect(view.error).toMatch(/Nothing in the workshop was changed\.$/)
  546. expect(restoreFromDisk).not.toHaveBeenCalled()
  547. expect(existsSync(h.restoreTarget as string)).toBe(false)
  548. expect(isRestoring(ORG)).toBe(false)
  549. })
  550. it('says why when restic refuses', async () => {
  551. const { ResticError } = await import('@/lib/backup/restic')
  552. vi.mocked(resticRestore).mockRejectedValueOnce(
  553. new ResticError({
  554. message: 'Fatal: wrong password or no key found',
  555. code: 12,
  556. stderr: '',
  557. command: 'restic restore',
  558. reason: 'wrong_password',
  559. })
  560. )
  561. const runId = addRun()
  562. const view = await runRestore(runId)
  563. expect(view.status).toBe('failed')
  564. expect(view.error).toMatch(/different password/)
  565. expect(restoreFromDisk).not.toHaveBeenCalled()
  566. })
  567. it('stays queued while a backup holds the lock', async () => {
  568. ;(globalThis as Row).__torqvoiceBackupLock = { runId: 'other', since: Date.now() }
  569. const runId = addRun()
  570. const view = await runRestore(runId)
  571. expect(view.status).toBe('queued')
  572. expect(view.error).toBeNull()
  573. expect(h.runs.get(runId)).toMatchObject({ status: 'queued', startedAt: null })
  574. expect(resticBackup).not.toHaveBeenCalled()
  575. expect(resticRestore).not.toHaveBeenCalled()
  576. expect(isRestoring(ORG)).toBe(false)
  577. // Somebody else's lock is left alone.
  578. expect((globalThis as Row).__torqvoiceBackupLock).toEqual({
  579. runId: 'other',
  580. since: expect.any(Number),
  581. })
  582. })
  583. it('takes a safety copy of the workshop before restic restores, and keeps its id on the row', async () => {
  584. const runId = addRun()
  585. const view = await runRestore(runId)
  586. expect(view.status).toBe('done')
  587. expect(view.safetySnapshotId).toBe(SAFETY)
  588. expect(h.runs.get(runId)?.safetySnapshotId).toBe(SAFETY)
  589. expect(h.events).toEqual(['safety', 'restore', 'import'])
  590. // The id was on the row before restic restored anything.
  591. expect(h.safetyAtRestore).toEqual([SAFETY])
  592. // A snapshot of this workshop into the restore's own repository, with
  593. // the secret keys in it even though the destination keeps data only.
  594. expect(vi.mocked(writeBackupData).mock.calls[0][0]).toBe(ORG)
  595. expect(vi.mocked(writeBackupData).mock.calls[0][1]).toMatchObject({ includeSecrets: true })
  596. const [opts, input] = vi.mocked(resticBackup).mock.calls[0]
  597. expect(opts.repository.repository).toBe(h.repoDir)
  598. expect(input).toMatchObject({ host: `torqvoice-${ORG}` })
  599. expect(input.tags).toEqual([`org:${ORG}`, 'pre-restore'])
  600. expect(vi.mocked(logAudit).mock.calls[0][1]).toMatchObject({
  601. metadata: { safetySnapshotId: SAFETY, bySupport: false },
  602. })
  603. })
  604. it('keeps the secret keys in the safety copy whatever the destination is set to keep', async () => {
  605. h.settings = { contents: 'data' }
  606. await runRestore(addRun())
  607. expect(vi.mocked(writeBackupData).mock.calls[0][1]).toMatchObject({ includeSecrets: true })
  608. })
  609. it('leaves the workshop untouched when the safety copy cannot be taken', async () => {
  610. h.safetyError = new Error('bucket gone')
  611. const runId = addRun()
  612. const view = await runRestore(runId)
  613. expect(view.status).toBe('failed')
  614. expect(view.error?.startsWith(SAFETY_FAILED)).toBe(true)
  615. expect(view.error).toContain('bucket gone')
  616. expect(view.safetySnapshotId).toBeNull()
  617. expect(resticRestore).not.toHaveBeenCalled()
  618. expect(restoreFromDisk).not.toHaveBeenCalled()
  619. expect(logAudit).not.toHaveBeenCalled()
  620. expect(notify).not.toHaveBeenCalled()
  621. expect(isRestoring(ORG)).toBe(false)
  622. expect((globalThis as Row).__torqvoiceBackupLock).toBeNull()
  623. })
  624. it('freezes the workshop while the restore runs and unfreezes it after', async () => {
  625. expect(isRestoring(ORG)).toBe(false)
  626. await runRestore(addRun())
  627. expect(h.seenAtImport?.restoring).toBe(true)
  628. expect(isRestoring(ORG)).toBe(false)
  629. })
  630. it('unfreezes the workshop after a failed restore too', async () => {
  631. h.importError = new Error('Could not restore customers: boom')
  632. const view = await runRestore(addRun())
  633. expect(view.status).toBe('failed')
  634. expect(h.seenAtImport?.restoring).toBe(true)
  635. expect(isRestoring(ORG)).toBe(false)
  636. })
  637. it("writes a restore support started in the name of the workshop's first owner", async () => {
  638. const runId = addRun({ requestedBy: null, reason: 'Customer asked by phone' })
  639. const view = await runRestore(runId)
  640. expect(view.status).toBe('done')
  641. expect(vi.mocked(db.organizationMember.findFirst).mock.calls[0][0]).toMatchObject({
  642. where: { organizationId: ORG, role: 'owner' },
  643. })
  644. expect(vi.mocked(restoreFromDisk).mock.calls[0][0]).toEqual({
  645. organizationId: ORG,
  646. userId: 'owner1',
  647. })
  648. expect(vi.mocked(logAudit).mock.calls[0][0]).toEqual({ userId: 'owner1', organizationId: ORG })
  649. expect(vi.mocked(logAudit).mock.calls[0][1]).toMatchObject({
  650. metadata: { bySupport: true, reason: 'Customer asked by phone' },
  651. })
  652. // The workshop hears that support put it back, and why.
  653. expect(notify).toHaveBeenCalledWith(
  654. expect.objectContaining({ organizationId: ORG, type: 'backup_restored' })
  655. )
  656. expect(vi.mocked(notify).mock.calls[0][0].message).toContain('backupAlerts.restoredBySupport')
  657. })
  658. it("a member's restore is written in that member's name, without looking up the owner", async () => {
  659. await runRestore(addRun({ requestedBy: 'user1' }))
  660. expect(db.organizationMember.findFirst).not.toHaveBeenCalled()
  661. expect(vi.mocked(restoreFromDisk).mock.calls[0][0]).toEqual({
  662. organizationId: ORG,
  663. userId: 'user1',
  664. })
  665. })
  666. it('fails a support restore of a workshop with no owner before anything is changed', async () => {
  667. h.owner = null
  668. const view = await runRestore(addRun({ requestedBy: null }))
  669. expect(view.status).toBe('failed')
  670. expect(view.error).toMatch(/no owner/)
  671. expect(resticBackup).not.toHaveBeenCalled()
  672. expect(restoreFromDisk).not.toHaveBeenCalled()
  673. })
  674. it('refuses a run that is not a restore, and leaves a finished one alone', async () => {
  675. const backup = addRun({ trigger: 'manual', snapshotId: null })
  676. expect((await runRestore(backup)).status).toBe('failed')
  677. const done = addRun({ status: 'done' })
  678. expect((await runRestore(done)).status).toBe('done')
  679. expect(resticRestore).not.toHaveBeenCalled()
  680. })
  681. })
  682. describe('locateRestored', () => {
  683. it('finds data.json and the uploads folder named after the workshop', async () => {
  684. const dir = path.join(root, 'locate')
  685. rmSync(dir, { recursive: true, force: true })
  686. for (const [rel, body] of Object.entries(FULL_TREE)) {
  687. mkdirSync(path.dirname(path.join(dir, rel)), { recursive: true })
  688. writeFileSync(path.join(dir, rel), body)
  689. }
  690. // Another workshop's folder on the same old server is not ours.
  691. mkdirSync(path.join(dir, 'srv/old-server/data/uploads/org2/logos'), { recursive: true })
  692. expect(await locateRestored(dir, ORG)).toEqual({
  693. dataFile: path.join(dir, 'tmp/torqvoice-backup-staging/org1/data.json'),
  694. filesDir: path.join(dir, 'srv/old-server/data/uploads/org1'),
  695. })
  696. // A snapshot an earlier install took has its uploads under the old id.
  697. expect(await locateRestored(dir, 'org3')).toEqual({
  698. dataFile: path.join(dir, 'tmp/torqvoice-backup-staging/org1/data.json'),
  699. filesDir: path.join(dir, 'srv/old-server/data/uploads/org1'),
  700. })
  701. expect(
  702. readFileSync(path.join(dir, 'tmp/torqvoice-backup-staging/org1/data.json'), 'utf8')
  703. ).toBe('{"version":2,"data":{}}')
  704. })
  705. it('names the restore folder after the workshop and the run, under the data root', () => {
  706. expect(restoreDirFor(ORG, 'run1')).toBe(path.join(dataDir, 'restore-tmp', 'org1-run1'))
  707. expect(() => restoreDirFor('../x', 'run1')).toThrow()
  708. })
  709. })