import { randomUUID } from "node:crypto"; import fs from "fs-extra"; import JSZip from "jszip"; import tmp from "tmp"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { Attachment, Collection, Document, ImportTask, User, } from "@server/models"; import FileStorage from "@server/storage/files"; import { CollectionPermission, ImportTaskPhase, ImportTaskState, IntegrationService, } from "@shared/types"; import { buildAdmin, buildDocument, buildImport, buildTeam, buildUser, } from "@server/test/factories"; import JSONImportsProcessor from "../processors/JSONImportsProcessor"; import JSONAPIImportTask, { rewriteAttachmentReferences, } from "./JSONAPIImportTask"; // Fixed external IDs and email used across the user-mapping tests — these // are the values written into every freshly generated zip. const FIXTURE_USER_ID = "ccec260a-e060-4925-ade8-17cfabaf2cac"; const FIXTURE_USER_EMAIL = "hmac.devo@gmail.com"; interface BuiltZip { filePath: string; documentOneUrlId: string; documentTwoUrlId: string; cleanup: () => Promise; } /** * Builds a self-contained JSON export zip in a tmp file. Each call produces * fresh urlIds and document ids so concurrent tests (across files) never * collide on `urlId` uniqueness. The structure matches what ExportJSONTask * produces: a single collection JSON + metadata.json at the zip root, * documents carrying source user attribution, plus one referenced * attachment. */ async function buildJSONExportZip(): Promise { const collectionExternalId = randomUUID(); const collectionUrlId = randomUrlId(); const documentOneId = randomUUID(); const documentOneUrlId = randomUrlId(); const documentTwoId = randomUUID(); const documentTwoUrlId = randomUrlId(); const attachmentExternalId = randomUUID(); const attachmentKey = `uploads/${FIXTURE_USER_ID}/${attachmentExternalId}/pikachu.jpg`; const collectionExport = { collection: { id: collectionExternalId, urlId: collectionUrlId, name: "Test JSON", data: { type: "doc", content: [{ type: "paragraph" }] }, sort: { field: "index", direction: "asc" }, icon: "beaker", color: "#FF825C", permission: null, documentStructure: [ { id: documentOneId, title: "Document 1", children: [] }, { id: documentTwoId, title: "Document 2", children: [] }, ], }, documents: { [documentOneId]: { id: documentOneId, urlId: documentOneUrlId, title: "Document 1", icon: null, color: null, data: { type: "doc", content: [ { type: "paragraph", content: [{ type: "text", text: "Some random text" }], }, { type: "paragraph", content: [ { type: "text", text: "see doc two", marks: [ { type: "link", attrs: { href: `/doc/document-2-${documentTwoUrlId}`, title: null, }, }, ], }, ], }, { type: "paragraph", content: [ { type: "text", text: "see collection", marks: [ { type: "link", attrs: { href: `/collection/test-json-${collectionUrlId}`, title: null, }, }, ], }, ], }, { type: "paragraph", content: [ { type: "image", attrs: { src: `/api/attachments.redirect?id=${attachmentExternalId}`, alt: null, layoutClass: null, title: null, }, }, ], }, ], }, createdById: FIXTURE_USER_ID, createdByName: "hmac.devo", createdByEmail: FIXTURE_USER_EMAIL, createdAt: "2024-07-18T18:03:41.622Z", updatedAt: "2024-07-18T18:04:46.133Z", publishedAt: "2024-07-18T18:03:45.710Z", fullWidth: false, parentDocumentId: null, }, [documentTwoId]: { id: documentTwoId, urlId: documentTwoUrlId, title: "Document 2", icon: null, color: null, data: { type: "doc", content: [ { type: "paragraph", content: [{ type: "text", text: "Doc two body" }], }, ], }, createdById: FIXTURE_USER_ID, createdByName: "hmac.devo", createdByEmail: FIXTURE_USER_EMAIL, createdAt: "2024-07-18T18:03:41.622Z", updatedAt: "2024-07-18T18:04:46.133Z", publishedAt: "2024-07-18T18:03:45.710Z", fullWidth: false, parentDocumentId: null, }, }, attachments: { [attachmentExternalId]: { id: attachmentExternalId, documentId: documentOneId, contentType: "image/jpeg", name: "pikachu.jpg", size: 6, key: attachmentKey, }, }, }; const metadata = { exportVersion: 1, version: "0.78.0-0", createdAt: "2024-07-18T18:18:14.221Z", createdById: FIXTURE_USER_ID, createdByEmail: FIXTURE_USER_EMAIL, }; const zip = new JSZip(); zip.file("metadata.json", JSON.stringify(metadata)); zip.file("Test JSON.json", JSON.stringify(collectionExport)); zip.file(attachmentKey, Buffer.from("pixels")); const buffer = await zip.generateAsync({ type: "nodebuffer" }); const filePath: string = await new Promise((resolve, reject) => { tmp.file({ postfix: ".zip" }, (err, p) => (err ? reject(err) : resolve(p))); }); await fs.writeFile(filePath, buffer); return { filePath, documentOneUrlId, documentTwoUrlId, cleanup: async () => { await fs.rm(filePath, { force: true }).catch(() => {}); }, }; } function randomUrlId(): string { return Math.random().toString(36).slice(2, 12).padEnd(10, "x").slice(0, 10); } /** * Drives a JSON import end-to-end against an in-memory fixture: runs the * bootstrap task, every subsequent per-page wave, then invokes the * `imports.processed` handler so collections/documents/attachments land. */ async function runImport(opts: { teamId: string; createdById: string; zipPath: string; }): Promise<{ importId: string }> { vi.spyOn(FileStorage, "getFileHandle").mockResolvedValue({ path: opts.zipPath, cleanup: async () => {}, }); const importModel = await buildImport({ teamId: opts.teamId, createdById: opts.createdById, service: IntegrationService.JSON, integrationId: null, input: [ { externalId: randomUUID(), permission: CollectionPermission.Read }, ], scratch: { storageKey: "fixture-key" }, }); // Seed the bootstrap row that JSONImportsProcessor would have created. const placeholderExternalId = (importModel.input[0] as { externalId: string }) .externalId; const bootstrapTask = await ImportTask.create< ImportTask >({ state: ImportTaskState.Created, phase: ImportTaskPhase.Bootstrap, input: [{ externalId: placeholderExternalId }], importId: importModel.id, } as Parameters[0]); // Bull's schedule() would queue follow-on work. Drive it inline by // re-running perform() against every Created task in createdAt order. vi.spyOn(JSONAPIImportTask.prototype, "schedule").mockResolvedValue( undefined as never ); await new JSONAPIImportTask().perform({ importTaskId: bootstrapTask.id }); let remaining = await ImportTask.findOne>( { where: { state: ImportTaskState.Created, importId: importModel.id }, order: [["createdAt", "ASC"]], } ); while (remaining) { await new JSONAPIImportTask().perform({ importTaskId: remaining.id }); remaining = await ImportTask.findOne>({ where: { state: ImportTaskState.Created, importId: importModel.id }, order: [["createdAt", "ASC"]], }); } // Once all per-task work is done, APIImportTask transitions the Import to // Processed and fires the persistence pass via JSONImportsProcessor. await new JSONAPIImportTask().perform({ importTaskId: bootstrapTask.id, }); await new JSONImportsProcessor().perform({ name: "imports.processed", modelId: importModel.id, teamId: opts.teamId, actorId: opts.createdById, ip: "127.0.0.1", changes: { attributes: {}, previous: {} }, }); return { importId: importModel.id }; } describe("JSONAPIImportTask", () => { let zip: BuiltZip; beforeEach(async () => { zip = await buildJSONExportZip(); }); afterEach(async () => { await zip.cleanup(); }); it("imports collections, documents and attachments from the fixture", async () => { const admin = await buildAdmin(); const { importId } = await runImport({ teamId: admin.teamId, createdById: admin.id, zipPath: zip.filePath, }); const collections = await Collection.findAll({ where: { apiImportId: importId }, }); const documents = await Document.findAll({ where: { apiImportId: importId }, }); const attachments = await Attachment.findAll({ where: { teamId: admin.teamId }, }); expect(collections.length).toBe(1); expect(documents.length).toBe(2); expect(attachments.length).toBeGreaterThanOrEqual(1); }); it("rewrites internal document links to the new urlIds", async () => { const admin = await buildAdmin(); // Pre-create a document with the exported urlId so the importer is // forced to allocate a fresh urlId for Document 2. Without a collision // the original urlId would be preserved and link rewriting wouldn't be // exercised end-to-end. await buildDocument({ teamId: admin.teamId, userId: admin.id, urlId: zip.documentTwoUrlId, }); const { importId } = await runImport({ teamId: admin.teamId, createdById: admin.id, zipPath: zip.filePath, }); const documents = await Document.findAll({ where: { apiImportId: importId }, order: [["title", "ASC"]], }); expect(documents.length).toBe(2); const docOne = documents.find((d) => d.title === "Document 1"); const docTwo = documents.find((d) => d.title === "Document 2"); expect(docOne).toBeDefined(); expect(docTwo).toBeDefined(); expect(docTwo!.urlId).not.toBe(zip.documentTwoUrlId); const linkParagraph = docOne!.content?.content?.[1]; const linkText = linkParagraph?.content?.[0]; const linkMark = linkText?.marks?.find((m) => m.type === "link"); expect(linkMark?.attrs?.href).toBe(`/doc/document-2-${docTwo!.urlId}`); expect(linkMark?.attrs?.href).not.toContain(zip.documentTwoUrlId); }); it("rewrites internal collection links to slugged collection paths", async () => { const admin = await buildAdmin(); const { importId } = await runImport({ teamId: admin.teamId, createdById: admin.id, zipPath: zip.filePath, }); const collection = await Collection.findOne({ where: { apiImportId: importId }, rejectOnEmpty: true, }); const docOne = await Document.findOne({ where: { apiImportId: importId, title: "Document 1" }, rejectOnEmpty: true, }); const linkParagraph = docOne.content?.content?.[2]; const linkText = linkParagraph?.content?.[0]; const linkMark = linkText?.marks?.find((m) => m.type === "link"); expect(linkMark?.attrs?.href).toBe(collection.path); }); describe("user mapping", () => { it("maps createdById to an existing user by ID", async () => { let originalAuthor = await User.findByPk(FIXTURE_USER_ID); const teamId = originalAuthor?.teamId ?? (await buildTeam()).id; if (!originalAuthor) { originalAuthor = await buildUser({ id: FIXTURE_USER_ID, teamId }); } const admin = await buildAdmin({ teamId }); const { importId } = await runImport({ teamId, createdById: admin.id, zipPath: zip.filePath, }); const documents = await Document.findAll({ where: { apiImportId: importId }, }); expect(documents.length).toBe(2); for (const document of documents) { expect(document.createdById).toBe(originalAuthor.id); expect(document.lastModifiedById).toBe(originalAuthor.id); } }); it("falls back to email matching when ID does not match", async () => { const team = await buildTeam(); const originalAuthor = await buildUser({ teamId: team.id, email: FIXTURE_USER_EMAIL, }); const admin = await buildAdmin({ teamId: team.id }); const { importId } = await runImport({ teamId: team.id, createdById: admin.id, zipPath: zip.filePath, }); const documents = await Document.findAll({ where: { apiImportId: importId }, }); expect(documents.length).toBe(2); for (const document of documents) { expect(document.createdById).toBe(originalAuthor.id); expect(document.lastModifiedById).toBe(originalAuthor.id); } }); it("falls back to importing user when no match is found", async () => { const team = await buildTeam(); const admin = await buildAdmin({ teamId: team.id }); const { importId } = await runImport({ teamId: team.id, createdById: admin.id, zipPath: zip.filePath, }); const documents = await Document.findAll({ where: { apiImportId: importId }, }); expect(documents.length).toBe(2); for (const document of documents) { expect(document.createdById).toBe(admin.id); expect(document.lastModifiedById).toBe(admin.id); } }); it("does not match users from a different team", async () => { const team = await buildTeam(); const otherTeam = await buildTeam(); await buildUser({ teamId: otherTeam.id, email: FIXTURE_USER_EMAIL, }); const admin = await buildAdmin({ teamId: team.id }); const { importId } = await runImport({ teamId: team.id, createdById: admin.id, zipPath: zip.filePath, }); const documents = await Document.findAll({ where: { apiImportId: importId }, }); expect(documents.length).toBe(2); for (const document of documents) { expect(document.createdById).toBe(admin.id); } }); }); }); describe("rewriteAttachmentReferences", () => { it("rewrites image src to new attachment id", () => { const out = rewriteAttachmentReferences( { type: "doc", content: [ { type: "paragraph", content: [ { type: "image", attrs: { src: "/api/attachments.redirect?id=external-1", alt: null, }, }, ], }, ], }, { "external-1": "new-1" } ); const image = out.content?.[0].content?.[0]; expect(image?.attrs?.src).toBe("/api/attachments.redirect?id=new-1"); }); it("rewrites attachment node href and id together", () => { const out = rewriteAttachmentReferences( { type: "doc", content: [ { type: "attachment", attrs: { id: "external-2", href: "/api/attachments.redirect?id=external-2", title: "a.pdf", }, }, ], }, { "external-2": "new-2" } ); const attachment = out.content?.[0]; expect(attachment?.attrs?.href).toBe("/api/attachments.redirect?id=new-2"); expect(attachment?.attrs?.id).toBe("new-2"); }); it("leaves unknown references untouched", () => { const out = rewriteAttachmentReferences( { type: "doc", content: [ { type: "paragraph", content: [ { type: "image", attrs: { src: "/api/attachments.redirect?id=does-not-exist", }, }, ], }, ], }, { "external-1": "new-1" } ); const image = out.content?.[0].content?.[0]; expect(image?.attrs?.src).toBe( "/api/attachments.redirect?id=does-not-exist" ); }); });