chore: Update JSON importer to use zip streaming (#12380)

* chore: Update JSON importer to use zip streaming, new importer flow

* chore: Drop teamId from import urlId collision check and remove unused internal-id scaffolding

urlId is globally unique on Document/Collection so the team scope was wrong.
Also removes leftover internal-id generation in JSONAPIImportTask that was
never used in task input/output.

* Restore classes used upstream
This commit is contained in:
Tom Moor
2026-05-25 17:03:02 -04:00
committed by GitHub
parent f9dc1a3983
commit ecafd5f32a
18 changed files with 1397 additions and 473 deletions
+100 -5
View File
@@ -1,5 +1,9 @@
import { z } from "zod";
import type { IntegrationService, ProsemirrorDoc } from "./types";
import type {
IntegrationService,
ProsemirrorData,
ProsemirrorDoc,
} from "./types";
import {
CollectionPermission,
type ImportableIntegrationService,
@@ -28,12 +32,20 @@ export type MarkdownImportInput = z.infer<
typeof MarkdownImportInputItemSchema
>[];
export const JSONImportInputItemSchema = BaseImportInputItemSchema.extend({
externalId: z.string(),
});
export type JSONImportInput = z.infer<typeof JSONImportInputItemSchema>[];
export type ImportInput<T extends ImportableIntegrationService> =
T extends IntegrationService.Notion
? NotionImportInput
: T extends IntegrationService.Markdown
? MarkdownImportInput
: BaseImportInput;
: T extends IntegrationService.JSON
? JSONImportInput
: BaseImportInput;
export const BaseImportTaskInputItemSchema = z.object({
externalId: z.string(),
@@ -82,6 +94,36 @@ export interface MarkdownImportScratch {
manifest?: MarkdownAttachmentManifestItem[];
}
/**
* Manifest entry describing a single attachment discovered during the JSON
* zip bootstrap phase. `externalId` is the attachment's original id from the
* export — used to rewrite `/api/attachments.redirect?id=<externalId>`
* references in document/collection content into new redirect URLs that point
* at the freshly created Attachment row (`id`).
*/
export const JSONAttachmentManifestItemSchema = z.object({
id: z.uuid(),
externalId: z.string(),
name: z.string(),
mimeType: z.string(),
pathInZip: z.string(),
});
export type JSONAttachmentManifestItem = z.infer<
typeof JSONAttachmentManifestItemSchema
>;
/**
* JSON importer scratch state. `storageKey` is set at import creation (it's
* the only durable handle on the uploaded zip). `manifest` is added by the
* bootstrap phase so the completion phase can re-download the zip and create
* Attachment rows without re-parsing the JSON files.
*/
export interface JSONImportScratch {
storageKey: string;
manifest?: JSONAttachmentManifestItem[];
}
/**
* Per-importer scratch shape stored on `Import.scratch`. Holds cross-phase
* state that the importer needs between bootstrap and completion but that
@@ -89,7 +131,11 @@ export interface MarkdownImportScratch {
* `Processed`.
*/
export type ImportScratch<T extends ImportableIntegrationService> =
T extends IntegrationService.Markdown ? MarkdownImportScratch : never;
T extends IntegrationService.Markdown
? MarkdownImportScratch
: T extends IntegrationService.JSON
? JSONImportScratch
: never;
/**
* Per-page task input. Generated by the bootstrap task and consumed by
@@ -124,22 +170,71 @@ export type MarkdownImportTaskInput = (
| MarkdownPageImportTaskInputItem
)[];
/**
* Per-page task input for the JSON importer. Generated by the bootstrap task
* once the zip has been parsed; consumed by subsequent JSONAPIImportTask runs.
* `children` carries this document's direct descendants so each tree-depth
* runs as its own task wave, preserving parent-before-child ordering during
* persistence (createdAt of child tasks is strictly later than parents'). The
* type is defined as a TypeScript interface rather than via z.infer because
* it is only consumed internally — never validated at an API boundary — and
* zod's recursive-schema ergonomics aren't worth the cost here.
*/
export interface JSONPageImportTaskInputItem {
externalId: string;
parentExternalId?: string;
collectionExternalId?: string;
title: string;
urlId?: string;
icon?: string | null;
color?: string | null;
data: ProsemirrorData;
createdById?: string;
createdByName?: string;
createdByEmail?: string | null;
createdAt?: string;
updatedAt?: string;
publishedAt?: string | null;
/** Map of external attachment id → manifest entry id, scoped to this doc. */
attachmentIdMap: Record<string, string>;
children?: JSONPageImportTaskInputItem[];
}
/**
* JSON import task input — a bootstrap row carrying only the base placeholder
* item (the zip's `storageKey` lives on `Import.scratch`), or a page row
* carrying per-document content.
*/
export type JSONImportTaskInput = (
| BaseImportTaskInput[number]
| JSONPageImportTaskInputItem
)[];
export type ImportTaskInput<T extends ImportableIntegrationService> =
T extends IntegrationService.Notion
? NotionImportTaskInput
: T extends IntegrationService.Markdown
? MarkdownImportTaskInput
: BaseImportTaskInput;
: T extends IntegrationService.JSON
? JSONImportTaskInput
: BaseImportTaskInput;
// No reason to be here except for co-location with import task input.
export type ImportTaskOutput = {
externalId: string;
title: string;
icon?: string;
icon?: string | null;
color?: string | null;
urlId?: string;
author?: string;
/** Original author's id in the source system, used for user remapping. */
createdById?: string;
/** Original author's email in the source system, used for user remapping. */
createdByEmail?: string | null;
content: ProsemirrorDoc;
createdAt?: Date;
updatedAt?: Date;
publishedAt?: Date | null;
}[];
export const IssueSource = z.object({
+5 -1
View File
@@ -167,16 +167,20 @@ export enum IntegrationService {
Figma = "figma",
Notion = "notion",
Markdown = "markdown",
JSON = "json",
}
export type ImportableIntegrationService = Extract<
IntegrationService,
IntegrationService.Notion | IntegrationService.Markdown
| IntegrationService.Notion
| IntegrationService.Markdown
| IntegrationService.JSON
>;
export const ImportableIntegrationService = {
Notion: IntegrationService.Notion,
Markdown: IntegrationService.Markdown,
JSON: IntegrationService.JSON,
} as const;
export type IssueTrackerIntegrationService = Extract<