mirror of
https://github.com/outline/outline.git
synced 2026-06-13 03:14:59 +03:00
chore: Update JSON importer to use zip streaming (#12380)
* chore: Update JSON importer to use zip streaming, new importer flow * chore: Drop teamId from import urlId collision check and remove unused internal-id scaffolding urlId is globally unique on Document/Collection so the team scope was wrong. Also removes leftover internal-id generation in JSONAPIImportTask that was never used in task input/output. * Restore classes used upstream
This commit is contained in:
+100
-5
@@ -1,5 +1,9 @@
|
||||
import { z } from "zod";
|
||||
import type { IntegrationService, ProsemirrorDoc } from "./types";
|
||||
import type {
|
||||
IntegrationService,
|
||||
ProsemirrorData,
|
||||
ProsemirrorDoc,
|
||||
} from "./types";
|
||||
import {
|
||||
CollectionPermission,
|
||||
type ImportableIntegrationService,
|
||||
@@ -28,12 +32,20 @@ export type MarkdownImportInput = z.infer<
|
||||
typeof MarkdownImportInputItemSchema
|
||||
>[];
|
||||
|
||||
export const JSONImportInputItemSchema = BaseImportInputItemSchema.extend({
|
||||
externalId: z.string(),
|
||||
});
|
||||
|
||||
export type JSONImportInput = z.infer<typeof JSONImportInputItemSchema>[];
|
||||
|
||||
export type ImportInput<T extends ImportableIntegrationService> =
|
||||
T extends IntegrationService.Notion
|
||||
? NotionImportInput
|
||||
: T extends IntegrationService.Markdown
|
||||
? MarkdownImportInput
|
||||
: BaseImportInput;
|
||||
: T extends IntegrationService.JSON
|
||||
? JSONImportInput
|
||||
: BaseImportInput;
|
||||
|
||||
export const BaseImportTaskInputItemSchema = z.object({
|
||||
externalId: z.string(),
|
||||
@@ -82,6 +94,36 @@ export interface MarkdownImportScratch {
|
||||
manifest?: MarkdownAttachmentManifestItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Manifest entry describing a single attachment discovered during the JSON
|
||||
* zip bootstrap phase. `externalId` is the attachment's original id from the
|
||||
* export — used to rewrite `/api/attachments.redirect?id=<externalId>`
|
||||
* references in document/collection content into new redirect URLs that point
|
||||
* at the freshly created Attachment row (`id`).
|
||||
*/
|
||||
export const JSONAttachmentManifestItemSchema = z.object({
|
||||
id: z.uuid(),
|
||||
externalId: z.string(),
|
||||
name: z.string(),
|
||||
mimeType: z.string(),
|
||||
pathInZip: z.string(),
|
||||
});
|
||||
|
||||
export type JSONAttachmentManifestItem = z.infer<
|
||||
typeof JSONAttachmentManifestItemSchema
|
||||
>;
|
||||
|
||||
/**
|
||||
* JSON importer scratch state. `storageKey` is set at import creation (it's
|
||||
* the only durable handle on the uploaded zip). `manifest` is added by the
|
||||
* bootstrap phase so the completion phase can re-download the zip and create
|
||||
* Attachment rows without re-parsing the JSON files.
|
||||
*/
|
||||
export interface JSONImportScratch {
|
||||
storageKey: string;
|
||||
manifest?: JSONAttachmentManifestItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-importer scratch shape stored on `Import.scratch`. Holds cross-phase
|
||||
* state that the importer needs between bootstrap and completion but that
|
||||
@@ -89,7 +131,11 @@ export interface MarkdownImportScratch {
|
||||
* `Processed`.
|
||||
*/
|
||||
export type ImportScratch<T extends ImportableIntegrationService> =
|
||||
T extends IntegrationService.Markdown ? MarkdownImportScratch : never;
|
||||
T extends IntegrationService.Markdown
|
||||
? MarkdownImportScratch
|
||||
: T extends IntegrationService.JSON
|
||||
? JSONImportScratch
|
||||
: never;
|
||||
|
||||
/**
|
||||
* Per-page task input. Generated by the bootstrap task and consumed by
|
||||
@@ -124,22 +170,71 @@ export type MarkdownImportTaskInput = (
|
||||
| MarkdownPageImportTaskInputItem
|
||||
)[];
|
||||
|
||||
/**
|
||||
* Per-page task input for the JSON importer. Generated by the bootstrap task
|
||||
* once the zip has been parsed; consumed by subsequent JSONAPIImportTask runs.
|
||||
* `children` carries this document's direct descendants so each tree-depth
|
||||
* runs as its own task wave, preserving parent-before-child ordering during
|
||||
* persistence (createdAt of child tasks is strictly later than parents'). The
|
||||
* type is defined as a TypeScript interface rather than via z.infer because
|
||||
* it is only consumed internally — never validated at an API boundary — and
|
||||
* zod's recursive-schema ergonomics aren't worth the cost here.
|
||||
*/
|
||||
export interface JSONPageImportTaskInputItem {
|
||||
externalId: string;
|
||||
parentExternalId?: string;
|
||||
collectionExternalId?: string;
|
||||
title: string;
|
||||
urlId?: string;
|
||||
icon?: string | null;
|
||||
color?: string | null;
|
||||
data: ProsemirrorData;
|
||||
createdById?: string;
|
||||
createdByName?: string;
|
||||
createdByEmail?: string | null;
|
||||
createdAt?: string;
|
||||
updatedAt?: string;
|
||||
publishedAt?: string | null;
|
||||
/** Map of external attachment id → manifest entry id, scoped to this doc. */
|
||||
attachmentIdMap: Record<string, string>;
|
||||
children?: JSONPageImportTaskInputItem[];
|
||||
}
|
||||
|
||||
/**
|
||||
* JSON import task input — a bootstrap row carrying only the base placeholder
|
||||
* item (the zip's `storageKey` lives on `Import.scratch`), or a page row
|
||||
* carrying per-document content.
|
||||
*/
|
||||
export type JSONImportTaskInput = (
|
||||
| BaseImportTaskInput[number]
|
||||
| JSONPageImportTaskInputItem
|
||||
)[];
|
||||
|
||||
export type ImportTaskInput<T extends ImportableIntegrationService> =
|
||||
T extends IntegrationService.Notion
|
||||
? NotionImportTaskInput
|
||||
: T extends IntegrationService.Markdown
|
||||
? MarkdownImportTaskInput
|
||||
: BaseImportTaskInput;
|
||||
: T extends IntegrationService.JSON
|
||||
? JSONImportTaskInput
|
||||
: BaseImportTaskInput;
|
||||
|
||||
// No reason to be here except for co-location with import task input.
|
||||
export type ImportTaskOutput = {
|
||||
externalId: string;
|
||||
title: string;
|
||||
icon?: string;
|
||||
icon?: string | null;
|
||||
color?: string | null;
|
||||
urlId?: string;
|
||||
author?: string;
|
||||
/** Original author's id in the source system, used for user remapping. */
|
||||
createdById?: string;
|
||||
/** Original author's email in the source system, used for user remapping. */
|
||||
createdByEmail?: string | null;
|
||||
content: ProsemirrorDoc;
|
||||
createdAt?: Date;
|
||||
updatedAt?: Date;
|
||||
publishedAt?: Date | null;
|
||||
}[];
|
||||
|
||||
export const IssueSource = z.object({
|
||||
|
||||
+5
-1
@@ -167,16 +167,20 @@ export enum IntegrationService {
|
||||
Figma = "figma",
|
||||
Notion = "notion",
|
||||
Markdown = "markdown",
|
||||
JSON = "json",
|
||||
}
|
||||
|
||||
export type ImportableIntegrationService = Extract<
|
||||
IntegrationService,
|
||||
IntegrationService.Notion | IntegrationService.Markdown
|
||||
| IntegrationService.Notion
|
||||
| IntegrationService.Markdown
|
||||
| IntegrationService.JSON
|
||||
>;
|
||||
|
||||
export const ImportableIntegrationService = {
|
||||
Notion: IntegrationService.Notion,
|
||||
Markdown: IntegrationService.Markdown,
|
||||
JSON: IntegrationService.JSON,
|
||||
} as const;
|
||||
|
||||
export type IssueTrackerIntegrationService = Extract<
|
||||
|
||||
Reference in New Issue
Block a user