API importer for Notion (#8710)

.env.sample

@@ -147,6 +147,10 @@ DISCORD_SERVER_ID=
 # DISCORD_SERVER_ID and DISCORD_SERVER_ROLES must be set together.
 DISCORD_SERVER_ROLES=
 
+# –––––––––––––– IMPORTS ––––––––––––––
+NOTION_CLIENT_ID=
+NOTION_CLIENT_SECRET=
+
 # –––––––––––––––– OPTIONAL ––––––––––––––––
 
 # Base64 encoded private key and certificate for HTTPS termination. This is only

app/components/Dialogs.tsx

@@ -23,7 +23,10 @@ function Dialogs() {
           key={id}
           isOpen={modal.isOpen}
           fullscreen={modal.fullscreen ?? false}
-          onRequestClose={() => dialogs.closeModal(id)}
+          onRequestClose={() => {
+            modal.onClose?.();
+            dialogs.closeModal(id);
+          }}
           title={modal.title}
           style={modal.style}
         >

app/components/WebsocketProvider.tsx

@@ -6,7 +6,11 @@ import * as React from "react";
 import { withTranslation, WithTranslation } from "react-i18next";
 import { io, Socket } from "socket.io-client";
 import { toast } from "sonner";
-import { FileOperationState, FileOperationType } from "@shared/types";
+import {
+  FileOperationState,
+  FileOperationType,
+  ImportState,
+} from "@shared/types";
 import RootStore from "~/stores/RootStore";
 import Collection from "~/models/Collection";
 import Comment from "~/models/Comment";
@@ -15,6 +19,7 @@ import FileOperation from "~/models/FileOperation";
 import Group from "~/models/Group";
 import GroupMembership from "~/models/GroupMembership";
 import GroupUser from "~/models/GroupUser";
+import Import from "~/models/Import";
 import Membership from "~/models/Membership";
 import Notification from "~/models/Notification";
 import Pin from "~/models/Pin";
@@ -100,6 +105,7 @@ class WebsocketProvider extends React.Component<Props> {
       subscriptions,
       fileOperations,
       notifications,
+      imports,
     } = this.props;
 
     const currentUserId = auth?.user?.id;
@@ -620,6 +626,23 @@ class WebsocketProvider extends React.Component<Props> {
       }
     );
 
+    this.socket.on("imports.create", (event: PartialExcept<Import, "id">) => {
+      imports.add(event);
+    });
+
+    this.socket.on("imports.update", (event: PartialExcept<Import, "id">) => {
+      imports.add(event);
+
+      if (
+        event.state === ImportState.Completed &&
+        event.createdBy?.id === auth.user?.id
+      ) {
+        toast.success(event.name, {
+          description: this.props.t("Your import completed"),
+        });
+      }
+    });
+
     this.socket.on(
       "subscriptions.create",
       (event: PartialExcept<Subscription, "id">) => {

app/hooks/usePolicy.ts

@@ -18,6 +18,7 @@ export default function usePolicy(entity?: string | Model | null) {
       ? entity
       : entity.id
     : "";
+  const policy = policies.get(entityId);
 
   React.useEffect(() => {
     if (
@@ -28,11 +29,11 @@ export default function usePolicy(entity?: string | Model | null) {
     ) {
       // The policy for this model is missing and we have an authenticated session, attempt to
       // reload relationships for this model.
-      if (!policies.get(entity.id) && user) {
+      if (!policy && user) {
         void entity.loadRelations();
       }
     }
-  }, [policies, user, entity]);
+  }, [policies, policy, user, entity]);
 
   return policies.abilities(entityId);
 }

app/menus/ImportMenu.tsx

@@ -0,0 +1,62 @@
+import { observer } from "mobx-react";
+import { CrossIcon, TrashIcon } from "outline-icons";
+import * as React from "react";
+import { useTranslation } from "react-i18next";
+import { useMenuState } from "reakit/Menu";
+import Import from "~/models/Import";
+import ContextMenu from "~/components/ContextMenu";
+import OverflowMenuButton from "~/components/ContextMenu/OverflowMenuButton";
+import Template from "~/components/ContextMenu/Template";
+import usePolicy from "~/hooks/usePolicy";
+import { MenuItem } from "~/types";
+
+type Props = {
+  /** Import to which actions will be applied. */
+  importModel: Import;
+  /** Callback to handle import cancellation. */
+  onCancel: () => Promise<void>;
+  /** Callback to handle import deletion. */
+  onDelete: () => Promise<void>;
+};
+
+export const ImportMenu = observer(
+  ({ importModel, onCancel, onDelete }: Props) => {
+    const { t } = useTranslation();
+    const can = usePolicy(importModel);
+    const menu = useMenuState({
+      modal: true,
+    });
+
+    const items = React.useMemo(
+      () =>
+        [
+          {
+            type: "button",
+            title: t("Cancel"),
+            visible: can.cancel,
+            icon: <CrossIcon />,
+            dangerous: true,
+            onClick: onCancel,
+          },
+          {
+            type: "button",
+            title: t("Delete"),
+            visible: can.delete,
+            icon: <TrashIcon />,
+            dangerous: true,
+            onClick: onDelete,
+          },
+        ] satisfies MenuItem[],
+      [t, can.delete, can.cancel, onCancel, onDelete]
+    );
+
+    return (
+      <>
+        <OverflowMenuButton aria-label={t("Show menu")} {...menu} />
+        <ContextMenu {...menu} aria-label={t("Import menu options")}>
+          <Template {...menu} items={items} />
+        </ContextMenu>
+      </>
+    );
+  }
+);

app/models/Import.ts

@@ -0,0 +1,54 @@
+import { observable } from "mobx";
+import { ImportableIntegrationService, ImportState } from "@shared/types";
+import ImportsStore from "~/stores/ImportsStore";
+import User from "./User";
+import Model from "./base/Model";
+import Field from "./decorators/Field";
+import { AfterChange } from "./decorators/Lifecycle";
+import Relation from "./decorators/Relation";
+
+class Import extends Model {
+  static modelName = "Import";
+
+  store: ImportsStore;
+
+  /** The name of the import. */
+  name: string;
+
+  /** The current state of the import. */
+  @Field
+  @observable
+  state: ImportState;
+
+  /** The external service from which the import is created. */
+  service: ImportableIntegrationService;
+
+  /** The count of documents created in the import. */
+  @observable
+  documentCount: number;
+
+  /** The user who created the import. */
+  @Relation(() => User, {})
+  createdBy: User;
+
+  /** The ID of the user who created the import. */
+  createdById: string;
+
+  /**
+   * Cancel the import – this will stop the import process and mark it as
+   * cancelled at the first opportunity.
+   */
+  cancel = async () => this.store.cancel(this);
+
+  // hooks
+
+  @AfterChange
+  static removePolicies(model: Import, previousAttributes: Partial<Import>) {
+    if (previousAttributes.state && previousAttributes.state !== model.state) {
+      const { policies } = model.store.rootStore;
+      policies.remove(model.id);
+    }
+  }
+}
+
+export default Import;

app/models/base/Model.ts

@@ -43,12 +43,6 @@ export default abstract class Model {
     this: Model,
     options: { withoutPolicies?: boolean } = {}
   ): Promise<any> {
-    const relations = getRelationsForModelClass(
-      this.constructor as typeof Model
-    );
-    if (!relations) {
-      return;
-    }
     // this is to ensure that multiple loads don’t happen in parallel
     if (this.loadingRelations) {
       return this.loadingRelations;
@@ -56,14 +50,20 @@ export default abstract class Model {
 
     const promises = [];
 
-    for (const properties of relations.values()) {
-      const store = this.store.rootStore.getStoreForModelName(
-        properties.relationClassResolver().modelName
-      );
-      if ("fetch" in store) {
-        const id = this[properties.idKey];
-        if (id) {
-          promises.push(store.fetch(id as string));
+    const relations = getRelationsForModelClass(
+      this.constructor as typeof Model
+    );
+
+    if (relations) {
+      for (const properties of relations.values()) {
+        const store = this.store.rootStore.getStoreForModelName(
+          properties.relationClassResolver().modelName
+        );
+        if ("fetch" in store) {
+          const id = this[properties.idKey];
+          if (id) {
+            promises.push(store.fetch(id as string));
+          }
         }
       }
     }

app/scenes/Settings/Import.tsx

@@ -1,10 +1,13 @@
+import orderBy from "lodash/orderBy";
 import { observer } from "mobx-react";
 import { NewDocumentIcon } from "outline-icons";
 import * as React from "react";
 import { useTranslation, Trans } from "react-i18next";
+import { Pagination } from "@shared/constants";
 import { FileOperationType } from "@shared/types";
 import { cdnPath } from "@shared/utils/urls";
 import FileOperation from "~/models/FileOperation";
+import ImportModel from "~/models/Import";
 import Button from "~/components/Button";
 import Heading from "~/components/Heading";
 import MarkdownIcon from "~/components/Icons/MarkdownIcon";
@@ -15,16 +18,146 @@ import Scene from "~/components/Scene";
 import Text from "~/components/Text";
 import env from "~/env";
 import useStores from "~/hooks/useStores";
+import { Hook, PluginManager } from "~/utils/PluginManager";
 import FileOperationListItem from "./components/FileOperationListItem";
 import ImportJSONDialog from "./components/ImportJSONDialog";
+import { ImportListItem } from "./components/ImportListItem";
 import ImportMarkdownDialog from "./components/ImportMarkdownDialog";
-import ImportNotionDialog from "./components/ImportNotionDialog";
+
+type Config = {
+  /** The title of the import. */
+  title: string;
+  /** The auxiliary descriptive text of the import. */
+  subtitle: string;
+  /** An icon to denote the kind of import. */
+  icon: React.ReactElement;
+  /** Trigger for the import. */
+  action: React.ReactElement;
+};
+
+function useImportsConfig() {
+  const { t } = useTranslation();
+  const { dialogs } = useStores();
+  const appName = env.APP_NAME;
+
+  return React.useMemo(() => {
+    const items: Config[] = [
+      {
+        title: t("Markdown"),
+        subtitle: t(
+          "Import a zip file of Markdown documents (exported from version 0.67.0 or earlier)"
+        ),
+        icon: <MarkdownIcon size={28} />,
+        action: (
+          <Button
+            type="submit"
+            onClick={() => {
+              dialogs.openModal({
+                title: t("Import data"),
+                content: <ImportMarkdownDialog />,
+              });
+            }}
+            neutral
+          >
+            {t("Import")}…
+          </Button>
+        ),
+      },
+      {
+        title: "JSON",
+        subtitle: t(
+          "Import a JSON data file exported from another {{ appName }} instance",
+          {
+            appName,
+          }
+        ),
+        icon: <OutlineIcon size={28} cover />,
+        action: (
+          <Button
+            type="submit"
+            onClick={() => {
+              dialogs.openModal({
+                title: t("Import data"),
+                content: <ImportJSONDialog />,
+              });
+            }}
+            neutral
+          >
+            {t("Import")}…
+          </Button>
+        ),
+      },
+    ];
+
+    PluginManager.getHooks(Hook.Imports).forEach((plugin) => {
+      items.push({ ...plugin.value });
+    });
+
+    items.push({
+      title: "Confluence",
+      subtitle: t("Import pages from a Confluence instance"),
+      icon: <img src={cdnPath("/images/confluence.png")} width={28} />,
+      action: (
+        <Button type="submit" disabled neutral>
+          {t("Enterprise")}
+        </Button>
+      ),
+    });
+
+    return items;
+  }, [t, dialogs, appName]);
+}
 
 function Import() {
   const { t } = useTranslation();
-  const { dialogs, fileOperations } = useStores();
+  const { fileOperations, imports } = useStores();
+  const configs = useImportsConfig();
   const appName = env.APP_NAME;
 
+  const [, setForceRender] = React.useState(0);
+  const offset = React.useMemo(() => ({ imports: 0, fileOperations: 0 }), []);
+
+  const fetchImports = React.useCallback(async () => {
+    const [importsArr, fileOpsArr] = await Promise.all([
+      imports.fetchPage({
+        offset: offset.imports,
+        limit: Pagination.defaultLimit,
+      }),
+      fileOperations.fetchPage({
+        type: FileOperationType.Import,
+        offset: offset.fileOperations,
+        limit: Pagination.defaultLimit,
+      }),
+    ]);
+
+    const pageImports = orderBy(
+      [...importsArr, ...fileOpsArr],
+      "createdAt",
+      "desc"
+    ).slice(0, Pagination.defaultLimit);
+
+    const apiImportsCount = pageImports.filter(
+      (item) => item instanceof ImportModel
+    ).length;
+
+    offset.imports += apiImportsCount;
+    offset.fileOperations += pageImports.length - apiImportsCount;
+
+    // needed to re-render after mobx store and offset is updated
+    setForceRender((s) => ++s);
+
+    return pageImports;
+  }, [imports, fileOperations, offset]);
+
+  const allImports = orderBy(
+    [
+      ...imports.orderedData,
+      ...fileOperations.filter({ type: FileOperationType.Import }),
+    ],
+    "createdAt",
+    "desc"
+  ).slice(0, offset.imports + offset.fileOperations);
+
   return (
     <Scene title={t("Import")} icon={<NewDocumentIcon />}>
       <Heading>{t("Import")}</Heading>
@@ -38,100 +171,33 @@ function Import() {
       </Text>
 
       <div>
-        <Item
-          border={false}
-          image={<MarkdownIcon size={28} />}
-          title={t("Markdown")}
-          subtitle={t(
-            "Import a zip file of Markdown documents (exported from version 0.67.0 or earlier)"
-          )}
-          actions={
-            <Button
-              type="submit"
-              onClick={() => {
-                dialogs.openModal({
-                  title: t("Import data"),
-                  content: <ImportMarkdownDialog />,
-                });
-              }}
-              neutral
-            >
-              {t("Import")}…
-            </Button>
-          }
-        />
-        <Item
-          border={false}
-          image={<OutlineIcon size={28} cover />}
-          title="JSON"
-          subtitle={t(
-            "Import a JSON data file exported from another {{ appName }} instance",
-            {
-              appName,
-            }
-          )}
-          actions={
-            <Button
-              type="submit"
-              onClick={() => {
-                dialogs.openModal({
-                  title: t("Import data"),
-                  content: <ImportJSONDialog />,
-                });
-              }}
-              neutral
-            >
-              {t("Import")}…
-            </Button>
-          }
-        />
-        <Item
-          border={false}
-          image={<img src={cdnPath("/images/notion.png")} width={28} />}
-          title="Notion"
-          subtitle={t("Import pages exported from Notion")}
-          actions={
-            <Button
-              type="submit"
-              onClick={() => {
-                dialogs.openModal({
-                  title: t("Import data"),
-                  content: <ImportNotionDialog />,
-                });
-              }}
-              neutral
-            >
-              {t("Import")}…
-            </Button>
-          }
-        />
-        <Item
-          border={false}
-          image={<img src={cdnPath("/images/confluence.png")} width={28} />}
-          title="Confluence"
-          subtitle={t("Import pages from a Confluence instance")}
-          actions={
-            <Button type="submit" disabled neutral>
-              {t("Enterprise")}
-            </Button>
-          }
-        />
+        {configs.map((config) => (
+          <Item
+            key={config.title}
+            title={config.title}
+            subtitle={config.subtitle}
+            image={config.icon}
+            actions={config.action}
+            border={false}
+          />
+        ))}
       </div>
       <br />
       <PaginatedList
-        items={fileOperations.imports}
-        fetch={fileOperations.fetchPage}
-        options={{
-          type: FileOperationType.Import,
-        }}
+        items={allImports}
+        fetch={fetchImports}
         heading={
           <h2>
             <Trans>Recent imports</Trans>
           </h2>
         }
-        renderItem={(item: FileOperation) => (
-          <FileOperationListItem key={item.id} fileOperation={item} />
-        )}
+        renderItem={(item: ImportModel | FileOperation) =>
+          item instanceof ImportModel ? (
+            <ImportListItem key={item.id} importModel={item} />
+          ) : (
+            <FileOperationListItem key={item.id} fileOperation={item} />
+          )
+        }
       />
     </Scene>
   );

app/scenes/Settings/components/ImportListItem.tsx

@@ -0,0 +1,148 @@
+import capitalize from "lodash/capitalize";
+import { observer } from "mobx-react";
+import { CrossIcon, DoneIcon, WarningIcon } from "outline-icons";
+import React from "react";
+import { useTranslation } from "react-i18next";
+import { toast } from "sonner";
+import { useTheme } from "styled-components";
+import { ImportState } from "@shared/types";
+import Import from "~/models/Import";
+import { Action } from "~/components/Actions";
+import ConfirmationDialog from "~/components/ConfirmationDialog";
+import ListItem from "~/components/List/Item";
+import Spinner from "~/components/Spinner";
+import Time from "~/components/Time";
+import useCurrentUser from "~/hooks/useCurrentUser";
+import useStores from "~/hooks/useStores";
+import { ImportMenu } from "~/menus/ImportMenu";
+
+type Props = {
+  /** Import that's displayed as list item. */
+  importModel: Import;
+};
+
+export const ImportListItem = observer(({ importModel }: Props) => {
+  const { t } = useTranslation();
+  const { dialogs } = useStores();
+  const user = useCurrentUser();
+  const theme = useTheme();
+  const showProgress =
+    importModel.state !== ImportState.Canceled &&
+    importModel.state !== ImportState.Errored;
+
+  const stateMap = React.useMemo(
+    () => ({
+      [ImportState.Created]: t("Processing"),
+      [ImportState.InProgress]: t("Processing"),
+      [ImportState.Processed]: t("Processing"),
+      [ImportState.Completed]: t("Completed"),
+      [ImportState.Errored]: t("Failed"),
+      [ImportState.Canceled]: t("Canceled"),
+    }),
+    [t]
+  );
+
+  const iconMap = React.useMemo(
+    () => ({
+      [ImportState.Created]: <Spinner />,
+      [ImportState.InProgress]: <Spinner />,
+      [ImportState.Processed]: <Spinner />,
+      [ImportState.Completed]: <DoneIcon color={theme.accent} />,
+      [ImportState.Errored]: <WarningIcon color={theme.danger} />,
+      [ImportState.Canceled]: <CrossIcon color={theme.textTertiary} />,
+    }),
+    [theme]
+  );
+
+  const handleCancel = React.useCallback(async () => {
+    const onCancel = async () => {
+      try {
+        await importModel.cancel();
+        toast.success(t("Import canceled"));
+      } catch (err) {
+        toast.error(err.message);
+      }
+    };
+
+    dialogs.openModal({
+      title: t("Are you sure you want to cancel this import?"),
+      content: (
+        <ConfirmationDialog
+          onSubmit={onCancel}
+          submitText={t("Cancel")}
+          savingText={`${t("Canceling")}…`}
+          danger
+        >
+          {t(
+            "Canceling this import will discard any progress made. This cannot be undone."
+          )}
+        </ConfirmationDialog>
+      ),
+    });
+  }, [t, dialogs, importModel]);
+
+  const handleDelete = React.useCallback(async () => {
+    const onDelete = async () => {
+      try {
+        await importModel.delete();
+        toast.success(t("Import deleted"));
+      } catch (err) {
+        toast.error(err.message);
+      }
+    };
+
+    dialogs.openModal({
+      title: t("Are you sure you want to delete this import?"),
+      content: (
+        <ConfirmationDialog
+          onSubmit={onDelete}
+          savingText={`${t("Deleting")}…`}
+          danger
+        >
+          {t(
+            "Deleting this import will also delete all collections and documents that were created from it. This cannot be undone."
+          )}
+        </ConfirmationDialog>
+      ),
+    });
+  }, [t, dialogs, importModel]);
+
+  return (
+    <ListItem
+      title={importModel.name}
+      image={iconMap[importModel.state]}
+      subtitle={
+        <>
+          {stateMap[importModel.state]}&nbsp;•&nbsp;
+          {t(`{{userName}} requested`, {
+            userName:
+              user.id === importModel.createdBy.id
+                ? t("You")
+                : importModel.createdBy.name,
+          })}
+          &nbsp;
+          <Time dateTime={importModel.createdAt} addSuffix shorten />
+          &nbsp;•&nbsp;
+          {capitalize(importModel.service)}
+          {showProgress && (
+            <>
+              &nbsp;•&nbsp;
+              {t("{{ count }} document imported", {
+                count: importModel.documentCount,
+              })}
+            </>
+          )}
+        </>
+      }
+      actions={
+        <Action>
+          <ImportMenu
+            importModel={importModel}
+            onCancel={handleCancel}
+            onDelete={handleDelete}
+          />
+        </Action>
+      }
+    />
+  );
+});

app/stores/DialogsStore.ts

@@ -8,6 +8,7 @@ type DialogDefinition = {
   isOpen: boolean;
   fullscreen?: boolean;
   style?: React.CSSProperties;
+  onClose?: () => void;
 };
 
 export default class DialogsStore {
@@ -50,6 +51,7 @@ export default class DialogsStore {
     fullscreen,
     replace,
     style,
+    onClose,
   }: {
     id?: string;
     title: string;
@@ -57,6 +59,7 @@ export default class DialogsStore {
     content: React.ReactNode;
     style?: React.CSSProperties;
     replace?: boolean;
+    onClose?: () => void;
   }) => {
     setTimeout(
       action(() => {
@@ -70,6 +73,7 @@ export default class DialogsStore {
           fullscreen,
           style,
           isOpen: true,
+          onClose,
         });
       }),
       0

app/stores/ImportsStore.ts

@@ -0,0 +1,25 @@
+import invariant from "invariant";
+import { action, runInAction } from "mobx";
+import Import from "~/models/Import";
+import { client } from "~/utils/ApiClient";
+import RootStore from "./RootStore";
+import Store from "./base/Store";
+
+export default class ImportsStore extends Store<Import> {
+  constructor(rootStore: RootStore) {
+    super(rootStore, Import);
+  }
+
+  @action
+  cancel = async (importModel: Import) => {
+    const res = await client.post("/imports.cancel", {
+      id: importModel.id,
+    });
+
+    runInAction("Import#cancel", () => {
+      invariant(res?.data, "Data should be available");
+      importModel.updateData(res.data);
+      this.addPolicies(res.policies);
+    });
+  };
+}

app/stores/RootStore.ts

@@ -14,6 +14,7 @@ import FileOperationsStore from "./FileOperationsStore";
 import GroupMembershipsStore from "./GroupMembershipsStore";
 import GroupUsersStore from "./GroupUsersStore";
 import GroupsStore from "./GroupsStore";
+import ImportsStore from "./ImportsStore";
 import IntegrationsStore from "./IntegrationsStore";
 import MembershipsStore from "./MembershipsStore";
 import NotificationsStore from "./NotificationsStore";
@@ -43,6 +44,7 @@ export default class RootStore {
   events: EventsStore;
   groups: GroupsStore;
   groupUsers: GroupUsersStore;
+  imports: ImportsStore;
   integrations: IntegrationsStore;
   memberships: MembershipsStore;
   notifications: NotificationsStore;
@@ -72,6 +74,7 @@ export default class RootStore {
     this.registerStore(EventsStore);
     this.registerStore(GroupsStore);
     this.registerStore(GroupUsersStore);
+    this.registerStore(ImportsStore);
     this.registerStore(IntegrationsStore);
     this.registerStore(MembershipsStore);
     this.registerStore(NotificationsStore);

app/utils/ApiClient.ts

@@ -17,6 +17,7 @@ import {
   RateLimitExceededError,
   RequestError,
   ServiceUnavailableError,
+  UnprocessableEntityError,
   UpdateRequiredError,
 } from "./errors";
 
@@ -214,6 +215,10 @@ class ApiClient {
       throw new ServiceUnavailableError(error.message);
     }
 
+    if (response.status === 422) {
+      throw new UnprocessableEntityError(error.message);
+    }
+
     if (response.status === 429) {
       throw new RateLimitExceededError(
         `Too many requests, try again in a minute.`

app/utils/PluginManager.ts

@@ -12,6 +12,7 @@ import isCloudHosted from "./isCloudHosted";
  */
 export enum Hook {
   Settings = "settings",
+  Imports = "imports",
   Icon = "icon",
 }
 
@@ -31,6 +32,16 @@ type PluginValueMap = {
     /** Whether the plugin is enabled in the current context. */
     enabled?: (team: Team, user: User) => boolean;
   };
+  [Hook.Imports]: {
+    /** The title of the import. */
+    title: string;
+    /** The auxiliary descriptive text of the import. */
+    subtitle: string;
+    /** An icon to denote the kind of import. */
+    icon: React.ReactElement;
+    /** Trigger for the import. */
+    action: React.ReactElement;
+  };
   [Hook.Icon]: React.ElementType;
 };
 

app/utils/errors.ts

@@ -16,6 +16,8 @@ export class ServiceUnavailableError extends ExtendableError {}
 
 export class BadGatewayError extends ExtendableError {}
 
+export class UnprocessableEntityError extends ExtendableError {}
+
 export class RateLimitExceededError extends ExtendableError {}
 
 export class RequestError extends ExtendableError {}

package.json

@@ -79,6 +79,7 @@
     "@hocuspocus/server": "1.1.2",
     "@joplin/turndown-plugin-gfm": "^1.0.49",
     "@juggle/resize-observer": "^3.4.0",
+    "@notionhq/client": "^2.2.16",
     "@octokit/auth-app": "^6.1.3",
     "@outlinewiki/koa-passport": "^4.2.1",
     "@outlinewiki/passport-azure-ad-oauth2": "^0.1.0",
@@ -95,6 +96,7 @@
     "@types/sanitize-filename": "^1.6.3",
     "@vitejs/plugin-react": "^3.1.0",
     "addressparser": "^1.0.1",
+    "async-sema": "^3.1.1",
     "autotrack": "^2.4.1",
     "babel-plugin-styled-components": "^2.1.4",
     "babel-plugin-transform-class-properties": "^6.24.1",

plugins/notion/client/Imports.tsx

@@ -0,0 +1,88 @@
+import { observer } from "mobx-react";
+import React from "react";
+import { useTranslation } from "react-i18next";
+import { useHistory, useLocation } from "react-router-dom";
+import { toast } from "sonner";
+import env from "@shared/env";
+import { IntegrationService } from "@shared/types";
+import Button from "~/components/Button";
+import useCurrentTeam from "~/hooks/useCurrentTeam";
+import useQuery from "~/hooks/useQuery";
+import useStores from "~/hooks/useStores";
+import { redirectTo } from "~/utils/urls";
+import { NotionUtils } from "../shared/NotionUtils";
+import { ImportDialog } from "./components/ImportDialog";
+
+export const Notion = observer(() => {
+  const { t } = useTranslation();
+  const { dialogs } = useStores();
+  const team = useCurrentTeam();
+  const history = useHistory();
+  const location = useLocation();
+  const queryParams = useQuery();
+
+  const appName = env.APP_NAME;
+  const authUrl = NotionUtils.authUrl({ state: { teamId: team.id } });
+
+  const service = queryParams.get("service");
+  const oauthSuccess = queryParams.get("success") === "";
+  const oauthError = queryParams.get("error");
+  const integrationId = queryParams.get("integrationId");
+
+  const clearQueryParams = React.useCallback(() => {
+    history.replace({
+      pathname: location.pathname,
+      search: "",
+    });
+  }, [history, location]);
+
+  const handleSubmit = React.useCallback(() => {
+    dialogs.closeAllModals();
+    clearQueryParams();
+  }, [dialogs, clearQueryParams]);
+
+  React.useEffect(() => {
+    if (
+      integrationId &&
+      oauthSuccess &&
+      service === IntegrationService.Notion
+    ) {
+      dialogs.openModal({
+        title: t("Import data"),
+        content: (
+          <ImportDialog integrationId={integrationId} onSubmit={handleSubmit} />
+        ),
+        onClose: clearQueryParams,
+      });
+    }
+  }, [t, dialogs, oauthSuccess, service, clearQueryParams]);
+
+  React.useEffect(() => {
+    if (!oauthError) {
+      return;
+    }
+
+    if (oauthError === "access_denied") {
+      toast.error(
+        t(
+          "Whoops, you need to accept the permissions in Notion to connect {{ appName }} to your workspace. Try again?",
+          {
+            appName,
+          }
+        )
+      );
+    } else {
+      toast.error(
+        t(
+          "Something went wrong while authenticating your request. Please try logging in again."
+        )
+      );
+    }
+  }, [t, appName, oauthError]);
+
+  return (
+    <Button type="submit" onClick={() => redirectTo(authUrl)} neutral>
+      {t("Import")}…
+    </Button>
+  );
+});

plugins/notion/client/components/ImportDialog.tsx

@@ -0,0 +1,78 @@
+import React from "react";
+import { useTranslation } from "react-i18next";
+import { toast } from "sonner";
+import { ImportInput } from "@shared/schema";
+import { CollectionPermission, IntegrationService } from "@shared/types";
+import Button from "~/components/Button";
+import Flex from "~/components/Flex";
+import InputSelectPermission from "~/components/InputSelectPermission";
+import Text from "~/components/Text";
+import useBoolean from "~/hooks/useBoolean";
+import useStores from "~/hooks/useStores";
+import { EmptySelectValue } from "~/types";
+
+type Props = {
+  /** The integrationId associated with this import flow. */
+  integrationId: string;
+  /** Callback to handle import creation. */
+  onSubmit: () => void;
+};
+
+export function ImportDialog({ integrationId, onSubmit }: Props) {
+  const { t } = useTranslation();
+  const { imports } = useStores();
+  const [submitting, setSubmitting, resetSubmitting] = useBoolean();
+  const [permission, setPermission] = React.useState<CollectionPermission>();
+
+  const handlePermissionChange = React.useCallback(
+    (value: CollectionPermission | typeof EmptySelectValue) => {
+      setPermission(value === EmptySelectValue ? undefined : value);
+    },
+    []
+  );
+
+  const handleStartImport = React.useCallback(async () => {
+    setSubmitting();
+
+    // TODO: This can send the page info + permission once we overcome the search timeout issues.
+    const input: ImportInput<IntegrationService.Notion> = [{ permission }];
+
+    try {
+      await imports.create(
+        { service: IntegrationService.Notion },
+        { integrationId, input }
+      );
+
+      toast.success(
+        t("Your import is being processed, you can safely leave this page")
+      );
+
+      onSubmit();
+    } catch (err) {
+      toast.error(err.message);
+      resetSubmitting();
+    }
+  }, [permission, onSubmit]);
+
+  return (
+    <Flex column gap={12}>
+      <div>
+        <InputSelectPermission
+          value={permission}
+          onChange={handlePermissionChange}
+        />
+        <Text as="span" type="secondary">
+          {t(
+            "Set the default permission level for collections created from the import"
+          )}
+          .
+        </Text>
+      </div>
+      <Flex justify="flex-end">
+        <Button onClick={handleStartImport} disabled={submitting}>
+          {t("Start import")}
+        </Button>
+      </Flex>
+    </Flex>
+  );
+}

plugins/notion/client/index.tsx

@@ -0,0 +1,19 @@
+import { t } from "i18next";
+import * as React from "react";
+import { cdnPath } from "@shared/utils/urls";
+import { Hook, PluginManager } from "~/utils/PluginManager";
+import config from "../plugin.json";
+import { Notion } from "./Imports";
+
+PluginManager.add([
+  {
+    ...config,
+    type: Hook.Imports,
+    value: {
+      title: "Notion",
+      subtitle: t("Import pages from Notion"),
+      icon: <img src={cdnPath("/images/notion.png")} width={28} />,
+      action: <Notion />,
+    },
+  },
+]);

plugins/notion/plugin.json

@@ -0,0 +1,5 @@
+{
+  "id": "notion",
+  "name": "Notion",
+  "description": "Adds a Notion integration for importing data."
+}

plugins/notion/server/api/notion.ts

@@ -0,0 +1,105 @@
+import Router from "koa-router";
+import { IntegrationService, IntegrationType } from "@shared/types";
+import { parseDomain } from "@shared/utils/domains";
+import Logger from "@server/logging/Logger";
+import auth from "@server/middlewares/authentication";
+import { transaction } from "@server/middlewares/transaction";
+import validate from "@server/middlewares/validate";
+import { Integration, IntegrationAuthentication, Team } from "@server/models";
+import { APIContext } from "@server/types";
+import { NotionClient } from "../notion";
+import * as T from "./schema";
+import { NotionUtils } from "plugins/notion/shared/NotionUtils";
+
+const router = new Router();
+
+router.get(
+  "notion.callback",
+  auth({ optional: true }),
+  validate(T.NotionCallbackSchema),
+  transaction(),
+  async (ctx: APIContext<T.NotionCallbackReq>) => {
+    const { code, state, error } = ctx.input.query;
+    const { user } = ctx.state.auth;
+    const { transaction } = ctx.state;
+
+    let parsedState;
+    try {
+      parsedState = NotionUtils.parseState(state);
+    } catch {
+      ctx.redirect(NotionUtils.errorUrl("invalid_state"));
+      return;
+    }
+
+    const { teamId } = parsedState;
+
+    // This code block accounts for the root domain being unable to access authentication for subdomains.
+    // We must forward to the appropriate subdomain to complete the oauth flow.
+    if (!user) {
+      if (teamId) {
+        try {
+          const team = await Team.findByPk(teamId, {
+            rejectOnEmpty: true,
+            transaction,
+          });
+
+          return parseDomain(ctx.host).teamSubdomain === team.subdomain
+            ? ctx.redirect("/")
+            : ctx.redirectOnClient(
+                NotionUtils.callbackUrl({
+                  baseUrl: team.url,
+                  params: ctx.request.querystring,
+                })
+              );
+        } catch (err) {
+          Logger.error(`Error fetching team for teamId: ${teamId}!`, err);
+          return ctx.redirect(NotionUtils.errorUrl("unauthenticated"));
+        }
+      } else {
+        return ctx.redirect(NotionUtils.errorUrl("unauthenticated"));
+      }
+    }
+
+    // Check error after any sub-domain redirection. Otherwise, the user will be redirected to the root domain.
+    if (error) {
+      ctx.redirect(NotionUtils.errorUrl(error));
+      return;
+    }
+
+    // validation middleware ensures that code is non-null at this point.
+    const data = await NotionClient.oauthAccess(code!);
+
+    const authentication = await IntegrationAuthentication.create(
+      {
+        service: IntegrationService.Notion,
+        userId: user.id,
+        teamId: user.teamId,
+        token: data.access_token,
+      },
+      { transaction }
+    );
+    const integration = await Integration.create<
+      Integration<IntegrationType.Import>
+    >(
+      {
+        service: IntegrationService.Notion,
+        type: IntegrationType.Import,
+        userId: user.id,
+        teamId: user.teamId,
+        authenticationId: authentication.id,
+        settings: {
+          externalWorkspace: {
+            id: data.workspace_id,
+            name: data.workspace_name ?? "Notion import",
+            iconUrl: data.workspace_icon ?? undefined,
+          },
+        },
+      },
+      { transaction }
+    );
+
+    ctx.redirect(NotionUtils.successUrl(integration.id));
+  }
+);
+
+export default router;

plugins/notion/server/api/schema.ts

@@ -0,0 +1,25 @@
+import isEmpty from "lodash/isEmpty";
+import { z } from "zod";
+import { BaseSchema } from "@server/routes/api/schema";
+
+export const NotionCallbackSchema = BaseSchema.extend({
+  query: z
+    .object({
+      code: z.string().nullish(),
+      state: z.string(),
+      error: z.string().nullish(),
+    })
+    .refine((req) => !(isEmpty(req.code) && isEmpty(req.error)), {
+      message: "one of code or error is required",
+    }),
+});
+
+export type NotionCallbackReq = z.infer<typeof NotionCallbackSchema>;
+
+export const NotionSearchSchema = BaseSchema.extend({
+  body: z.object({
+    integrationId: z.string().uuid(),
+  }),
+});
+
+export type NotionSearchReq = z.infer<typeof NotionSearchSchema>;

plugins/notion/server/env.ts

@@ -0,0 +1,19 @@
+import { IsOptional } from "class-validator";
+import { Environment } from "@server/env";
+import { Public } from "@server/utils/decorators/Public";
+import environment from "@server/utils/environment";
+import { CannotUseWithout } from "@server/utils/validators";
+
+class NotionPluginEnvironment extends Environment {
+  @IsOptional()
+  @Public
+  public NOTION_CLIENT_ID = this.toOptionalString(environment.NOTION_CLIENT_ID);
+
+  @IsOptional()
+  @CannotUseWithout("NOTION_CLIENT_ID")
+  public NOTION_CLIENT_SECRET = this.toOptionalString(
+    environment.NOTION_CLIENT_SECRET
+  );
+}
+
+export default new NotionPluginEnvironment();

plugins/notion/server/index.ts

@@ -0,0 +1,26 @@
+import { Hook, PluginManager } from "@server/utils/PluginManager";
+import config from "../plugin.json";
+import router from "./api/notion";
+import env from "./env";
+import { NotionImportsProcessor } from "./processors/NotionImportsProcessor";
+import NotionAPIImportTask from "./tasks/NotionAPIImportTask";
+
+const enabled = !!env.NOTION_CLIENT_ID && !!env.NOTION_CLIENT_SECRET;
+
+if (enabled) {
+  PluginManager.add([
+    {
+      ...config,
+      type: Hook.API,
+      value: router,
+    },
+    {
+      type: Hook.Processor,
+      value: NotionImportsProcessor,
+    },
+    {
+      type: Hook.Task,
+      value: NotionAPIImportTask,
+    },
+  ]);
+}

plugins/notion/server/notion.ts

@@ -0,0 +1,284 @@
+import {
+  Client,
+  isFullPage,
+  isFullPageOrDatabase,
+  isFullUser,
+} from "@notionhq/client";
+import {
+  BlockObjectResponse,
+  DatabaseObjectResponse,
+  PageObjectResponse,
+  RichTextItemResponse,
+} from "@notionhq/client/build/src/api-endpoints";
+import { RateLimit } from "async-sema";
+import compact from "lodash/compact";
+import { z } from "zod";
+import { Second } from "@shared/utils/time";
+import { NotionUtils } from "../shared/NotionUtils";
+import { Block, Page, PageType } from "../shared/types";
+import env from "./env";
+
+type PageInfo = {
+  title: string;
+  emoji?: string;
+  author?: string;
+  createdAt?: Date;
+  updatedAt?: Date;
+};
+
+const Credentials = Buffer.from(
+  `${env.NOTION_CLIENT_ID}:${env.NOTION_CLIENT_SECRET}`
+).toString("base64");
+
+const AccessTokenResponseSchema = z.object({
+  access_token: z.string(),
+  bot_id: z.string(),
+  workspace_id: z.string(),
+  workspace_name: z.string().nullish(),
+  workspace_icon: z.string().url().nullish(),
+});
+
+export class NotionClient {
+  private client: Client;
+  private limiter: ReturnType<typeof RateLimit>;
+  private pageSize = 25;
+
+  constructor(
+    accessToken: string,
+    rateLimit: { window: number; limit: number } = {
+      window: Second.ms,
+      limit: 3,
+    }
+  ) {
+    this.client = new Client({
+      auth: accessToken,
+    });
+    this.limiter = RateLimit(rateLimit.limit, {
+      timeUnit: rateLimit.window,
+      uniformDistribution: true,
+    });
+  }
+
+  static async oauthAccess(code: string) {
+    const headers = {
+      "Content-Type": "application/json",
+      Accept: "application/json",
+      Authorization: `Basic ${Credentials}`,
+    };
+    const body = {
+      grant_type: "authorization_code",
+      code,
+      redirect_uri: NotionUtils.callbackUrl(),
+    };
+
+    const res = await fetch(NotionUtils.tokenUrl, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+    });
+
+    return AccessTokenResponseSchema.parse(await res.json());
+  }
+
+  async fetchRootPages() {
+    const pages: Page[] = [];
+
+    let cursor: string | undefined;
+    let hasMore = true;
+
+    while (hasMore) {
+      await this.limiter();
+
+      const response = await this.client.search({
+        start_cursor: cursor,
+        page_size: this.pageSize,
+      });
+
+      response.results.forEach((item) => {
+        if (!isFullPageOrDatabase(item)) {
+          return;
+        }
+
+        if (item.parent.type === "workspace") {
+          pages.push({
+            type: item.object === "page" ? PageType.Page : PageType.Database,
+            id: item.id,
+            name: this.parseTitle(item),
+            emoji: this.parseEmoji(item),
+          });
+        }
+      });
+
+      hasMore = response.has_more;
+      cursor = response.next_cursor ?? undefined;
+    }
+
+    return pages;
+  }
+
+  async fetchPage(pageId: string) {
+    const pageInfo = await this.fetchPageInfo(pageId);
+    const blocks = await this.fetchBlockChildren(pageId);
+    return { ...pageInfo, blocks };
+  }
+
+  async fetchDatabase(databaseId: string) {
+    const databaseInfo = await this.fetchDatabaseInfo(databaseId);
+    const pages = await this.queryDatabase(databaseId);
+    return { ...databaseInfo, pages };
+  }
+
+  private async fetchBlockChildren(blockId: string) {
+    const blocks: Block[] = [];
+
+    let cursor: string | undefined;
+    let hasMore = true;
+
+    while (hasMore) {
+      await this.limiter();
+
+      const response = await this.client.blocks.children.list({
+        block_id: blockId,
+        start_cursor: cursor,
+        page_size: this.pageSize,
+      });
+
+      blocks.push(...(response.results as BlockObjectResponse[]));
+
+      hasMore = response.has_more;
+      cursor = response.next_cursor ?? undefined;
+    }
+
+    // Recursive fetch when direct children have their own children.
+    await Promise.all(
+      blocks.map(async (block) => {
+        if (
+          block.has_children &&
+          block.type !== "child_page" &&
+          block.type !== "child_database"
+        ) {
+          block.children = await this.fetchBlockChildren(block.id);
+        }
+      })
+    );
+
+    return blocks;
+  }
+
+  private async queryDatabase(databaseId: string) {
+    const pages: Page[] = [];
+
+    let cursor: string | undefined;
+    let hasMore = true;
+
+    while (hasMore) {
+      await this.limiter();
+
+      const response = await this.client.databases.query({
+        database_id: databaseId,
+        filter_properties: ["title"],
+        start_cursor: cursor,
+        page_size: this.pageSize,
+      });
+
+      const pagesFromRes = compact(
+        response.results.map<Page | undefined>((item) => {
+          if (!isFullPage(item)) {
+            return;
+          }
+
+          return {
+            type: PageType.Page,
+            id: item.id,
+            name: this.parseTitle(item),
+            emoji: this.parseEmoji(item),
+          };
+        })
+      );
+
+      pages.push(...pagesFromRes);
+
+      hasMore = response.has_more;
+      cursor = response.next_cursor ?? undefined;
+    }
+
+    return pages;
+  }
+
+  private async fetchPageInfo(pageId: string): Promise<PageInfo> {
+    await this.limiter();
+    const page = (await this.client.pages.retrieve({
+      page_id: pageId,
+    })) as PageObjectResponse;
+
+    const author = await this.fetchUsername(page.created_by.id);
+
+    return {
+      title: this.parseTitle(page),
+      emoji: this.parseEmoji(page),
+      author: author ?? undefined,
+      createdAt: !page.created_time ? undefined : new Date(page.created_time),
+      updatedAt: !page.last_edited_time
+        ? undefined
+        : new Date(page.last_edited_time),
+    };
+  }
+
+  private async fetchDatabaseInfo(databaseId: string): Promise<PageInfo> {
+    await this.limiter();
+    const database = (await this.client.databases.retrieve({
+      database_id: databaseId,
+    })) as DatabaseObjectResponse;
+
+    const author = await this.fetchUsername(database.created_by.id);
+
+    return {
+      title: this.parseTitle(database),
+      emoji: this.parseEmoji(database),
+      author: author ?? undefined,
+      createdAt: !database.created_time
+        ? undefined
+        : new Date(database.created_time),
+      updatedAt: !database.last_edited_time
+        ? undefined
+        : new Date(database.last_edited_time),
+    };
+  }
+
+  private async fetchUsername(userId: string) {
+    await this.limiter();
+    const user = await this.client.users.retrieve({ user_id: userId });
+
+    if (user.type === "person" || !user.bot.owner) {
+      return user.name;
+    }
+
+    // bot belongs to a user, get the user's name.
+    if (user.bot.owner.type === "user" && isFullUser(user.bot.owner.user)) {
+      return user.bot.owner.user.name;
+    }
+
+    // bot belongs to a workspace, fallback to bot's name.
+    return user.name;
+  }
+
+  private parseTitle(item: PageObjectResponse | DatabaseObjectResponse) {
+    let richTexts: RichTextItemResponse[];
+
+    if (item.object === "page") {
+      const titleProp = Object.values(item.properties).find(
+        (property) => property.type === "title"
+      );
+      richTexts = titleProp?.title ?? [];
+    } else {
+      richTexts = item.title;
+    }
+
+    return richTexts.map((richText) => richText.plain_text).join("");
+  }
+
+  private parseEmoji(item: PageObjectResponse | DatabaseObjectResponse) {
+    // Other icon types return a url to download from, which we don't support.
+    return item.icon?.type === "emoji" ? item.icon.emoji : undefined;
+  }
+}

plugins/notion/server/processors/NotionImportsProcessor.ts

@@ -0,0 +1,71 @@
+import { Transaction } from "sequelize";
+import { NotionImportInput, NotionImportTaskInput } from "@shared/schema";
+import { IntegrationService } from "@shared/types";
+import { Import, ImportTask, Integration } from "@server/models";
+import ImportsProcessor from "@server/queues/processors/ImportsProcessor";
+import { NotionClient } from "../notion";
+import NotionAPIImportTask from "../tasks/NotionAPIImportTask";
+
+export class NotionImportsProcessor extends ImportsProcessor<IntegrationService.Notion> {
+  /**
+   * Determine whether this is a "Notion" import.
+   *
+   * @param importModel Import model associated with the import.
+   * @returns boolean.
+   */
+  protected canProcess(
+    importModel: Import<IntegrationService.Notion>
+  ): boolean {
+    return importModel.service === IntegrationService.Notion;
+  }
+
+  /**
+   * Build task inputs which will be used for `NotionAPIImportTask`s.
+   *
+   * @param importInput Array of root externalId and associated info which were used to create the import.
+   * @returns `NotionImportTaskInput`.
+   */
+  protected async buildTasksInput(
+    importModel: Import<IntegrationService.Notion>,
+    transaction: Transaction
+  ): Promise<NotionImportTaskInput> {
+    const integration = await Integration.scope("withAuthentication").findByPk(
+      importModel.integrationId,
+      { rejectOnEmpty: true }
+    );
+
+    const notion = new NotionClient(integration.authentication.token);
+
+    const rootPages = await notion.fetchRootPages();
+
+    // App will send the default permission in an array with single item.
+    const defaultPermission = importModel.input[0].permission;
+
+    // TODO: This update can be deleted when we receive the page info + permission from app.
+    const importInput: NotionImportInput = rootPages.map((page) => ({
+      type: page.type,
+      externalId: page.id,
+      permission: defaultPermission,
+    }));
+
+    importModel.input = importInput;
+    await importModel.save({ transaction });
+
+    return rootPages.map((page) => ({
+      type: page.type,
+      externalId: page.id,
+    }));
+  }
+
+  /**
+   * Schedule the first `NotionAPIImportTask` for the import.
+   *
+   * @param importTask ImportTask model associated with the `NotionAPIImportTask`.
+   * @returns Promise that resolves when the task is scheduled.
+   */
+  protected async scheduleTask(
+    importTask: ImportTask<IntegrationService.Notion>
+  ): Promise<void> {
+    await NotionAPIImportTask.schedule({ importTaskId: importTask.id });
+  }
+}

plugins/notion/server/tasks/NotionAPIImportTask.ts

@@ -0,0 +1,144 @@
+import { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
+import { IntegrationService, ProsemirrorDoc } from "@shared/types";
+import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
+import { Integration } from "@server/models";
+import ImportTask from "@server/models/ImportTask";
+import APIImportTask, {
+  ProcessOutput,
+} from "@server/queues/tasks/APIImportTask";
+import { Block, PageType } from "../../shared/types";
+import { NotionClient } from "../notion";
+import { NotionConverter, NotionPage } from "../utils/NotionConverter";
+
+type ChildPage = { type: PageType; externalId: string };
+
+type ParsePageOutput = ImportTaskOutput[number] & {
+  collectionExternalId?: string;
+  children: ChildPage[];
+};
+
+export default class NotionAPIImportTask extends APIImportTask<IntegrationService.Notion> {
+  /**
+   * Process the Notion import task.
+   * This fetches data from Notion and converts it to task output.
+   *
+   * @param importTask ImportTask model to process.
+   * @returns Promise with output that resolves once processing has completed.
+   */
+  protected async process(
+    importTask: ImportTask<IntegrationService.Notion>
+  ): Promise<ProcessOutput<IntegrationService.Notion>> {
+    const integration = await Integration.scope("withAuthentication").findByPk(
+      importTask.import.integrationId,
+      { rejectOnEmpty: true }
+    );
+
+    const client = new NotionClient(integration.authentication.token);
+
+    const parsedPages = await Promise.all(
+      importTask.input.map(async (item) => this.processPage({ item, client }))
+    );
+
+    const taskOutput: ImportTaskOutput = parsedPages.map((parsedPage) => ({
+      externalId: parsedPage.externalId,
+      title: parsedPage.title,
+      emoji: parsedPage.emoji,
+      content: parsedPage.content,
+      author: parsedPage.author,
+      createdAt: parsedPage.createdAt,
+      updatedAt: parsedPage.updatedAt,
+    }));
+
+    const childTasksInput: ImportTaskInput<IntegrationService.Notion> =
+      parsedPages.flatMap((parsedPage) =>
+        parsedPage.children.map((childPage) => ({
+          type: childPage.type,
+          externalId: childPage.externalId,
+          parentExternalId: parsedPage.externalId,
+          collectionExternalId: parsedPage.collectionExternalId,
+        }))
+      );
+
+    return { taskOutput, childTasksInput };
+  }
+
+  /**
+   * Schedule the next `NotionAPIImportTask`.
+   *
+   * @param importTask ImportTask model associated with the `NotionAPIImportTask`.
+   * @returns Promise that resolves when the task is scheduled.
+   */
+  protected async scheduleNextTask(
+    importTask: ImportTask<IntegrationService.Notion>
+  ) {
+    await NotionAPIImportTask.schedule({ importTaskId: importTask.id });
+    return;
+  }
+
+  /**
+   * Fetch page data from Notion and convert it to expected output.
+   *
+   * @param item Object containing data about a notion page (or) database.
+   * @param client Notion client.
+   * @returns Promise of parsed page output that resolves when the task is scheduled.
+   */
+  private async processPage({
+    item,
+    client,
+  }: {
+    item: ImportTaskInput<IntegrationService.Notion>[number];
+    client: NotionClient;
+  }): Promise<ParsePageOutput> {
+    const collectionExternalId = item.collectionExternalId ?? item.externalId;
+
+    // Convert Notion database to an empty page with "pages in database" as its children.
+    if (item.type === PageType.Database) {
+      const { pages, ...databaseInfo } = await client.fetchDatabase(
+        item.externalId
+      );
+
+      return {
+        ...databaseInfo,
+        externalId: item.externalId,
+        content: ProsemirrorHelper.getEmptyDocument() as ProsemirrorDoc,
+        collectionExternalId,
+        children: pages.map((page) => ({
+          type: page.type,
+          externalId: page.id,
+        })),
+      };
+    }
+
+    const { blocks, ...pageInfo } = await client.fetchPage(item.externalId);
+
+    return {
+      ...pageInfo,
+      externalId: item.externalId,
+      content: NotionConverter.page({ children: blocks } as NotionPage),
+      collectionExternalId,
+      children: this.parseChildPages(blocks),
+    };
+  }
+
+  /**
+   * Parse Notion page blocks to get its child pages and databases.
+   *
+   * @param pageBlocks Array of blocks representing the page's content.
+   * @returns Array containing child page and child database info.
+   */
+  private parseChildPages(pageBlocks: Block[]): ChildPage[] {
+    const childPages: ChildPage[] = [];
+
+    pageBlocks.forEach((block) => {
+      if (block.type === "child_page") {
+        childPages.push({ type: PageType.Page, externalId: block.id });
+      } else if (block.type === "child_database") {
+        childPages.push({ type: PageType.Database, externalId: block.id });
+      } else if (block.children?.length) {
+        childPages.push(...this.parseChildPages(block.children));
+      }
+    });
+
+    return childPages;
+  }
+}

plugins/notion/server/utils/NotionConverter.test.ts

@@ -0,0 +1,15 @@
+import { Node } from "prosemirror-model";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import data from "@server/test/fixtures/notion-page.json";
+import { NotionConverter, NotionPage } from "./NotionConverter";
+
+describe("NotionConverter", () => {
+  it("converts a page", () => {
+    const response = NotionConverter.page({
+      children: data,
+    } as NotionPage);
+
+    expect(response).toMatchSnapshot();
+    expect(ProsemirrorHelper.toProsemirror(response)).toBeInstanceOf(Node);
+  });
+});

plugins/notion/server/utils/NotionConverter.ts

@@ -0,0 +1,584 @@
+import type {
+  BookmarkBlockObjectResponse,
+  BreadcrumbBlockObjectResponse,
+  BulletedListItemBlockObjectResponse,
+  DividerBlockObjectResponse,
+  Heading1BlockObjectResponse,
+  Heading2BlockObjectResponse,
+  Heading3BlockObjectResponse,
+  NumberedListItemBlockObjectResponse,
+  ParagraphBlockObjectResponse,
+  QuoteBlockObjectResponse,
+  RichTextItemResponse,
+  FileBlockObjectResponse,
+  PdfBlockObjectResponse,
+  ImageBlockObjectResponse,
+  EmbedBlockObjectResponse,
+  TableBlockObjectResponse,
+  ToDoBlockObjectResponse,
+  EquationBlockObjectResponse,
+  CodeBlockObjectResponse,
+  ToggleBlockObjectResponse,
+  PageObjectResponse,
+  VideoBlockObjectResponse,
+  CalloutBlockObjectResponse,
+  ColumnListBlockObjectResponse,
+  ColumnBlockObjectResponse,
+  LinkPreviewBlockObjectResponse,
+  SyncedBlockBlockObjectResponse,
+  LinkToPageBlockObjectResponse,
+} from "@notionhq/client/build/src/api-endpoints";
+import isArray from "lodash/isArray";
+import { NoticeTypes } from "@shared/editor/nodes/Notice";
+import { MentionType, ProsemirrorData, ProsemirrorDoc } from "@shared/types";
+import Logger from "@server/logging/Logger";
+import { Block } from "../../shared/types";
+
+export type NotionPage = PageObjectResponse & {
+  children: Block[];
+};
+
+/** Convert Notion blocks to Outline data. */
+export class NotionConverter {
+  /**
+   * Nodes which cannot contain block children in Outline, their children
+   * will be flattened into the parent.
+   */
+  private static nodesWithoutBlockChildren = ["paragraph", "toggle"];
+
+  public static page(item: NotionPage): ProsemirrorDoc {
+    return {
+      type: "doc",
+      content: this.mapChildren(item),
+    };
+  }
+
+  private static mapChildren(item: Block | NotionPage) {
+    const mapChild = (
+      child: Block
+    ): ProsemirrorData | ProsemirrorData[] | undefined => {
+      if (child.type === "child_page" || child.type === "child_database") {
+        return; // this will be created as a nested page, no need to handle/convert.
+      }
+
+      // @ts-expect-error Not all blocks have an interface
+      if (this[child.type]) {
+        // @ts-expect-error Not all blocks have an interface
+        const response = this[child.type](child);
+        if (
+          response &&
+          this.nodesWithoutBlockChildren.includes(response.type) &&
+          "children" in child
+        ) {
+          return [response, ...this.mapChildren(child)];
+        }
+        return response;
+      }
+
+      Logger.warn("Encountered unknown Notion block", child);
+      return undefined;
+    };
+
+    let wrappingList;
+    const children = [] as ProsemirrorData[];
+
+    if (!item.children) {
+      return [];
+    }
+
+    for (const child of item.children) {
+      const mapped = mapChild(child);
+      if (!mapped) {
+        continue;
+      }
+
+      // Ensure lists are wrapped correctly – we require a wrapping element
+      // whereas Notion does not
+      // TODO: Handle mixed list
+      if (child.type === "numbered_list_item") {
+        if (!wrappingList) {
+          wrappingList = {
+            type: "ordered_list",
+            content: [] as ProsemirrorData[],
+          };
+        }
+
+        wrappingList.content.push(...(isArray(mapped) ? mapped : [mapped]));
+        continue;
+      }
+      if (child.type === "bulleted_list_item") {
+        if (!wrappingList) {
+          wrappingList = {
+            type: "bullet_list",
+            content: [] as ProsemirrorData[],
+          };
+        }
+
+        wrappingList.content.push(...(isArray(mapped) ? mapped : [mapped]));
+        continue;
+      }
+      if (child.type === "to_do") {
+        if (!wrappingList) {
+          wrappingList = {
+            type: "checkbox_list",
+            content: [] as ProsemirrorData[],
+          };
+        }
+
+        wrappingList.content.push(...(isArray(mapped) ? mapped : [mapped]));
+        continue;
+      }
+      if (wrappingList) {
+        children.push(wrappingList);
+        wrappingList = undefined;
+      }
+      children.push(...(isArray(mapped) ? mapped : [mapped]));
+    }
+
+    if (wrappingList) {
+      children.push(wrappingList);
+    }
+
+    return children;
+  }
+
+  private static callout(item: Block<CalloutBlockObjectResponse>) {
+    const colorToNoticeType: Record<string, NoticeTypes> = {
+      default_background: NoticeTypes.Info,
+      blue_background: NoticeTypes.Info,
+      purple_background: NoticeTypes.Info,
+      green_background: NoticeTypes.Success,
+      orange_background: NoticeTypes.Tip,
+      yellow_background: NoticeTypes.Tip,
+      pink_background: NoticeTypes.Warning,
+      red_background: NoticeTypes.Warning,
+    };
+
+    return {
+      type: "container_notice",
+      attrs: {
+        style:
+          colorToNoticeType[item.callout.color as string] ?? NoticeTypes.Info,
+      },
+      content: [
+        {
+          type: "paragraph",
+          content: item.callout.rich_text.map(this.rich_text).filter(Boolean),
+        },
+        ...this.mapChildren(item),
+      ],
+    };
+  }
+
+  private static column_list(item: Block<ColumnListBlockObjectResponse>) {
+    return this.mapChildren(item);
+  }
+
+  private static column(item: Block<ColumnBlockObjectResponse>) {
+    return this.mapChildren(item);
+  }
+
+  private static bookmark(item: BookmarkBlockObjectResponse) {
+    return {
+      type: "paragraph",
+      content: [
+        {
+          text: item.bookmark.caption.map(this.rich_text_to_plaintext).join(""),
+          type: "text",
+          marks: [
+            {
+              type: "link",
+              attrs: {
+                href: item.bookmark.url,
+                title: null,
+              },
+            },
+          ],
+        },
+      ],
+    };
+  }
+
+  private static breadcrumb(_: BreadcrumbBlockObjectResponse) {
+    return undefined;
+  }
+
+  private static bulleted_list_item(
+    item: Block<BulletedListItemBlockObjectResponse>
+  ) {
+    return {
+      type: "list_item",
+      content: [
+        {
+          type: "paragraph",
+          content: item.bulleted_list_item.rich_text
+            .map(this.rich_text)
+            .filter(Boolean),
+        },
+        ...this.mapChildren(item),
+      ],
+    };
+  }
+
+  private static code(item: CodeBlockObjectResponse) {
+    return {
+      type: "code_fence",
+      attrs: {
+        language: item.code.language,
+      },
+      content: [
+        {
+          type: "text",
+          text: item.code.rich_text.map(this.rich_text_to_plaintext).join(""),
+        },
+      ],
+    };
+  }
+
+  private static numbered_list_item(
+    item: Block<NumberedListItemBlockObjectResponse>
+  ) {
+    return {
+      type: "list_item",
+      content: [
+        {
+          type: "paragraph",
+          content: item.numbered_list_item.rich_text
+            .map(this.rich_text)
+            .filter(Boolean),
+        },
+        ...this.mapChildren(item),
+      ],
+    };
+  }
+
+  private static rich_text(item: RichTextItemResponse) {
+    const annotationToMark: Record<
+      keyof RichTextItemResponse["annotations"],
+      string
+    > = {
+      bold: "strong",
+      code: "code_inline",
+      italic: "em",
+      underline: "underline",
+      strikethrough: "strikethrough",
+      color: "highlight",
+    };
+
+    const mapAttrs = () =>
+      Object.entries(item.annotations)
+        .filter(([key]) => key !== "color")
+        .filter(([, enabled]) => enabled)
+        .map(([key]) => ({
+          type: annotationToMark[key as keyof typeof annotationToMark],
+        }));
+
+    if (item.type === "mention") {
+      if (item.mention.type === "page") {
+        return {
+          type: "mention",
+          attrs: {
+            type: MentionType.Document,
+            label: item.plain_text,
+            modelId: item.mention.page.id,
+          },
+        };
+      }
+      if (item.mention.type === "link_mention") {
+        return {
+          type: "text",
+          text: item.plain_text,
+          marks: [
+            {
+              type: "link",
+              attrs: {
+                href: item.mention.link_mention.href,
+              },
+            },
+          ],
+        };
+      }
+      if (item.mention.type === "link_preview") {
+        return {
+          type: "text",
+          text: item.plain_text,
+          marks: [
+            {
+              type: "link",
+              attrs: {
+                href: item.mention.link_preview.url,
+              },
+            },
+          ],
+        };
+      }
+
+      if (!item.plain_text) {
+        return undefined;
+      }
+
+      return {
+        type: "text",
+        text: item.plain_text,
+      };
+    }
+
+    if (item.type === "equation") {
+      return {
+        type: "math_inline",
+        content: [
+          {
+            type: "text",
+            text: item.equation.expression,
+          },
+        ],
+      };
+    }
+
+    if (!item.text.content) {
+      return undefined;
+    }
+
+    return {
+      type: "text",
+      text: item.text.content,
+      marks: [
+        ...mapAttrs(),
+        ...(item.text.link
+          ? [{ type: "link", attrs: { href: item.text.link.url } }]
+          : []),
+      ].filter(Boolean),
+    };
+  }
+
+  private static rich_text_to_plaintext(item: RichTextItemResponse) {
+    return item.plain_text;
+  }
+
+  private static divider(_: DividerBlockObjectResponse) {
+    return {
+      type: "hr",
+    };
+  }
+
+  private static equation(item: EquationBlockObjectResponse) {
+    return {
+      type: "math_block",
+      content: [
+        {
+          type: "text",
+          text: item.equation.expression,
+        },
+      ],
+    };
+  }
+
+  private static embed(item: EmbedBlockObjectResponse) {
+    return {
+      type: "embed",
+      attrs: {
+        href: item.embed.url,
+      },
+    };
+  }
+
+  private static file(item: FileBlockObjectResponse) {
+    return {
+      type: "attachment",
+      attrs: {
+        href: "file" in item.file ? item.file.file.url : item.file.external.url,
+        title: item.file.name,
+      },
+    };
+  }
+
+  private static pdf(item: PdfBlockObjectResponse) {
+    return {
+      type: "attachment",
+      attrs: {
+        href: "file" in item.pdf ? item.pdf.file.url : item.pdf.external.url,
+        title: item.pdf.caption.map(this.rich_text_to_plaintext).join(""),
+      },
+    };
+  }
+
+  private static heading_1(item: Heading1BlockObjectResponse) {
+    return {
+      type: "heading",
+      attrs: {
+        level: 1,
+      },
+      content: item.heading_1.rich_text.map(this.rich_text).filter(Boolean),
+    };
+  }
+
+  private static heading_2(item: Heading2BlockObjectResponse) {
+    return {
+      type: "heading",
+      attrs: {
+        level: 2,
+      },
+      content: item.heading_2.rich_text.map(this.rich_text).filter(Boolean),
+    };
+  }
+
+  private static heading_3(item: Heading3BlockObjectResponse) {
+    return {
+      type: "heading",
+      attrs: {
+        level: 3,
+      },
+      content: item.heading_3.rich_text.map(this.rich_text).filter(Boolean),
+    };
+  }
+
+  private static image(item: ImageBlockObjectResponse) {
+    return {
+      type: "paragraph",
+      content: [
+        {
+          type: "image",
+          attrs: {
+            src:
+              "file" in item.image
+                ? item.image.file.url
+                : item.image.external.url,
+            alt: item.image.caption.map(this.rich_text_to_plaintext).join(""),
+          },
+        },
+      ],
+    };
+  }
+
+  private static link_preview(item: LinkPreviewBlockObjectResponse) {
+    return {
+      type: "paragraph",
+      content: [
+        {
+          type: "text",
+          text: item.link_preview.url,
+          marks: [
+            {
+              type: "link",
+              attrs: {
+                href: item.link_preview.url,
+              },
+            },
+          ],
+        },
+      ],
+    };
+  }
+
+  private static link_to_page(item: LinkToPageBlockObjectResponse) {
+    if (item.link_to_page.type !== "page_id") {
+      return undefined;
+    }
+    return {
+      type: "mention",
+      attrs: {
+        modelId: item.link_to_page.page_id,
+        type: MentionType.Document,
+        label: "Page",
+      },
+    };
+  }
+
+  private static paragraph(item: ParagraphBlockObjectResponse) {
+    return {
+      type: "paragraph",
+      content: item.paragraph.rich_text.map(this.rich_text).filter(Boolean),
+    };
+  }
+
+  private static quote(item: Block<QuoteBlockObjectResponse>) {
+    return {
+      type: "blockquote",
+      content: [
+        {
+          type: "paragraph",
+          content: item.quote.rich_text.map(this.rich_text).filter(Boolean),
+        },
+        ...this.mapChildren(item),
+      ],
+    };
+  }
+
+  private static synced_block(item: Block<SyncedBlockBlockObjectResponse>) {
+    return this.mapChildren(item);
+  }
+
+  private static table(
+    item: TableBlockObjectResponse & {
+      children: Array<{
+        table_row: {
+          cells: Array<Array<RichTextItemResponse>>;
+        };
+        type?: "table_row";
+        object?: "block";
+      }>;
+    }
+  ) {
+    return {
+      type: "table",
+      content: item.children.map((tr, y) => ({
+        type: "tr",
+        content: tr.table_row.cells.map((td, x) => ({
+          type:
+            (item.table.has_row_header && y === 0) ||
+            (item.table.has_column_header && x === 0)
+              ? "th"
+              : "td",
+          content: [
+            {
+              type: "paragraph",
+              content: td.map(this.rich_text).filter(Boolean),
+            },
+          ],
+        })),
+      })),
+    };
+  }
+
+  private static toggle(item: ToggleBlockObjectResponse) {
+    return {
+      type: "paragraph",
+      content: item.toggle.rich_text.map(this.rich_text).filter(Boolean),
+    };
+  }
+
+  private static to_do(item: Block<ToDoBlockObjectResponse>) {
+    return {
+      type: "checkbox_item",
+      attrs: {
+        checked: item.to_do.checked,
+      },
+      content: [
+        {
+          type: "paragraph",
+          content: item.to_do.rich_text.map(this.rich_text).filter(Boolean),
+        },
+        ...this.mapChildren(item),
+      ],
+    };
+  }
+
+  private static video(item: VideoBlockObjectResponse) {
+    if (item.video.type === "file") {
+      return {
+        type: "video",
+        attrs: {
+          src: item.video.file.url,
+          title: item.video.caption.map(this.rich_text_to_plaintext).join(""),
+        },
+      };
+    }
+
+    return {
+      type: "embed",
+      attrs: {
+        href: item.video.external.url,
+      },
+    };
+  }
+}

plugins/notion/shared/NotionUtils.ts

@@ -0,0 +1,58 @@
+import queryString from "query-string";
+import env from "@shared/env";
+import { IntegrationService } from "@shared/types";
+import { settingsPath } from "@shared/utils/routeHelpers";
+
+export type OAuthState = {
+  teamId: string;
+};
+
+export class NotionUtils {
+  public static tokenUrl = "https://api.notion.com/v1/oauth/token";
+  private static authBaseUrl = "https://api.notion.com/v1/oauth/authorize";
+
+  private static settingsUrl = settingsPath("import");
+
+  static parseState(state: string): OAuthState {
+    return JSON.parse(state);
+  }
+
+  static successUrl(integrationId: string) {
+    const params = {
+      success: "",
+      service: IntegrationService.Notion,
+      integrationId,
+    };
+    return `${this.settingsUrl}?${queryString.stringify(params)}`;
+  }
+
+  static errorUrl(error: string) {
+    const params = {
+      error,
+      service: IntegrationService.Notion,
+    };
+    return `${this.settingsUrl}?${queryString.stringify(params)}`;
+  }
+
+  static callbackUrl(
+    { baseUrl, params }: { baseUrl: string; params?: string } = {
+      baseUrl: `${env.URL}`,
+      params: undefined,
+    }
+  ) {
+    return params
+      ? `${baseUrl}/api/notion.callback?${params}`
+      : `${baseUrl}/api/notion.callback`;
+  }
+
+  static authUrl({ state }: { state: OAuthState }) {
+    const params = {
+      client_id: env.NOTION_CLIENT_ID,
+      redirect_uri: this.callbackUrl(),
+      state: JSON.stringify(state),
+      response_type: "code",
+      owner: "user",
+    };
+    return `${this.authBaseUrl}?${queryString.stringify(params)}`;
+  }
+}

plugins/notion/shared/types.ts

@@ -0,0 +1,18 @@
+import { BlockObjectResponse } from "@notionhq/client/build/src/api-endpoints";
+
+export enum PageType {
+  Page = "page",
+  Database = "database",
+}
+
+export type Page = {
+  type: PageType;
+  id: string;
+  name: string;
+  emoji?: string;
+};
+
+// Transformed block structure with "children".
+export type Block<T = BlockObjectResponse> = T & {
+  children?: Block[];
+};

plugins/webhooks/server/tasks/DeliverWebhookTask.ts

@@ -231,6 +231,12 @@ export default class DeliverWebhookTask extends BaseTask<Props> {
       case "userMemberships.update":
         // Ignored
         return;
+      case "imports.create":
+      case "imports.update":
+      case "imports.processed":
+      case "imports.delete":
+        // Ignored
+        return;
       default:
         assertUnreachable(event);
     }

server/commands/documentCreator.ts

@@ -19,6 +19,7 @@ type Props = Optional<
     | "collectionId"
     | "parentDocumentId"
     | "importId"
+    | "apiImportId"
     | "template"
     | "fullWidth"
     | "sourceMetadata"
@@ -51,6 +52,7 @@ export default async function documentCreator({
   templateDocument,
   fullWidth,
   importId,
+  apiImportId,
   createdAt,
   // allows override for import
   updatedAt,
@@ -116,6 +118,7 @@ export default async function documentCreator({
     templateId,
     publishedAt,
     importId,
+    apiImportId,
     sourceMetadata,
     fullWidth: fullWidth ?? templateDocument?.fullWidth,
     icon: icon ?? templateDocument?.icon,
@@ -142,7 +145,7 @@ export default async function documentCreator({
       teamId: document.teamId,
       actorId: user.id,
       data: {
-        source: importId ? "import" : undefined,
+        source: importId || apiImportId ? "import" : undefined,
         title: document.title,
         templateId,
       },

server/errors.ts

@@ -201,6 +201,12 @@ export function AuthenticationProviderDisabledError(
   });
 }
 
+export function UnprocessableEntityError(
+  message = "Cannot process the request"
+) {
+  return httpErrors(422, message, { id: "unprocessable_entity" });
+}
+
 export function ClientClosedRequestError(
   message = "Client closed request before response was received"
 ) {

server/migrations/20250306181804-create-imports.js

@@ -0,0 +1,93 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.createTable(
+        "imports",
+        {
+          id: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            primaryKey: true,
+          },
+          name: {
+            type: Sequelize.STRING,
+            allowNull: false,
+          },
+          service: {
+            type: Sequelize.STRING,
+            allowNull: false,
+          },
+          state: {
+            type: Sequelize.STRING,
+            allowNull: false,
+          },
+          input: {
+            type: Sequelize.JSONB,
+            allowNull: false,
+          },
+          documentCount: {
+            type: Sequelize.INTEGER,
+            allowNull: false,
+            defaultValue: 0,
+          },
+          integrationId: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            references: {
+              model: "integrations",
+            },
+          },
+          createdById: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            references: {
+              model: "users",
+            },
+          },
+          teamId: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            references: {
+              model: "teams",
+            },
+          },
+          createdAt: {
+            type: Sequelize.DATE,
+            allowNull: false,
+          },
+          updatedAt: {
+            type: Sequelize.DATE,
+            allowNull: false,
+          },
+          deletedAt: {
+            type: Sequelize.DATE,
+            allowNull: true,
+          },
+        },
+        { transaction }
+      );
+
+      await queryInterface.addIndex("imports", ["service", "teamId"], {
+        transaction,
+      });
+      await queryInterface.addIndex("imports", ["state", "teamId"], {
+        transaction,
+      });
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.removeIndex("imports", ["service", "teamId"], {
+        transaction,
+      });
+      await queryInterface.removeIndex("imports", ["state", "teamId"], {
+        transaction,
+      });
+      await queryInterface.dropTable("imports", { transaction });
+    });
+  },
+};

server/migrations/20250310043011-create-import-tasks.js

@@ -0,0 +1,67 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.createTable(
+        "import_tasks",
+        {
+          id: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            primaryKey: true,
+          },
+          state: {
+            type: Sequelize.STRING,
+            allowNull: false,
+          },
+          input: {
+            type: Sequelize.JSONB,
+            allowNull: false,
+          },
+          output: {
+            type: Sequelize.JSONB,
+            allowNull: true,
+          },
+          importId: {
+            type: Sequelize.UUID,
+            allowNull: false,
+            onDelete: "cascade",
+            references: {
+              model: "imports",
+            },
+          },
+          createdAt: {
+            type: Sequelize.DATE,
+            allowNull: false,
+          },
+          updatedAt: {
+            type: Sequelize.DATE,
+            allowNull: false,
+          },
+        },
+        { transaction }
+      );
+
+      await queryInterface.addIndex("import_tasks", ["importId"], {
+        transaction,
+      });
+      await queryInterface.addIndex("import_tasks", ["state", "importId"], {
+        transaction,
+      });
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.removeIndex("import_tasks", ["importId"], {
+        transaction,
+      });
+      await queryInterface.removeIndex("import_tasks", ["state", "importId"], {
+        transaction,
+      });
+      await queryInterface.dropTable("import_tasks", { transaction });
+    });
+  },
+};

server/migrations/20250314105847-add-apiImportId-to-collections-and-documents.js

@@ -0,0 +1,59 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.addColumn(
+        "collections",
+        "apiImportId",
+        {
+          type: Sequelize.UUID,
+          allowNull: true,
+          references: {
+            model: "imports",
+          },
+        },
+        { transaction }
+      );
+      await queryInterface.addColumn(
+        "documents",
+        "apiImportId",
+        {
+          type: Sequelize.UUID,
+          allowNull: true,
+          references: {
+            model: "imports",
+          },
+        },
+        { transaction }
+      );
+
+      await queryInterface.addIndex("collections", ["apiImportId"], {
+        transaction,
+      });
+      await queryInterface.addIndex("documents", ["apiImportId"], {
+        transaction,
+      });
+    });
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.sequelize.transaction(async transaction => {
+      await queryInterface.removeIndex("collections", [
+        "apiImportId",
+        { transaction },
+      ]);
+      await queryInterface.removeIndex("documents", [
+        "apiImportId",
+        { transaction },
+      ]);
+      await queryInterface.removeColumn("collections", "apiImportId", {
+        transaction,
+      });
+      await queryInterface.removeColumn("documents", "apiImportId", {
+        transaction,
+      });
+    });
+  },
+};

server/models/Collection.ts

@@ -47,6 +47,7 @@ import FileOperation from "./FileOperation";
 import Group from "./Group";
 import GroupMembership from "./GroupMembership";
 import GroupUser from "./GroupUser";
+import Import from "./Import";
 import Team from "./Team";
 import User from "./User";
 import UserMembership from "./UserMembership";
@@ -351,6 +352,13 @@ class Collection extends ParanoidModel<
   @Column(DataType.UUID)
   importId: string | null;
 
+  @BelongsTo(() => Import, "apiImportId")
+  apiImport: Import<any> | null;
+
+  @ForeignKey(() => Import)
+  @Column(DataType.UUID)
+  apiImportId: string | null;
+
   @BelongsTo(() => User, "archivedById")
   archivedBy?: User | null;
 
@@ -713,6 +721,7 @@ class Collection extends ParanoidModel<
     index?: number,
     options: FindOptions & {
       save?: boolean;
+      silent?: boolean;
       documentJson?: NavigationNode;
       includeArchived?: boolean;
     } = {}

server/models/Document.ts

@@ -57,6 +57,7 @@ import FileOperation from "./FileOperation";
 import Group from "./Group";
 import GroupMembership from "./GroupMembership";
 import GroupUser from "./GroupUser";
+import Import from "./Import";
 import Revision from "./Revision";
 import Star from "./Star";
 import Team from "./Team";
@@ -537,6 +538,13 @@ class Document extends ArchivableModel<
   @Column(DataType.UUID)
   importId: string | null;
 
+  @BelongsTo(() => Import, "apiImportId")
+  apiImport: Import<any> | null;
+
+  @ForeignKey(() => Import)
+  @Column(DataType.UUID)
+  apiImportId: string | null;
+
   @AllowNull
   @Column(DataType.JSONB)
   sourceMetadata: SourceMetadata | null;

server/models/Import.ts

@@ -0,0 +1,87 @@
+import { InferAttributes, InferCreationAttributes } from "sequelize";
+import {
+  BelongsTo,
+  Column,
+  DataType,
+  Default,
+  DefaultScope,
+  ForeignKey,
+  IsIn,
+  IsNumeric,
+  Table,
+} from "sequelize-typescript";
+import { type ImportInput } from "@shared/schema";
+import { ImportableIntegrationService, ImportState } from "@shared/types";
+import { ImportValidation } from "@shared/validations";
+import Integration from "./Integration";
+import Team from "./Team";
+import User from "./User";
+import ParanoidModel from "./base/ParanoidModel";
+import Fix from "./decorators/Fix";
+import Length from "./validators/Length";
+import NotContainsUrl from "./validators/NotContainsUrl";
+
+@DefaultScope(() => ({
+  include: [
+    {
+      association: "createdBy",
+      required: true,
+      paranoid: false,
+    },
+  ],
+}))
+@Table({ tableName: "imports", modelName: "import" })
+@Fix
+class Import<T extends ImportableIntegrationService> extends ParanoidModel<
+  InferAttributes<Import<T>>,
+  Partial<InferCreationAttributes<Import<T>>>
+> {
+  @NotContainsUrl
+  @Length({
+    max: ImportValidation.maxNameLength,
+    msg: `name must be ${ImportValidation.maxNameLength} characters or less`,
+  })
+  @Column(DataType.STRING)
+  name: string;
+
+  @IsIn([Object.values(ImportableIntegrationService)])
+  @Column(DataType.STRING)
+  service: T;
+
+  @IsIn([Object.values(ImportState)])
+  @Column(DataType.STRING)
+  state: ImportState;
+
+  @Column(DataType.JSONB)
+  input: ImportInput<T>;
+
+  @IsNumeric
+  @Default(0)
+  @Column(DataType.INTEGER)
+  documentCount: number;
+
+  // associations
+
+  @BelongsTo(() => Integration, "integrationId")
+  integration: Integration;
+
+  @ForeignKey(() => Integration)
+  @Column(DataType.UUID)
+  integrationId: string;
+
+  @BelongsTo(() => User, "createdById")
+  createdBy: User;
+
+  @ForeignKey(() => User)
+  @Column(DataType.UUID)
+  createdById: string;
+
+  @BelongsTo(() => Team, "teamId")
+  team: Team;
+
+  @ForeignKey(() => Team)
+  @Column(DataType.UUID)
+  teamId: string;
+}
+
+export default Import;

server/models/ImportTask.ts

@@ -0,0 +1,58 @@
+import { InferAttributes, InferCreationAttributes } from "sequelize";
+import {
+  AllowNull,
+  BelongsTo,
+  Column,
+  DataType,
+  ForeignKey,
+  IsIn,
+  Table,
+} from "sequelize-typescript";
+import { type ImportTaskInput, ImportTaskOutput } from "@shared/schema";
+import { ImportableIntegrationService, ImportTaskState } from "@shared/types";
+import Import from "./Import";
+import IdModel from "./base/IdModel";
+import Fix from "./decorators/Fix";
+
+// Not all fields are automatically inferred by Sequelize.
+// see https://sequelize.org/docs/v7/models/model-typing/#manual-attribute-typing
+type NonInferredAttributes<T extends ImportableIntegrationService> = {
+  input: ImportTaskInput<T>;
+};
+
+export type ImportTaskAttributes<T extends ImportableIntegrationService> =
+  InferAttributes<ImportTask<T>> & NonInferredAttributes<T>;
+
+export type ImportTaskCreationAttributes<
+  T extends ImportableIntegrationService
+> = Partial<InferCreationAttributes<ImportTask<T>>> &
+  Partial<NonInferredAttributes<T>>;
+
+@Table({ tableName: "import_tasks", modelName: "import_task" })
+@Fix
+class ImportTask<T extends ImportableIntegrationService> extends IdModel<
+  ImportTaskAttributes<T>,
+  ImportTaskCreationAttributes<T>
+> {
+  @IsIn([Object.values(ImportTaskState)])
+  @Column(DataType.STRING)
+  state: ImportTaskState;
+
+  @Column(DataType.JSONB)
+  input: ImportTaskInput<T>;
+
+  @AllowNull
+  @Column(DataType.JSONB)
+  output: ImportTaskOutput | null;
+
+  // associations
+
+  @BelongsTo(() => Import, "importId")
+  import: Import<T>;
+
+  @ForeignKey(() => Import)
+  @Column(DataType.UUID)
+  importId: string;
+}
+
+export default ImportTask;

server/models/index.ts

@@ -24,6 +24,10 @@ export { default as Group } from "./Group";
 
 export { default as GroupUser } from "./GroupUser";
 
+export { default as Import } from "./Import";
+
+export { default as ImportTask } from "./ImportTask";
+
 export { default as Integration } from "./Integration";
 
 export { default as IntegrationAuthentication } from "./IntegrationAuthentication";

server/policies/fileOperation.ts

@@ -5,7 +5,7 @@ import { and, isTeamAdmin, isTeamMutable, or } from "./utils";
 
 allow(
   User,
-  ["createFileOperation", "createImport", "createExport"],
+  ["createFileOperation", "createExport"],
   Team,
   // Note: Not checking for isTeamMutable here because we want to allow exporting data in read-only.
   isTeamAdmin

server/policies/import.ts

@@ -0,0 +1,31 @@
+import { ImportState } from "@shared/types";
+import { User, Team, Import } from "@server/models";
+import { allow, can } from "./cancan";
+import { and, isTeamAdmin, isTeamMutable, or } from "./utils";
+
+allow(User, ["createImport", "listImports"], Team, (actor, team) =>
+  and(isTeamAdmin(actor, team), isTeamMutable(actor))
+);
+
+allow(User, "read", Import, (actor, importModel) =>
+  and(isTeamAdmin(actor, importModel), isTeamMutable(actor))
+);
+
+allow(User, "delete", Import, (actor, importModel) =>
+  and(
+    can(actor, "read", importModel),
+    importModel?.state === ImportState.Completed
+  )
+);
+
+allow(User, "cancel", Import, (actor, importModel) =>
+  and(
+    can(actor, "read", importModel),
+    importModel?.createdById === actor.id,
+    or(
+      importModel?.state === ImportState.Created,
+      importModel?.state === ImportState.InProgress,
+      importModel?.state === ImportState.Processed
+    )
+  )
+);

server/policies/index.ts

@@ -9,6 +9,7 @@ import "./collection";
 import "./comment";
 import "./document";
 import "./fileOperation";
+import "./import";
 import "./integration";
 import "./pins";
 import "./reaction";

server/presenters/import.ts

@@ -0,0 +1,19 @@
+import { Import } from "@server/models";
+import presentUser from "./user";
+
+export default function presentImport(
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  importModel: Import<any>
+) {
+  return {
+    id: importModel.id,
+    name: importModel.name,
+    service: importModel.service,
+    state: importModel.state,
+    documentCount: importModel.documentCount,
+    createdBy: presentUser(importModel.createdBy),
+    createdById: importModel.createdById,
+    createdAt: importModel.createdAt,
+    updatedAt: importModel.updatedAt,
+  };
+}

server/presenters/index.ts

@@ -10,6 +10,7 @@ import presentFileOperation from "./fileOperation";
 import presentGroup from "./group";
 import presentGroupMembership from "./groupMembership";
 import presentGroupUser from "./groupUser";
+import presentImport from "./import";
 import presentIntegration from "./integration";
 import presentMembership from "./membership";
 import presentPin from "./pin";
@@ -39,6 +40,7 @@ export {
   presentGroup,
   presentGroupUser,
   presentGroupMembership,
+  presentImport,
   presentIntegration,
   presentMembership,
   presentPublicTeam,

server/queues/processors/BaseProcessor.ts

@@ -4,4 +4,15 @@ export default abstract class BaseProcessor {
   static applicableEvents: (Event["name"] | "*")[] = [];
 
   public abstract perform(event: Event): Promise<void>;
+
+  /**
+   * Handle failure when all attempts are exhausted for the processor.
+   *
+   * @param event processor event
+   * @returns A promise that resolves once the processor handles the failure.
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public onFailed(event: Event): Promise<void> {
+    return Promise.resolve();
+  }
 }

server/queues/processors/ImportsProcessor.ts

@@ -0,0 +1,529 @@
+import fractionalIndex from "fractional-index";
+import chunk from "lodash/chunk";
+import keyBy from "lodash/keyBy";
+import truncate from "lodash/truncate";
+import { Fragment, Node } from "prosemirror-model";
+import { CreateOptions, CreationAttributes, Transaction } from "sequelize";
+import { v4 as uuidv4 } from "uuid";
+import { randomElement } from "@shared/random";
+import { ImportInput, ImportTaskInput } from "@shared/schema";
+import {
+  ImportableIntegrationService,
+  ImportState,
+  ImportTaskState,
+  MentionType,
+  ProsemirrorData,
+  ProsemirrorDoc,
+} from "@shared/types";
+import { colorPalette } from "@shared/utils/collections";
+import { CollectionValidation } from "@shared/validations";
+import collectionDestroyer from "@server/commands/collectionDestroyer";
+import { createContext } from "@server/context";
+import { schema } from "@server/editor";
+import Logger from "@server/logging/Logger";
+import { Attachment, Collection, Document, Import, User } from "@server/models";
+import ImportTask, {
+  ImportTaskAttributes,
+  ImportTaskCreationAttributes,
+} from "@server/models/ImportTask";
+import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { sequelize } from "@server/storage/database";
+import { Event, ImportEvent } from "@server/types";
+import BaseProcessor from "./BaseProcessor";
+
+export const PagePerImportTask = 3;
+
+export default abstract class ImportsProcessor<
+  T extends ImportableIntegrationService
+> extends BaseProcessor {
+  static applicableEvents: Event["name"][] = [
+    "imports.create",
+    "imports.processed",
+    "imports.delete",
+  ];
+
+  /**
+   * Run the import processor.
+   *
+   * @param event The import event
+   */
+  public async perform(event: ImportEvent) {
+    await sequelize.transaction(async (transaction) => {
+      const importModel = await Import.findByPk<Import<T>>(event.modelId, {
+        rejectOnEmpty: true,
+        paranoid: false,
+        transaction,
+        lock: transaction.LOCK.UPDATE,
+      });
+
+      if (
+        !this.canProcess(importModel) ||
+        importModel.state === ImportState.Errored ||
+        importModel.state === ImportState.Canceled
+      ) {
+        return;
+      }
+
+      switch (event.name) {
+        case "imports.create":
+          return this.onCreation(importModel, transaction);
+
+        case "imports.processed":
+          return this.onProcessed(importModel, transaction);
+
+        case "imports.delete":
+          return this.onDeletion(importModel, event, transaction);
+      }
+    });
+  }
+
+  public async onFailed(event: ImportEvent) {
+    await sequelize.transaction(async (transaction) => {
+      const importModel = await Import.findByPk(event.modelId, {
+        rejectOnEmpty: true,
+      });
+
+      importModel.state = ImportState.Errored;
+      await importModel.saveWithCtx(
+        createContext({
+          user: importModel.createdBy,
+          transaction,
+        })
+      );
+    });
+  }
+
+  /**
+   * Handle "imports.create" event.
+   *
+   * @param importModel Import model associated with the event.
+   * @param transaction Sequelize transaction.
+   * @returns Promise that resolves when the creation flow setup is done.
+   */
+  private async onCreation(importModel: Import<T>, transaction: Transaction) {
+    if (!importModel.input.length) {
+      return;
+    }
+
+    const tasksInput = await this.buildTasksInput(importModel, transaction);
+
+    const importTasks = await Promise.all(
+      chunk(tasksInput, PagePerImportTask).map((input) => {
+        const attrs = {
+          state: ImportTaskState.Created,
+          input,
+          importId: importModel.id,
+        } as ImportTaskCreationAttributes<T>;
+
+        return ImportTask.create<
+          ImportTask<T>,
+          CreateOptions<ImportTaskAttributes<T>>
+        >(attrs as unknown as CreationAttributes<ImportTask<T>>, {
+          transaction,
+        });
+      })
+    );
+
+    importModel.state = ImportState.InProgress;
+    await importModel.save({ transaction });
+
+    transaction.afterCommit(() => this.scheduleTask(importTasks[0]));
+  }
+
+  /**
+   * Handle "imports.processed" event.
+   * This event is received when all the tasks for the import has been completed.
+   * This method is responsible for persisting the collections and documents associated with the import.
+   *
+   * @param importModel Import model associated with the event.
+   * @param transaction Sequelize transaction.
+   * @returns Promise that resolves when mapping and persistence is completed.
+   */
+  private async onProcessed(importModel: Import<T>, transaction: Transaction) {
+    const { collections } = await this.createCollectionsAndDocuments({
+      importModel,
+      transaction,
+    });
+
+    // Once all collections and documents are created, update collection's document structure.
+    // This ensures the root documents have the whole subtree available in the structure.
+    for (const collection of collections) {
+      await Document.unscoped().findAllInBatches<Document>(
+        {
+          where: { parentDocumentId: null, collectionId: collection.id },
+          order: [
+            ["createdAt", "DESC"],
+            ["id", "ASC"],
+          ],
+          transaction,
+        },
+        async (documents) => {
+          for (const document of documents) {
+            await collection.addDocumentToStructure(document, 0, {
+              save: false,
+              silent: true,
+              transaction,
+            });
+          }
+        }
+      );
+
+      await collection.save({ silent: true, transaction });
+    }
+
+    importModel.state = ImportState.Completed;
+    await importModel.saveWithCtx(
+      createContext({
+        user: importModel.createdBy,
+        transaction,
+      })
+    );
+  }
+
+  /**
+   * Handle "imports.delete" event.
+   * This method is responsible for deleting the collections and documents associated with the import.
+   *
+   * @param importModel Import model associated with the event.
+   * @param event Received event.
+   * @param transaction Sequelize transaction.
+   * @returns Promise that resolves when the collections and documents are deleted.
+   */
+  private async onDeletion(
+    importModel: Import<T>,
+    event: ImportEvent,
+    transaction: Transaction
+  ) {
+    if (importModel.state !== ImportState.Completed) {
+      return;
+    }
+
+    const user = await User.findByPk(event.actorId, {
+      rejectOnEmpty: true,
+      paranoid: false,
+      transaction,
+    });
+
+    const collections = await Collection.findAll({
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+      where: {
+        teamId: importModel.teamId,
+        apiImportId: importModel.id,
+      },
+    });
+
+    for (const collection of collections) {
+      Logger.debug("processor", "Destroying collection created from import", {
+        collectionId: collection.id,
+      });
+
+      await collectionDestroyer({
+        collection,
+        transaction,
+        user,
+        ip: event.ip,
+      });
+    }
+  }
+
+  /**
+   * Create collections and documents associated with the import.
+   *
+   * @param importModel Import model associated with the event.
+   * @param transaction Sequelize transaction.
+   * @returns Promise of collection models that are created.
+   */
+  private async createCollectionsAndDocuments({
+    importModel,
+    transaction,
+  }: {
+    importModel: Import<T>;
+    transaction: Transaction;
+  }): Promise<{ collections: Collection[] }> {
+    const now = new Date();
+    const createdCollections: Collection[] = [];
+    // External id to internal model id.
+    const idMap: Record<string, string> = {};
+    // These will be imported as collections.
+    const importInput = keyBy(importModel.input, "externalId");
+    const ctx = createContext({ user: importModel.createdBy, transaction });
+
+    const firstCollection = await Collection.findFirstCollectionForUser(
+      importModel.createdBy,
+      {
+        attributes: ["index"],
+        transaction,
+      }
+    );
+
+    let collectionIdx = firstCollection?.index ?? null;
+
+    await ImportTask.findAllInBatches<ImportTask<T>>(
+      {
+        where: { importId: importModel.id },
+        order: [
+          ["createdAt", "ASC"],
+          ["id", "ASC"], // for stable order when multiple tasks have same "createdAt" value.
+        ], // ordering ensures collections are created first.
+        batchLimit: 5, // output data per task could be huge, keep a low batch size to prevent OOM.
+        transaction,
+      },
+      async (importTasks) => {
+        for (const importTask of importTasks) {
+          const outputMap = keyBy(importTask.output ?? [], "externalId");
+
+          for (const input of importTask.input) {
+            const externalId = input.externalId;
+            const internalId = this.getInternalId(externalId, idMap);
+
+            const parentExternalId = input.parentExternalId;
+            const parentInternalId = parentExternalId
+              ? this.getInternalId(parentExternalId, idMap)
+              : undefined;
+
+            const collectionExternalId = input.collectionExternalId;
+            const collectionInternalId = collectionExternalId
+              ? this.getInternalId(collectionExternalId, idMap)
+              : undefined;
+
+            const output = outputMap[externalId];
+
+            const collectionItem = importInput[externalId];
+
+            const attachments = await Attachment.findAll({
+              attributes: ["id", "size"],
+              where: { documentId: externalId }, // This will be set for root pages too (which will be imported as collection)
+              transaction,
+            });
+
+            const transformedContent = this.updateMentionsAndAttachments({
+              content: output.content,
+              attachments,
+              importInput,
+              idMap,
+              actorId: importModel.createdById,
+            });
+
+            if (collectionItem) {
+              // imported collection will be placed in the beginning.
+              collectionIdx = fractionalIndex(null, collectionIdx);
+
+              const description = DocumentHelper.toMarkdown(
+                transformedContent,
+                {
+                  includeTitle: false,
+                }
+              );
+
+              const collection = Collection.build({
+                id: internalId,
+                name: output.title,
+                icon: output.emoji ?? "collection",
+                color: output.emoji ? undefined : randomElement(colorPalette),
+                content: transformedContent,
+                description: truncate(description, {
+                  length: CollectionValidation.maxDescriptionLength,
+                }),
+                createdById: importModel.createdById,
+                teamId: importModel.createdBy.teamId,
+                apiImportId: importModel.id,
+                index: collectionIdx,
+                sort: Collection.DEFAULT_SORT,
+                permission: collectionItem.permission,
+                createdAt: output.createdAt ?? now,
+                updatedAt: output.updatedAt ?? now,
+              });
+
+              await collection.saveWithCtx(
+                ctx,
+                { silent: true },
+                {
+                  name: "create",
+                  data: { name: output.title, source: "import" },
+                }
+              );
+
+              createdCollections.push(collection);
+
+              // Unset documentId for attachments in collection overview.
+              await Attachment.update(
+                { documentId: null },
+                { where: { documentId: externalId }, silent: true, transaction }
+              );
+
+              continue;
+            }
+
+            // Document at the root of a collection when there's no parent (or) the parent is the collection.
+            const isRootDocument =
+              !parentExternalId || !!importInput[parentExternalId];
+
+            const document = Document.build({
+              id: internalId,
+              title: output.title,
+              icon: output.emoji,
+              content: transformedContent,
+              text: DocumentHelper.toMarkdown(transformedContent, {
+                includeTitle: false,
+              }),
+              collectionId: collectionInternalId,
+              parentDocumentId: isRootDocument ? undefined : parentInternalId,
+              createdById: importModel.createdById,
+              lastModifiedById: importModel.createdById,
+              teamId: importModel.createdBy.teamId,
+              apiImportId: importModel.id,
+              sourceMetadata: {
+                externalId,
+                externalName: output.title,
+                createdByName: output.author,
+              },
+              createdAt: output.createdAt ?? now,
+              updatedAt: output.updatedAt ?? now,
+              publishedAt: output.updatedAt ?? output.createdAt ?? now,
+            });
+
+            await document.saveWithCtx(
+              ctx,
+              { silent: true },
+              {
+                name: "create",
+                data: { title: output.title, source: "import" },
+              }
+            );
+
+            // Update document id for attachments in document content.
+            await Attachment.update(
+              { documentId: internalId },
+              { where: { documentId: externalId }, silent: true, transaction }
+            );
+          }
+        }
+      }
+    );
+
+    return { collections: createdCollections };
+  }
+
+  /**
+   * Transform the mentions and attachments in ProseMirrorDoc to their internal references.
+   *
+   * @param content ProseMirrorDoc that represents collection (or) document content.
+   * @param attachments Array of attachment models created for the import.
+   * @param idMap Map of internalId to externalId.
+   * @param importInput Contains the root externalId and associated info which were used to create the import.
+   * @param actorId ID of the user who created the import.
+   * @returns Updated ProseMirrorDoc.
+   */
+  private updateMentionsAndAttachments({
+    content,
+    attachments,
+    idMap,
+    importInput,
+    actorId,
+  }: {
+    content: ProsemirrorDoc;
+    attachments: Attachment[];
+    idMap: Record<string, string>;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    importInput: Record<string, ImportInput<any>[number]>;
+    actorId: string;
+  }): ProsemirrorDoc {
+    // special case when the doc content is empty
+    if (!content.content.length) {
+      return content;
+    }
+
+    const attachmentsMap = keyBy(attachments, "id");
+    const doc = ProsemirrorHelper.toProsemirror(content);
+
+    const transformMentionNode = (node: Node): Node => {
+      const json = node.toJSON() as ProsemirrorData;
+      const attrs = json.attrs ?? {};
+
+      attrs.id = uuidv4();
+      attrs.actorId = actorId;
+
+      const externalId = attrs.modelId as string;
+      attrs.modelId = this.getInternalId(externalId, idMap);
+
+      const isCollectionMention = !!importInput[externalId]; // the referenced externalId is a root page.
+      attrs.type = isCollectionMention
+        ? MentionType.Collection
+        : MentionType.Document;
+
+      json.attrs = attrs;
+      return Node.fromJSON(schema, json);
+    };
+
+    const transformAttachmentNode = (node: Node): Node => {
+      const json = node.toJSON() as ProsemirrorData;
+      const attrs = json.attrs ?? {};
+
+      attrs.size = attachmentsMap[attrs.id as string].size;
+
+      json.attrs = attrs;
+      return Node.fromJSON(schema, json);
+    };
+
+    const transformFragment = (fragment: Fragment): Fragment => {
+      const nodes: Node[] = [];
+
+      fragment.forEach((node) => {
+        nodes.push(
+          node.type.name === "mention"
+            ? transformMentionNode(node)
+            : node.type.name === "attachment"
+            ? transformAttachmentNode(node)
+            : node.copy(transformFragment(node.content))
+        );
+      });
+
+      return Fragment.fromArray(nodes);
+    };
+
+    return doc.copy(transformFragment(doc.content)).toJSON();
+  }
+
+  /**
+   * Get internalId for the given externalId.
+   * Returned internalId will be used as "id" for collections and documents created in the import.
+   *
+   * @param externalId externalId from a source.
+   * @param idMap Map of internalId to externalId.
+   * @returns Mapped internalId.
+   */
+  private getInternalId(externalId: string, idMap: Record<string, string>) {
+    const internalId = idMap[externalId] ?? uuidv4();
+    idMap[externalId] = internalId;
+    return internalId;
+  }
+
+  /**
+   * Determine whether this import can be processed by this processor.
+   *
+   * @param importModel Import model associated with the import.
+   * @returns boolean.
+   */
+  protected abstract canProcess(importModel: Import<T>): boolean;
+
+  /**
+   * Build task inputs which will be used for `APIImportTask`s.
+   *
+   * @param importInput Array of root externalId and associated info which were used to create the import.
+   * @returns `ImportTaskInput`.
+   */
+  protected abstract buildTasksInput(
+    importModel: Import<T>,
+    transaction: Transaction
+  ): Promise<ImportTaskInput<T>>;
+
+  /**
+   * Schedule the first `APIImportTask` for the import.
+   *
+   * @param importTask ImportTask model associated with the `APIImportTask`.
+   * @returns Promise that resolves when the task is scheduled.
+   */
+  protected abstract scheduleTask(importTask: ImportTask<T>): Promise<void>;
+}

server/queues/processors/WebsocketsProcessor.ts

@@ -17,6 +17,7 @@ import {
   Notification,
   UserMembership,
   User,
+  Import,
 } from "@server/models";
 import { cannot } from "@server/policies";
 import {
@@ -33,6 +34,7 @@ import {
   presentUser,
   presentGroupMembership,
   presentGroupUser,
+  presentImport,
 } from "@server/presenters";
 import presentNotification from "@server/presenters/notification";
 import { Event } from "../../types";
@@ -49,7 +51,10 @@ export default class WebsocketsProcessor {
         if (!document) {
           return;
         }
-        if (event.name === "documents.create" && document.importId) {
+        if (
+          event.name === "documents.create" &&
+          event.data.source === "import"
+        ) {
           return;
         }
 
@@ -453,6 +458,18 @@ export default class WebsocketsProcessor {
           .emit(event.name, presentFileOperation(fileOperation));
       }
 
+      case "imports.create":
+      case "imports.update": {
+        const importModel = await Import.findByPk(event.modelId);
+        if (!importModel) {
+          return;
+        }
+
+        return socketio
+          .to(`user-${event.actorId}`)
+          .emit(event.name, presentImport(importModel));
+      }
+
       case "pins.create":
       case "pins.update": {
         const pin = await Pin.findByPk(event.modelId);

server/queues/processors/index.ts

@@ -4,9 +4,11 @@ import BaseProcessor from "./BaseProcessor";
 
 const processors: Record<string, typeof BaseProcessor> = {};
 
+const AbstractProcessors = ["ImportsProcessor"];
+
 requireDirectory<{ default: typeof BaseProcessor }>(__dirname).forEach(
   ([module, id]) => {
-    if (id === "index") {
+    if (id === "index" || AbstractProcessors.includes(id)) {
       return;
     }
     processors[id] = module.default;

server/queues/tasks/APIImportTask.ts

@@ -0,0 +1,391 @@
+import { JobOptions } from "bull";
+import chunk from "lodash/chunk";
+import uniqBy from "lodash/uniqBy";
+import { Fragment, Node } from "prosemirror-model";
+import { Transaction, WhereOptions } from "sequelize";
+import { v4 as uuidv4 } from "uuid";
+import { ImportTaskInput, ImportTaskOutput } from "@shared/schema";
+import {
+  AttachmentPreset,
+  ImportableIntegrationService,
+  ImportState,
+  ImportTaskState,
+  ProsemirrorData,
+  ProsemirrorDoc,
+} from "@shared/types";
+import { ProsemirrorHelper as SharedProseMirrorHelper } from "@shared/utils/ProsemirrorHelper";
+import { createContext } from "@server/context";
+import { schema } from "@server/editor";
+import Logger from "@server/logging/Logger";
+import { Attachment, Import, ImportTask, User } from "@server/models";
+import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
+import { sequelize } from "@server/storage/database";
+import { PagePerImportTask } from "../processors/ImportsProcessor";
+import BaseTask, { TaskPriority } from "./BaseTask";
+import UploadAttachmentsForImportTask from "./UploadAttachmentsForImportTask";
+
+export type ProcessOutput<T extends ImportableIntegrationService> = {
+  taskOutput: ImportTaskOutput;
+  childTasksInput: ImportTaskInput<T>;
+};
+
+type Props = {
+  /** id of the import_task */
+  importTaskId: string;
+};
+
+export default abstract class APIImportTask<
+  T extends ImportableIntegrationService
+> extends BaseTask<Props> {
+  /**
+   * Run the import task.
+   *
+   * @param importTaskId id of the import_task model.
+   * @returns Promise that resolves once the task has completed.
+   */
+  public async perform({ importTaskId }: Props) {
+    let importTask = await ImportTask.findByPk<ImportTask<T>>(importTaskId, {
+      rejectOnEmpty: true,
+      include: [
+        {
+          model: Import,
+          as: "import",
+          required: true,
+        },
+      ],
+    });
+
+    // Don't process any further when the associated import is canceled by the user.
+    if (importTask.import.state === ImportState.Canceled) {
+      importTask.state = ImportTaskState.Canceled;
+      await importTask.save();
+      return;
+    }
+
+    switch (importTask.state) {
+      case ImportTaskState.Created: {
+        importTask.state = ImportTaskState.InProgress;
+        importTask = await importTask.save();
+        return await this.onProcess(importTask);
+      }
+
+      case ImportTaskState.InProgress:
+        return await this.onProcess(importTask);
+
+      case ImportTaskState.Completed:
+        return await this.onCompletion(importTask);
+
+      default:
+    }
+  }
+
+  /**
+   * Handle failure when all attempts of APIImportTask has failed.
+   *
+   * @param importTaskId id of the import_task model.
+   * @returns Promise that resolves once failure has been handled.
+   */
+  public async onFailed({ importTaskId }: Props) {
+    await sequelize.transaction(async (transaction) => {
+      const importTask = await ImportTask.findByPk<ImportTask<T>>(
+        importTaskId,
+        {
+          rejectOnEmpty: true,
+          include: [
+            {
+              model: Import,
+              as: "import",
+              required: true,
+            },
+          ],
+          transaction,
+          lock: Transaction.LOCK.UPDATE,
+        }
+      );
+
+      importTask.state = ImportTaskState.Errored;
+      await importTask.save({ transaction });
+
+      const associatedImport = importTask.import;
+      associatedImport.state = ImportState.Errored;
+      await associatedImport.saveWithCtx(
+        createContext({
+          user: associatedImport.createdBy,
+          transaction,
+        })
+      );
+    });
+  }
+
+  /**
+   * Creation flow for the task.
+   * This fetches data from external source, stores the task output and creates subsequent import_task models.
+   *
+   * @param importTask import_task model to process.
+   * @returns Promise that resolves once processing has completed.
+   */
+  private async onProcess(importTask: ImportTask<T>) {
+    const { taskOutput, childTasksInput } = await this.process(importTask);
+
+    const taskOutputWithReplacements = await Promise.all(
+      taskOutput.map(async (item) => ({
+        ...item,
+        content: await this.uploadAttachments({
+          doc: item.content,
+          externalId: item.externalId,
+          createdBy: importTask.import.createdBy,
+        }),
+      }))
+    );
+
+    await sequelize.transaction(async (transaction) => {
+      await Promise.all(
+        chunk(childTasksInput, PagePerImportTask).map(async (input) => {
+          await ImportTask.create(
+            {
+              state: ImportTaskState.Created,
+              input,
+              importId: importTask.importId,
+            },
+            { transaction }
+          );
+        })
+      );
+
+      importTask.output = taskOutputWithReplacements;
+      importTask.state = ImportTaskState.Completed;
+      await importTask.save({ transaction });
+
+      const associatedImport = importTask.import;
+      associatedImport.documentCount += importTask.input.length;
+      await associatedImport.saveWithCtx(
+        createContext({
+          user: associatedImport.createdBy,
+          transaction,
+        })
+      );
+    });
+
+    await this.scheduleNextTask(importTask);
+  }
+
+  /**
+   * Completion flow for the task.
+   * This determines if there are any more import_tasks to process (or) all tasks for the import have been processed, and schedules the next step.
+   *
+   * @param importTask import_task model to process.
+   * @returns Promise that resolves once processing has completed.
+   */
+  private async onCompletion(importTask: ImportTask<T>) {
+    const where: WhereOptions<ImportTask<T>> = {
+      state: ImportTaskState.Created,
+      importId: importTask.importId,
+    };
+
+    const nextImportTask = await ImportTask.findOne<ImportTask<T>>({
+      where,
+      order: [["createdAt", "ASC"]],
+    });
+
+    // Tasks available to process for this import.
+    if (nextImportTask) {
+      return await this.scheduleNextTask(nextImportTask);
+    }
+
+    // All tasks for this import have been processed.
+    await sequelize.transaction(async (transaction) => {
+      const associatedImport = importTask.import;
+      associatedImport.state = ImportState.Processed;
+      await associatedImport.saveWithCtx(
+        createContext({
+          user: associatedImport.createdBy,
+          transaction,
+        }),
+        undefined,
+        { name: "processed" }
+      );
+    });
+  }
+
+  /**
+   * Process the import task.
+   * This fetches data from external source and converts it to task output.
+   *
+   * @param importTask ImportTask model to process.
+   * @returns Promise with output that resolves once processing has completed.
+   */
+  protected abstract process(
+    importTask: ImportTask<T>
+  ): Promise<ProcessOutput<T>>;
+
+  /**
+   * Schedule the next `APIImportTask`.
+   *
+   * @param importTask ImportTask model associated with the `APIImportTask`.
+   * @returns Promise that resolves when the task is scheduled.
+   */
+  protected abstract scheduleNextTask(importTask: ImportTask<T>): Promise<void>;
+
+  /**
+   * Upload attachments found in the external document.
+   *
+   * @param doc ProseMirrorDoc that represents collection (or) document content.
+   * @param externalId id of the document in the external service.
+   * @param createdBy user who created the import.
+   * @returns Updated ProseMirrorDoc.
+   */
+  private async uploadAttachments({
+    doc,
+    externalId,
+    createdBy,
+  }: {
+    doc: ProsemirrorDoc;
+    externalId: string;
+    createdBy: User;
+  }): Promise<ProsemirrorDoc> {
+    const docNode = ProsemirrorHelper.toProsemirror(doc);
+    const nodes = [
+      ...SharedProseMirrorHelper.getImages(docNode),
+      ...SharedProseMirrorHelper.getVideos(docNode),
+      ...SharedProseMirrorHelper.getAttachments(docNode),
+    ];
+
+    if (!nodes.length) {
+      return doc;
+    }
+
+    const urlToAttachment: Record<string, Attachment> = {};
+
+    // perf: dedup url.
+    const attachmentsData = uniqBy(
+      nodes.map((node) => {
+        const url = String(
+          node.type.name === "attachment" ? node.attrs.href : node.attrs.src
+        );
+        const name = String(
+          node.type.name === "image" ? node.attrs.alt : node.attrs.title
+        ).trim();
+
+        return { url, name: name.length !== 0 ? name : node.type.name };
+      }),
+      "url"
+    );
+
+    await sequelize.transaction(async (transaction) => {
+      const dbPromises = attachmentsData.map(async (item) => {
+        const modelId = uuidv4();
+        const acl = AttachmentHelper.presetToAcl(
+          AttachmentPreset.DocumentAttachment
+        );
+        const key = AttachmentHelper.getKey({
+          acl,
+          id: modelId,
+          name: item.name,
+          userId: createdBy.id,
+        });
+
+        const attachment = await Attachment.create(
+          {
+            id: modelId,
+            key,
+            acl,
+            size: 0,
+            expiresAt: AttachmentHelper.presetToExpiry(
+              AttachmentPreset.DocumentAttachment
+            ),
+            contentType: "application/octet-stream",
+            documentId: externalId,
+            teamId: createdBy.teamId,
+            userId: createdBy.id,
+          },
+          { transaction }
+        );
+
+        urlToAttachment[item.url] = attachment;
+      });
+
+      return await Promise.all(dbPromises);
+    });
+
+    try {
+      const uploadItems = Object.entries(urlToAttachment).map(
+        ([url, attachment]) => ({ attachmentId: attachment.id, url })
+      );
+      // publish task after attachments are persisted in DB.
+      const job = await UploadAttachmentsForImportTask.schedule(uploadItems);
+      await job.finished();
+    } catch (err) {
+      // upload attachments failure is not critical enough to fail the whole import.
+      Logger.error(
+        `upload attachment task failed for externalId ${externalId}`,
+        err
+      );
+    }
+
+    return this.replaceAttachmentUrls(docNode, urlToAttachment).toJSON();
+  }
+
+  /**
+   * Replace remote url to internal redirect url for attachments.
+   *
+   * @param doc ProseMirror node that represents collection (or) document content.
+   * @param urlToAttachment Map of remote url to attachment model.
+   * @returns Updated Prosemirror node.
+   */
+  private replaceAttachmentUrls(
+    doc: Node,
+    urlToAttachment: Record<string, Attachment>
+  ): Node {
+    const attachmentTypes = ["attachment", "image", "video"];
+
+    const transformAttachmentNode = (node: Node): Node => {
+      const json = node.toJSON() as ProsemirrorData;
+      const attrs = json.attrs ?? {};
+
+      if (node.type.name === "attachment") {
+        const attachmentModel = urlToAttachment[attrs.href as string];
+        // attachment node uses 'href' attribute.
+        attrs.href = attachmentModel.redirectUrl;
+        // attachment node can have id.
+        attrs.id = attachmentModel.id;
+      } else if (node.type.name === "image" || node.type.name === "video") {
+        // image & video nodes use 'src' attribute.
+        attrs.src = urlToAttachment[attrs.src as string].redirectUrl;
+      }
+
+      json.attrs = attrs;
+      return Node.fromJSON(schema, json);
+    };
+
+    const transformFragment = (fragment: Fragment): Fragment => {
+      const nodes: Node[] = [];
+
+      fragment.forEach((node) => {
+        nodes.push(
+          attachmentTypes.includes(node.type.name)
+            ? transformAttachmentNode(node)
+            : node.copy(transformFragment(node.content))
+        );
+      });
+
+      return Fragment.fromArray(nodes);
+    };
+
+    return doc.copy(transformFragment(doc.content));
+  }
+
+  /**
+   * Job options such as priority and retry strategy, as defined by Bull.
+   */
+  public get options(): JobOptions {
+    return {
+      priority: TaskPriority.Normal,
+      attempts: 3,
+      backoff: {
+        type: "exponential",
+        delay: 60 * 1000,
+      },
+    };
+  }
+}

server/queues/tasks/BaseTask.ts

@@ -47,6 +47,17 @@ export default abstract class BaseTask<T extends Record<string, any>> {
    */
   public abstract perform(props: T): Promise<any>;
 
+  /**
+   * Handle failure when all attempts are exhausted for the task.
+   *
+   * @param props Properties to be used by the task
+   * @returns A promise that resolves once the task handles the failure.
+   */
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  public onFailed(props: T): Promise<void> {
+    return Promise.resolve();
+  }
+
   /**
    * Job options such as priority and retry strategy, as defined by Bull.
    */

server/queues/tasks/CleanupOldImportsTask.ts

@@ -0,0 +1,84 @@
+import { subDays } from "date-fns";
+import { Op } from "sequelize";
+import { ImportState } from "@shared/types";
+import Logger from "@server/logging/Logger";
+import { Import, ImportTask } from "@server/models";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
+
+type Props = Record<string, never>;
+
+/**
+ * A task that deletes the import_tasks for old imports which are completed, errored (or) canceled.
+ */
+export default class CleanupOldImportsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Day;
+
+  public async perform() {
+    // TODO: Hardcoded right now, configurable later
+    const retentionDays = 1;
+    const cutoffDate = subDays(new Date(), retentionDays);
+    const maxImportsPerTask = 1000;
+    let totalTasksDeleted = 0;
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      await Import.findAllInBatches<Import<any>>(
+        {
+          attributes: ["id"],
+          where: {
+            state: [
+              ImportState.Completed,
+              ImportState.Errored,
+              ImportState.Canceled,
+            ],
+            createdAt: {
+              [Op.lt]: cutoffDate,
+            },
+          },
+          order: [
+            ["createdAt", "ASC"],
+            ["id", "ASC"],
+          ],
+          batchLimit: 50,
+          totalLimit: maxImportsPerTask,
+        },
+        async (imports) => {
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+          await ImportTask.findAllInBatches<ImportTask<any>>(
+            {
+              attributes: ["id"],
+              where: {
+                importId: imports.map((importModel) => importModel.id),
+              },
+              order: [
+                ["createdAt", "ASC"],
+                ["id", "ASC"],
+              ],
+              batchLimit: 1000,
+            },
+            async (importTasks) => {
+              totalTasksDeleted += await ImportTask.destroy({
+                where: {
+                  id: importTasks.map((importTask) => importTask.id),
+                },
+              });
+            }
+          );
+        }
+      );
+    } finally {
+      if (totalTasksDeleted > 0) {
+        Logger.info("task", `Deleted old import_tasks`, {
+          totalTasksDeleted,
+        });
+      }
+    }
+  }
+
+  public get options() {
+    return {
+      attempts: 1,
+      priority: TaskPriority.Background,
+    };
+  }
+}

server/queues/tasks/ErrorTimedOutImportsTask.ts

@@ -0,0 +1,89 @@
+import { subHours } from "date-fns";
+import { Op } from "sequelize";
+import { ImportState, ImportTaskState } from "@shared/types";
+import Logger from "@server/logging/Logger";
+import { Import, ImportTask } from "@server/models";
+import { sequelize } from "@server/storage/database";
+import BaseTask, { TaskPriority, TaskSchedule } from "./BaseTask";
+
+type Props = {
+  limit: number;
+};
+
+/**
+ * A task that moves the stuck imports to errored state.
+ */
+export default class ErrorTimedOutImportsTask extends BaseTask<Props> {
+  static cron = TaskSchedule.Hour;
+
+  public async perform({ limit }: Props) {
+    // TODO: Hardcoded right now, configurable later
+    const thresholdHours = 12;
+    const cutOffTime = subHours(new Date(), thresholdHours);
+    const importsErrored: Record<string, boolean> = {};
+
+    try {
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      await ImportTask.findAllInBatches<ImportTask<any>>(
+        {
+          where: {
+            state: [ImportTaskState.Created, ImportTaskState.InProgress],
+            createdAt: {
+              [Op.lt]: cutOffTime,
+            },
+          },
+          include: [
+            {
+              model: Import.unscoped(),
+              as: "import",
+              required: true,
+            },
+          ],
+          order: [
+            ["createdAt", "ASC"],
+            ["id", "ASC"],
+          ],
+          batchLimit: 1000,
+          totalLimit: limit,
+        },
+        async (importTasks) => {
+          for (const importTask of importTasks) {
+            const associatedImport = importTask.import;
+
+            if (associatedImport.state === ImportState.Canceled) {
+              continue; // import_tasks for a canceled import are not considered stuck.
+            }
+
+            await sequelize.transaction(async (transaction) => {
+              importTask.state = ImportTaskState.Errored;
+              await importTask.save({ transaction });
+
+              // this import could have been seen before in another import_task.
+              if (!importsErrored[associatedImport.id]) {
+                associatedImport.state = ImportState.Errored;
+                await associatedImport.save({ transaction });
+                importsErrored[associatedImport.id] = true;
+              }
+            });
+          }
+        }
+      );
+    } finally {
+      const totalImportsErrored = Object.keys(importsErrored).length;
+
+      if (totalImportsErrored > 0) {
+        Logger.info(
+          "task",
+          `Updated ${totalImportsErrored} imports to error status`
+        );
+      }
+    }
+  }
+
+  public get options() {
+    return {
+      attempts: 1,
+      priority: TaskPriority.Background,
+    };
+  }
+}

server/queues/tasks/UploadAttachmentsForImportTask.ts

@@ -0,0 +1,69 @@
+import { Sema } from "async-sema";
+import Logger from "@server/logging/Logger";
+import { Attachment } from "@server/models";
+import FileStorage from "@server/storage/files";
+import BaseTask, { TaskPriority } from "./BaseTask";
+
+const ConcurrentUploads = 5;
+
+type Item = {
+  /** The ID of the attachment */
+  attachmentId: string;
+  /** The remote URL to upload */
+  url: string;
+};
+
+/**
+ * A task that uploads a list of provided urls to known attachments.
+ */
+export default class UploadAttachmentsForImportTask extends BaseTask<Item[]> {
+  public async perform(items: Item[]) {
+    const sema = new Sema(ConcurrentUploads, {
+      // perf: pre-allocate the queue size
+      capacity:
+        items.length > ConcurrentUploads ? items.length : ConcurrentUploads,
+    });
+
+    const uploadPromises = items.map(async (item) => {
+      try {
+        await sema.acquire();
+
+        const attachment = await Attachment.findByPk(item.attachmentId, {
+          rejectOnEmpty: true,
+        });
+
+        // This means the attachment has already been uploaded.
+        if (String(attachment.size) !== "0") {
+          return;
+        }
+
+        const res = await FileStorage.storeFromUrl(
+          item.url,
+          attachment.key,
+          attachment.acl
+        );
+
+        if (res) {
+          await attachment.update({
+            size: res.contentLength,
+            contentType: res.contentType,
+          });
+        }
+      } catch (err) {
+        Logger.error("error uploading attachments for import", err);
+        throw err;
+      } finally {
+        sema.release();
+      }
+    });
+
+    return await Promise.all(uploadPromises);
+  }
+
+  public get options() {
+    return {
+      attempts: 3,
+      priority: TaskPriority.Normal,
+    };
+  }
+}

server/routes/api/fileOperations/fileOperations.ts

@@ -7,7 +7,7 @@ import { transaction } from "@server/middlewares/transaction";
 import validate from "@server/middlewares/validate";
 import { FileOperation, Team } from "@server/models";
 import { authorize } from "@server/policies";
-import { presentFileOperation } from "@server/presenters";
+import { presentFileOperation, presentPolicies } from "@server/presenters";
 import FileStorage from "@server/storage/files";
 import { APIContext } from "@server/types";
 import pagination from "../middlewares/pagination";
@@ -51,7 +51,7 @@ router.post(
     const team = await Team.findByPk(user.teamId);
     authorize(user, "update", team);
 
-    const [exports, total] = await Promise.all([
+    const [fileOperations, total] = await Promise.all([
       FileOperation.findAll({
         where,
         order: [[sort, direction]],
@@ -65,7 +65,8 @@ router.post(
 
     ctx.body = {
       pagination: { ...ctx.state.pagination, total },
-      data: exports.map(presentFileOperation),
+      data: fileOperations.map(presentFileOperation),
+      policies: presentPolicies(user, fileOperations),
     };
   }
 );

server/routes/api/imports/imports.test.ts

@@ -0,0 +1,308 @@
+import { NotionImportInput } from "@shared/schema";
+import {
+  CollectionPermission,
+  ImportableIntegrationService,
+  ImportState,
+  IntegrationService,
+  IntegrationType,
+} from "@shared/types";
+import { Import, Integration } from "@server/models";
+import {
+  buildAdmin,
+  buildImport,
+  buildIntegration,
+  buildUser,
+} from "@server/test/factories";
+import { getTestServer } from "@server/test/support";
+
+const server = getTestServer();
+
+describe("#imports.create", () => {
+  it("should create an import", async () => {
+    const admin = await buildAdmin();
+    const integration = await Integration.create<
+      Integration<IntegrationType.Import>
+    >({
+      service: IntegrationService.Notion,
+      type: IntegrationType.Import,
+      userId: admin.id,
+      teamId: admin.teamId,
+      settings: {
+        externalWorkspace: {
+          id: "testId",
+          name: "testWorkspaceName",
+        },
+      },
+    });
+    const input: NotionImportInput = [{ permission: undefined }];
+
+    const res = await server.post("/api/imports.create", {
+      body: {
+        integrationId: integration.id,
+        service: IntegrationService.Notion,
+        input,
+        token: admin.getJwtToken(),
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.id).toBeTruthy();
+    expect(body.data.name).toEqual("testWorkspaceName");
+    expect(body.data.state).toEqual(ImportState.Created);
+    expect(body.data.service).toEqual(IntegrationService.Notion);
+    expect(body.data.createdById).toEqual(admin.id);
+  });
+
+  it("should not allow more than one active import at a time", async () => {
+    const admin = await buildAdmin();
+    const integration = await buildIntegration({
+      userId: admin.id,
+      teamId: admin.teamId,
+    });
+    const input: NotionImportInput = [
+      { permission: CollectionPermission.Read },
+    ];
+    await buildImport({
+      createdById: admin.id,
+      teamId: admin.teamId,
+      integrationId: integration.id,
+    });
+
+    const res = await server.post("/api/imports.create", {
+      body: {
+        integrationId: integration.id,
+        service: ImportableIntegrationService.Notion,
+        input,
+        token: admin.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(422);
+  });
+
+  it("should require authentication", async () => {
+    const res = await server.post("/api/imports.create");
+
+    expect(res.status).toEqual(401);
+  });
+
+  it("should require user to be admin", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/imports.create", {
+      body: {
+        token: user.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#imports.list", () => {
+  it("should list all imports", async () => {
+    const admin = await buildAdmin();
+    const [importOne, importTwo] = await Promise.all([
+      await buildImport({
+        createdById: admin.id,
+        teamId: admin.teamId,
+      }),
+      await buildImport({
+        createdById: admin.id,
+        teamId: admin.teamId,
+      }),
+    ]);
+
+    const res = await server.post("/api/imports.list", {
+      body: {
+        service: IntegrationService.Notion,
+        token: admin.getJwtToken(),
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.length).toEqual(2);
+
+    const importIds = body.data.map(
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (importModel: Import<any>) => importModel.id
+    );
+    expect(importIds).toContain(importOne.id);
+    expect(importIds).toContain(importTwo.id);
+  });
+
+  it("should require authentication", async () => {
+    const res = await server.post("/api/imports.list");
+
+    expect(res.status).toEqual(401);
+  });
+
+  it("should require user to be admin", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/imports.list", {
+      body: {
+        token: user.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#imports.info", () => {
+  it("should return the import", async () => {
+    const admin = await buildAdmin();
+    const importModel = await buildImport({
+      createdById: admin.id,
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/imports.info", {
+      body: {
+        id: importModel.id,
+        token: admin.getJwtToken(),
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.id).toEqual(importModel.id);
+  });
+
+  it("should require authentication", async () => {
+    const res = await server.post("/api/imports.info");
+
+    expect(res.status).toEqual(401);
+  });
+
+  it("should require user to be admin", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/imports.info", {
+      body: {
+        token: user.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#imports.delete", () => {
+  it("should delete the import", async () => {
+    const admin = await buildAdmin();
+    const importModel = await buildImport({
+      state: ImportState.Completed,
+      createdById: admin.id,
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/imports.delete", {
+      body: {
+        id: importModel.id,
+        token: admin.getJwtToken(),
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.success).toEqual(true);
+  });
+
+  it("should throw error when import is not in deletable state", async () => {
+    const admin = await buildAdmin();
+    const importModel = await buildImport({
+      state: ImportState.InProgress,
+      createdById: admin.id,
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/imports.delete", {
+      body: {
+        id: importModel.id,
+        token: admin.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+
+  it("should require authentication", async () => {
+    const res = await server.post("/api/imports.delete");
+
+    expect(res.status).toEqual(401);
+  });
+
+  it("should require user to be admin", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/imports.delete", {
+      body: {
+        token: user.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});
+
+describe("#imports.cancel", () => {
+  it("should cancel the import", async () => {
+    const admin = await buildAdmin();
+    const importModel = await buildImport({
+      createdById: admin.id,
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/imports.cancel", {
+      body: {
+        id: importModel.id,
+        token: admin.getJwtToken(),
+      },
+    });
+    const body = await res.json();
+
+    expect(res.status).toEqual(200);
+    expect(body.data.id).toEqual(importModel.id);
+    expect(body.data.state).toEqual(ImportState.Canceled);
+  });
+
+  it("should throw error when import is not in cancelable state", async () => {
+    const admin = await buildAdmin();
+    const importModel = await buildImport({
+      state: ImportState.Completed,
+      createdById: admin.id,
+      teamId: admin.teamId,
+    });
+
+    const res = await server.post("/api/imports.cancel", {
+      body: {
+        id: importModel.id,
+        token: admin.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+
+  it("should require authentication", async () => {
+    const res = await server.post("/api/imports.cancel");
+
+    expect(res.status).toEqual(401);
+  });
+
+  it("should require user to be admin", async () => {
+    const user = await buildUser();
+
+    const res = await server.post("/api/imports.cancel", {
+      body: {
+        token: user.getJwtToken(),
+      },
+    });
+
+    expect(res.status).toEqual(403);
+  });
+});

server/routes/api/imports/imports.ts

@@ -0,0 +1,185 @@
+import Router from "koa-router";
+import truncate from "lodash/truncate";
+import { WhereOptions } from "sequelize";
+import { ImportState, IntegrationType, UserRole } from "@shared/types";
+import { ImportValidation } from "@shared/validations";
+import { UnprocessableEntityError } from "@server/errors";
+import auth from "@server/middlewares/authentication";
+import { transaction } from "@server/middlewares/transaction";
+import validate from "@server/middlewares/validate";
+import { Integration } from "@server/models";
+import Import from "@server/models/Import";
+import { authorize } from "@server/policies";
+import { presentImport, presentPolicies } from "@server/presenters";
+import { APIContext } from "@server/types";
+import pagination from "../middlewares/pagination";
+import * as T from "./schema";
+
+const router = new Router();
+
+router.post(
+  "imports.create",
+  auth({ role: UserRole.Admin }),
+  validate(T.ImportsCreateSchema),
+  transaction(),
+  async (ctx: APIContext<T.ImportsCreateReq>) => {
+    const { integrationId, service, input } = ctx.input.body;
+    const { user } = ctx.state.auth;
+
+    authorize(user, "createImport", user.team);
+
+    const importInProgress = await Import.count({
+      where: {
+        state: [
+          ImportState.Created,
+          ImportState.InProgress,
+          ImportState.Processed,
+        ],
+        teamId: user.teamId,
+      },
+    });
+
+    if (importInProgress) {
+      throw UnprocessableEntityError("An import is already in progress");
+    }
+
+    const integration = await Integration.findByPk<
+      Integration<IntegrationType.Import>
+    >(integrationId, {
+      rejectOnEmpty: true,
+    });
+    authorize(user, "read", integration);
+
+    const name = integration.settings.externalWorkspace.name;
+
+    const importModel = await Import.createWithCtx(ctx, {
+      name: truncate(name, { length: ImportValidation.maxNameLength }),
+      service,
+      state: ImportState.Created,
+      input,
+      integrationId,
+      createdById: user.id,
+      teamId: user.teamId,
+    });
+    importModel.createdBy = user;
+
+    ctx.body = {
+      data: presentImport(importModel),
+      policies: presentPolicies(user, [importModel]),
+    };
+  }
+);
+
+router.post(
+  "imports.list",
+  auth({ role: UserRole.Admin }),
+  pagination(),
+  validate(T.ImportsListSchema),
+  async (ctx: APIContext<T.ImportsListReq>) => {
+    const { service, sort, direction } = ctx.input.body;
+    const { user } = ctx.state.auth;
+
+    authorize(user, "listImports", user.team);
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const where: WhereOptions<Import<any>> = { teamId: user.teamId };
+
+    if (service) {
+      where.service = service;
+    }
+
+    const [imports, total] = await Promise.all([
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      Import.findAll<Import<any>>({
+        where,
+        order: [[sort, direction]],
+        offset: ctx.state.pagination.offset,
+        limit: ctx.state.pagination.limit,
+      }),
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      Import.count<Import<any>>({
+        where,
+      }),
+    ]);
+
+    ctx.body = {
+      pagination: { ...ctx.state.pagination, total },
+      data: imports.map(presentImport),
+      policies: presentPolicies(user, imports),
+    };
+  }
+);
+
+router.post(
+  "imports.info",
+  auth({ role: UserRole.Admin }),
+  validate(T.ImportsInfoSchema),
+  async (ctx: APIContext<T.ImportsInfoReq>) => {
+    const { id } = ctx.input.body;
+    const { user } = ctx.state.auth;
+
+    const importModel = await Import.findByPk(id, {
+      rejectOnEmpty: true,
+    });
+    authorize(user, "read", importModel);
+
+    ctx.body = {
+      data: presentImport(importModel),
+      policies: presentPolicies(user, [importModel]),
+    };
+  }
+);
+
+router.post(
+  "imports.delete",
+  auth({ role: UserRole.Admin }),
+  validate(T.ImportsDeleteSchema),
+  transaction(),
+  async (ctx: APIContext<T.ImportsDeleteReq>) => {
+    const { id } = ctx.input.body;
+    const { user } = ctx.state.auth;
+    const { transaction } = ctx.state;
+
+    const importModel = await Import.findByPk(id, {
+      rejectOnEmpty: true,
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    authorize(user, "delete", importModel);
+
+    await importModel.destroyWithCtx(ctx);
+
+    ctx.body = {
+      success: true,
+    };
+  }
+);
+
+router.post(
+  "imports.cancel",
+  auth({ role: UserRole.Admin }),
+  validate(T.ImportsCancelSchema),
+  transaction(),
+  async (ctx: APIContext<T.ImportsCancelReq>) => {
+    const { id } = ctx.input.body;
+    const { user } = ctx.state.auth;
+    const { transaction } = ctx.state;
+
+    let importModel = await Import.findByPk(id, {
+      rejectOnEmpty: true,
+      transaction,
+      lock: transaction.LOCK.UPDATE,
+    });
+    authorize(user, "cancel", importModel);
+
+    importModel.state = ImportState.Canceled;
+    importModel = await importModel.saveWithCtx(ctx);
+
+    ctx.body = {
+      data: presentImport(importModel),
+      policies: presentPolicies(user, [importModel]),
+    };
+  }
+);
+
+export default router;

server/routes/api/imports/index.ts

@@ -0,0 +1 @@
+export { default } from "./imports";

server/routes/api/imports/schema.ts

@@ -0,0 +1,69 @@
+import { z } from "zod";
+import { NotionImportInputItemSchema } from "@shared/schema";
+import {
+  ImportableIntegrationService,
+  IntegrationService,
+} from "@shared/types";
+import { BaseSchema } from "../schema";
+
+const BaseIdSchema = z.object({
+  /** Id of the import */
+  id: z.string().uuid(),
+});
+
+const ImportsSortParamsSchema = z.object({
+  /** Specifies the attributes by which imports will be sorted in the list. */
+  sort: z
+    .string()
+    .refine((val) => ["createdAt", "updatedAt", "service"].includes(val), {
+      message: "Invalid sort parameter",
+    })
+    .default("createdAt"),
+
+  /** Specifies the sort order with respect to sort field. */
+  direction: z
+    .string()
+    .optional()
+    .transform((val) => (val !== "ASC" ? "DESC" : val)),
+});
+
+const BaseBodySchema = z.object({
+  integrationId: z.string().uuid(),
+});
+
+export const ImportsCreateSchema = BaseSchema.extend({
+  body: z.discriminatedUnion("service", [
+    BaseBodySchema.extend({
+      service: z.literal(IntegrationService.Notion),
+      input: z.array(NotionImportInputItemSchema),
+    }),
+  ]),
+});
+
+export type ImportsCreateReq = z.infer<typeof ImportsCreateSchema>;
+
+export const ImportsListSchema = BaseSchema.extend({
+  body: ImportsSortParamsSchema.extend({
+    service: z.nativeEnum(ImportableIntegrationService).optional(),
+  }),
+});
+
+export type ImportsListReq = z.infer<typeof ImportsListSchema>;
+
+export const ImportsInfoSchema = BaseSchema.extend({
+  body: BaseIdSchema,
+});
+
+export type ImportsInfoReq = z.infer<typeof ImportsInfoSchema>;
+
+export const ImportsDeleteSchema = BaseSchema.extend({
+  body: BaseIdSchema,
+});
+
+export type ImportsDeleteReq = z.infer<typeof ImportsDeleteSchema>;
+
+export const ImportsCancelSchema = BaseSchema.extend({
+  body: BaseIdSchema,
+});
+
+export type ImportsCancelReq = z.infer<typeof ImportsCancelSchema>;

server/routes/api/index.ts

@@ -20,6 +20,7 @@ import events from "./events";
 import fileOperationsRoute from "./fileOperations";
 import groupMemberships from "./groupMemberships";
 import groups from "./groups";
+import imports from "./imports";
 import installation from "./installation";
 import integrations from "./integrations";
 import apiErrorHandler from "./middlewares/apiErrorHandler";
@@ -97,6 +98,7 @@ router.use("/", fileOperationsRoute.routes());
 router.use("/", urls.routes());
 router.use("/", userMemberships.routes());
 router.use("/", reactions.routes());
+router.use("/", imports.routes());
 
 if (!env.isCloudHosted) {
   router.use("/", installation.routes());

server/services/worker.ts

@@ -111,6 +111,11 @@ export default function init() {
           try {
             await processor.perform(event);
           } catch (err) {
+            // last attempt has failed.
+            if (job.attemptsMade + 1 >= (job.opts.attempts || 1)) {
+              await processor.onFailed(event).catch(); // suppress exception from 'onFailed'.
+            }
+
             Logger.error(
               `Error processing ${event.name} in ${name}`,
               err,
@@ -154,6 +159,11 @@ export default function init() {
         try {
           return await task.perform(props);
         } catch (err) {
+          // last attempt has failed.
+          if (job.attemptsMade + 1 >= (job.opts.attempts || 1)) {
+            await task.onFailed(props).catch(); // suppress exception from 'onFailed'.
+          }
+
           Logger.error(`Error processing task in ${name}`, err, props);
           throw err;
         }

server/test/factories.ts

@@ -10,6 +10,7 @@ import {
   CollectionPermission,
   FileOperationState,
   FileOperationType,
+  ImportState,
   IntegrationService,
   IntegrationType,
   NotificationEventType,
@@ -41,6 +42,7 @@ import {
   SearchQuery,
   Pin,
   Comment,
+  Import,
 } from "@server/models";
 import AttachmentHelper from "@server/models/helpers/AttachmentHelper";
 
@@ -482,6 +484,43 @@ export async function buildFileOperation(
   });
 }
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export async function buildImport(overrides: Partial<Import<any>> = {}) {
+  if (!overrides.teamId) {
+    const team = await buildTeam();
+    overrides.teamId = team.id;
+  }
+
+  if (!overrides.createdById) {
+    const user = await buildAdmin({
+      teamId: overrides.teamId,
+    });
+    overrides.createdById = user.id;
+  }
+
+  if (!overrides.integrationId) {
+    const integration = await buildIntegration({
+      service: IntegrationService.Notion,
+      userId: overrides.createdById,
+      teamId: overrides.teamId,
+    });
+    overrides.integrationId = integration.id;
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  return Import.create<Import<any>>({
+    name: "testImport",
+    service: IntegrationService.Notion,
+    state: ImportState.Created,
+    input: [
+      {
+        permission: CollectionPermission.Read,
+      },
+    ],
+    ...overrides,
+  });
+}
+
 export async function buildAttachment(
   overrides: Partial<Attachment> = {},
   fileName?: string

server/types.ts

@@ -35,6 +35,7 @@ import type {
   Notification,
   Share,
   GroupMembership,
+  Import,
 } from "./models";
 
 export enum AuthenticationType {
@@ -467,6 +468,16 @@ export type NotificationEvent = BaseEvent<Notification> & {
   membershipId?: string;
 };
 
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export type ImportEvent = BaseEvent<Import<any>> & {
+  name:
+    | "imports.create"
+    | "imports.update"
+    | "imports.processed"
+    | "imports.delete";
+  modelId: string;
+};
+
 export type Event =
   | ApiKeyEvent
   | AttachmentEvent
@@ -492,7 +503,8 @@ export type Event =
   | ViewEvent
   | WebhookSubscriptionEvent
   | NotificationEvent
-  | EmptyTrashEvent;
+  | EmptyTrashEvent
+  | ImportEvent;
 
 export type NotificationMetadata = {
   notificationId?: string;

server/utils/indexing.ts

@@ -26,7 +26,7 @@ export async function collectionIndexing(
   for (const collection of sortable) {
     if (collection.index === null) {
       collection.index = fractionalIndex(previousIndex, null);
-      promises.push(collection.save({ transaction }));
+      promises.push(collection.save({ fields: ["index"], transaction })); // save only index to prevent overwriting other unfetched fields.
     }
 
     previousIndex = collection.index;

shared/i18n/locales/en_US/translation.json

@@ -542,6 +542,8 @@
   "Edit group": "Edit group",
   "Delete group": "Delete group",
   "Group options": "Group options",
+  "Cancel": "Cancel",
+  "Import menu options": "Import menu options",
   "Member options": "Member options",
   "New document in <em>{{ collectionName }}</em>": "New document in <em>{{ collectionName }}</em>",
   "New child document": "New child document",
@@ -613,7 +615,6 @@
   "Add a reply": "Add a reply",
   "Reply": "Reply",
   "Post": "Post",
-  "Cancel": "Cancel",
   "Upload image": "Upload image",
   "No resolved comments": "No resolved comments",
   "No comments yet": "No comments yet",
@@ -893,6 +894,13 @@
   "How does this work?": "How does this work?",
   "You can import a zip file that was previously exported from the JSON option in another instance. In {{ appName }}, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.": "You can import a zip file that was previously exported from the JSON option in another instance. In {{ appName }}, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.",
   "Drag and drop the zip file from the JSON export option in {{appName}}, or click to upload": "Drag and drop the zip file from the JSON export option in {{appName}}, or click to upload",
+  "Canceled": "Canceled",
+  "Import canceled": "Import canceled",
+  "Are you sure you want to cancel this import?": "Are you sure you want to cancel this import?",
+  "Canceling": "Canceling",
+  "Canceling this import will discard any progress made. This cannot be undone.": "Canceling this import will discard any progress made. This cannot be undone.",
+  "{{ count }} document imported": "{{ count }} document imported",
+  "{{ count }} document imported_plural": "{{ count }} documents imported",
   "You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.": "You can import a zip file that was previously exported from an Outline installation – collections, documents, and images will be imported. In Outline, open <em>Export</em> in the Settings sidebar and click on <em>Export Data</em>.",
   "Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload": "Drag and drop the zip file from the Markdown export option in {{appName}}, or click to upload",
   "Where do I find the file?": "Where do I find the file?",
@@ -949,13 +957,12 @@
   "New group": "New group",
   "Groups can be used to organize and manage the people on your team.": "Groups can be used to organize and manage the people on your team.",
   "No groups have been created yet": "No groups have been created yet",
-  "Quickly transfer your existing documents, pages, and files from other tools and services into {{appName}}. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.": "Quickly transfer your existing documents, pages, and files from other tools and services into {{appName}}. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.",
   "Import a zip file of Markdown documents (exported from version 0.67.0 or earlier)": "Import a zip file of Markdown documents (exported from version 0.67.0 or earlier)",
   "Import data": "Import data",
   "Import a JSON data file exported from another {{ appName }} instance": "Import a JSON data file exported from another {{ appName }} instance",
-  "Import pages exported from Notion": "Import pages exported from Notion",
   "Import pages from a Confluence instance": "Import pages from a Confluence instance",
   "Enterprise": "Enterprise",
+  "Quickly transfer your existing documents, pages, and files from other tools and services into {{appName}}. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.": "Quickly transfer your existing documents, pages, and files from other tools and services into {{appName}}. You can also drag and drop any HTML, Markdown, and text documents directly into Collections in the app.",
   "Recent imports": "Recent imports",
   "Could not load members": "Could not load members",
   "Everyone that has signed into {{appName}} is listed here. It’s possible that there are other users who have access through {{signinMethods}} but haven’t signed in yet.": "Everyone that has signed into {{appName}} is listed here. It’s possible that there are other users who have access through {{signinMethods}} but haven’t signed in yet.",
@@ -1090,6 +1097,8 @@
   "The URL of your Matomo instance. If you are using Matomo Cloud it will end in matomo.cloud/": "The URL of your Matomo instance. If you are using Matomo Cloud it will end in matomo.cloud/",
   "Site ID": "Site ID",
   "An ID that uniquely identifies the website in your Matomo instance.": "An ID that uniquely identifies the website in your Matomo instance.",
+  "Whoops, you need to accept the permissions in Notion to connect {{ appName }} to your workspace. Try again?": "Whoops, you need to accept the permissions in Notion to connect {{ appName }} to your workspace. Try again?",
+  "Import pages from Notion": "Import pages from Notion",
   "Add to Slack": "Add to Slack",
   "document published": "document published",
   "document updated": "document updated",

shared/schema.ts

@@ -0,0 +1,59 @@
+import { z } from "zod";
+import {
+  CollectionPermission,
+  type ImportableIntegrationService,
+  IntegrationService,
+  ProsemirrorDoc,
+} from "./types";
+import { PageType } from "plugins/notion/shared/types";
+
+const BaseImportInputItemSchema = z.object({
+  permission: z.nativeEnum(CollectionPermission).optional(),
+});
+
+export type BaseImportInput = z.infer<typeof BaseImportInputItemSchema>[];
+
+export const NotionImportInputItemSchema = BaseImportInputItemSchema.extend({
+  type: z.nativeEnum(PageType).optional(),
+  externalId: z.string().optional(),
+});
+
+export type NotionImportInput = z.infer<typeof NotionImportInputItemSchema>[];
+
+export type ImportInput<T extends ImportableIntegrationService> =
+  T extends IntegrationService.Notion ? NotionImportInput : BaseImportInput;
+
+export const BaseImportTaskInputItemSchema = z.object({
+  externalId: z.string(),
+  parentExternalId: z.string().optional(),
+  collectionExternalId: z.string().optional(),
+});
+
+export type BaseImportTaskInput = z.infer<
+  typeof BaseImportTaskInputItemSchema
+>[];
+
+export const NotionImportTaskInputItemSchema =
+  BaseImportTaskInputItemSchema.extend({
+    type: z.nativeEnum(PageType),
+  });
+
+export type NotionImportTaskInput = z.infer<
+  typeof NotionImportTaskInputItemSchema
+>[];
+
+export type ImportTaskInput<T extends ImportableIntegrationService> =
+  T extends IntegrationService.Notion
+    ? NotionImportTaskInput
+    : BaseImportTaskInput;
+
+// No reason to be here except for co-location with import task input.
+export type ImportTaskOutput = {
+  externalId: string;
+  title: string;
+  emoji?: string;
+  author?: string;
+  content: ProsemirrorDoc;
+  createdAt?: Date;
+  updatedAt?: Date;
+}[];

shared/types.ts

@@ -54,6 +54,23 @@ export enum FileOperationState {
   Expired = "expired",
 }
 
+export enum ImportState {
+  Created = "created",
+  InProgress = "in_progress",
+  Processed = "processed",
+  Completed = "completed",
+  Errored = "errored",
+  Canceled = "canceled",
+}
+
+export enum ImportTaskState {
+  Created = "created",
+  InProgress = "in_progress",
+  Completed = "completed",
+  Errored = "errored",
+  Canceled = "canceled",
+}
+
 export enum MentionType {
   User = "user",
   Document = "document",
@@ -86,6 +103,8 @@ export enum IntegrationType {
   Analytics = "analytics",
   /** An integration that maps an Outline user to an external service. */
   LinkedAccount = "linkedAccount",
+  /** An integration that imports documents into Outline. */
+  Import = "import",
 }
 
 export enum IntegrationService {
@@ -96,8 +115,18 @@ export enum IntegrationService {
   Matomo = "matomo",
   Umami = "umami",
   GitHub = "github",
+  Notion = "notion",
 }
 
+export type ImportableIntegrationService = Extract<
+  IntegrationService,
+  IntegrationService.Notion
+>;
+
+export const ImportableIntegrationService = {
+  Notion: IntegrationService.Notion,
+} as const;
+
 export type UserCreatableIntegrationService = Extract<
   IntegrationService,
   | IntegrationService.Diagrams
@@ -143,6 +172,8 @@ export type IntegrationSettings<T> = T extends IntegrationType.Embed
   ? { url: string; channel: string; channelId: string }
   : T extends IntegrationType.Command
   ? { serviceTeamId: string }
+  : T extends IntegrationType.Import
+  ? { externalWorkspace: { id: string; name: string; iconUrl?: string } }
   :
       | { url: string }
       | {
@@ -187,6 +218,8 @@ export type SourceMetadata = {
   createdByName?: string;
   /** An ID in the external source. */
   externalId?: string;
+  /** Original name in the external source. */
+  externalName?: string;
   /** Whether the item was created through a trial license. */
   trial?: boolean;
 };

shared/utils/ProsemirrorHelper.ts

@@ -234,6 +234,46 @@ export class ProsemirrorHelper {
     return images;
   }
 
+  /**
+   * Iterates through the document to find all of the videos.
+   *
+   * @param doc Prosemirror document node
+   * @returns Array<Node> of videos
+   */
+  static getVideos(doc: Node): Node[] {
+    const videos: Node[] = [];
+
+    doc.descendants((node) => {
+      if (node.type.name === "video") {
+        videos.push(node);
+      }
+
+      return true;
+    });
+
+    return videos;
+  }
+
+  /**
+   * Iterates through the document to find all of the attachments.
+   *
+   * @param doc Prosemirror document node
+   * @returns Array<Node> of attachments
+   */
+  static getAttachments(doc: Node): Node[] {
+    const attachments: Node[] = [];
+
+    doc.descendants((node) => {
+      if (node.type.name === "attachment") {
+        attachments.push(node);
+      }
+
+      return true;
+    });
+
+    return attachments;
+  }
+
   /**
    * Iterates through the document to find all of the tasks and their completion state.
    *

shared/validations.ts

@@ -51,6 +51,11 @@ export const DocumentValidation = {
   maxStateLength: 1500 * 1024,
 };
 
+export const ImportValidation = {
+  /** The maximum length of the import name */
+  maxNameLength: 100,
+};
+
 export const RevisionValidation = {
   minNameLength: 1,
   maxNameLength: 255,

yarn.lock

@@ -2867,6 +2867,14 @@
   resolved "https://registry.yarnpkg.com/@nolyfill/is-core-module/-/is-core-module-1.0.39.tgz#3dc35ba0f1e66b403c00b39344f870298ebb1c8e"
   integrity sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==
 
+"@notionhq/client@^2.2.16":
+  version "2.2.16"
+  resolved "https://registry.yarnpkg.com/@notionhq/client/-/client-2.2.16.tgz#6564cd77bc12e7dc68e4f5c4c6adef8384a33027"
+  integrity sha512-3GlkfhLw8+Jw8U2iFEmHA6WfCgYhZCXLxgPdqDJkYMFotELNpQO+yGSy2QWURsG8ndu21sLt+FEOfDbNcCtFMg==
+  dependencies:
+    "@types/node-fetch" "^2.5.10"
+    node-fetch "^2.6.1"
+
 "@octokit/app@^14.0.2":
   version "14.0.2"
   resolved "https://registry.yarnpkg.com/@octokit/app/-/app-14.0.2.tgz#b47c52020221351fb58640f113eb38b2ad3998fe"
@@ -3233,13 +3241,6 @@
     "@radix-ui/react-primitive" "2.0.1"
     "@radix-ui/react-slot" "1.1.1"
 
-"@radix-ui/react-compose-refs@1.0.0":
-  version "1.0.0"
-  resolved "https://registry.yarnpkg.com/@radix-ui/react-compose-refs/-/react-compose-refs-1.0.0.tgz#37595b1f16ec7f228d698590e78eeed18ff218ae"
-  integrity "sha1-N1lbHxbsfyKNaYWQ547u0Y/yGK4= sha512-0KaSv6sx787/hK3eF53iOkiSLwAGlFMx5lotrqD2pTjB18KbybKoEIgkNZTKC60YECDQTKGTRcDBILwZVqVKvA=="
-  dependencies:
-    "@babel/runtime" "^7.13.10"
-
 "@radix-ui/react-compose-refs@1.1.1":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.1.tgz#6f766faa975f8738269ebb8a23bad4f5a8d2faec"
@@ -3367,14 +3368,6 @@
     "@radix-ui/react-compose-refs" "1.1.1"
     "@radix-ui/react-use-layout-effect" "1.1.0"
 
-"@radix-ui/react-primitive@1.0.1":
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/@radix-ui/react-primitive/-/react-primitive-1.0.1.tgz#c1ebcce283dd2f02e4fbefdaa49d1cb13dbc990a"
-  integrity "sha1-wevM4oPdLwLk++/apJ0csT28mQo= sha512-fHbmislWVkZaIdeF6GZxF0A/NH/3BjrGIYj+Ae6eTmTCr7EB0RQAAVEiqsXK6p3/JcRqVSBQoceZroj30Jj3XA=="
-  dependencies:
-    "@babel/runtime" "^7.13.10"
-    "@radix-ui/react-slot" "1.0.1"
-
 "@radix-ui/react-primitive@2.0.1":
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/@radix-ui/react-primitive/-/react-primitive-2.0.1.tgz#6d9efc550f7520135366f333d1e820cf225fad9e"
@@ -3416,14 +3409,6 @@
     aria-hidden "^1.1.1"
     react-remove-scroll "^2.6.1"
 
-"@radix-ui/react-slot@1.0.1":
-  version "1.0.1"
-  resolved "https://registry.yarnpkg.com/@radix-ui/react-slot/-/react-slot-1.0.1.tgz#e7868c669c974d649070e9ecbec0b367ee0b4d81"
-  integrity "sha1-54aMZpyXTWSQcOnsvsCzZ+4LTYE= sha512-avutXAFL1ehGvAXtPquu0YK5oz6ctS474iM3vNGQIkswrVhdrS52e3uoMQBzZhNRAIE0jBnUyXWNmSjGHhCFcw=="
-  dependencies:
-    "@babel/runtime" "^7.13.10"
-    "@radix-ui/react-compose-refs" "1.0.0"
-
 "@radix-ui/react-slot@1.1.1":
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/@radix-ui/react-slot/-/react-slot-1.1.1.tgz#ab9a0ffae4027db7dc2af503c223c978706affc3"
@@ -5320,10 +5305,10 @@
   resolved "https://registry.yarnpkg.com/@types/natural-sort/-/natural-sort-0.0.24.tgz#9a89fcbabd963937fab9cc4ca527635c2a7c0cef"
   integrity "sha1-mon8ur2WOTf6ucxMpSdjXCp8DO8= sha512-+/F8JDyT0QUi2cE51S4Xsy4yuHVBiFsU5bq0g2XnTKkOj0Jz69o3TLYd1gnVTOrKOvg0+FUJqE9BA24FgnpuGg=="
 
-"@types/node-fetch@^2.6.9":
-  version "2.6.9"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.9.tgz#15f529d247f1ede1824f7e7acdaa192d5f28071e"
-  integrity "sha1-FfUp0kfx7eGCT356zaoZLV8oBx4= sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA=="
+"@types/node-fetch@^2.5.10", "@types/node-fetch@^2.6.9":
+  version "2.6.12"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.12.tgz#8ab5c3ef8330f13100a7479e2cd56d3386830a03"
+  integrity sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==
   dependencies:
     "@types/node" "*"
     form-data "^4.0.0"
@@ -6076,6 +6061,11 @@ async-lock@^1.3.1:
   resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.1.tgz#f2301c200600cde97acc386453b7126fa8aced3c"
   integrity "sha1-8jAcIAYAzel6zDhkU7cSb6is7Tw= sha512-zK7xap9UnttfbE23JmcrNIyueAn6jWshihJqA33U/hEnKprF/lVGBDsBv/bqLm2YMMl1DnpHhUY044eA0t1TUw=="
 
+async-sema@^3.1.1:
+  version "3.1.1"
+  resolved "https://registry.yarnpkg.com/async-sema/-/async-sema-3.1.1.tgz#e527c08758a0f8f6f9f15f799a173ff3c40ea808"
+  integrity sha512-tLRNUXati5MFePdAk8dw7Qt7DpxPB60ofAgn8WRhW6a2rcimZnYBP9oxHiv0OHy+Wz7kPMG+t4LGdt31+4EmGg==
+
 async@^3.2.3:
   version "3.2.3"
   resolved "https://registry.yarnpkg.com/async/-/async-3.2.3.tgz#ac53dafd3f4720ee9e8a160628f18ea91df196c9"
@@ -11978,7 +11968,7 @@ mobx-react@^6.3.1:
 mobx-utils@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/mobx-utils/-/mobx-utils-4.0.1.tgz#1bd8648332be7ca83d7023aaa0ba24bbf389a300"
-  integrity "sha1-G9hkgzK+fKg9cCOqoLoku/OJowA= sha512-hWYLNJNBoGY/iQbQuOzYkUsTGArpTTutrXaQQrXvxBMefDwhWyNHr7bx/g7syf6KQ1f6aKzgQICqC+zXSvGzJQ=="
+  integrity sha512-hWYLNJNBoGY/iQbQuOzYkUsTGArpTTutrXaQQrXvxBMefDwhWyNHr7bx/g7syf6KQ1f6aKzgQICqC+zXSvGzJQ==
 
 mobx@^4.15.4:
   version "4.15.7"
@@ -13438,7 +13428,7 @@ react-refresh@^0.14.0, react-refresh@^0.14.2:
   resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.2.tgz#3833da01ce32da470f1f936b9d477da5c7028bf9"
   integrity sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==
 
-react-remove-scroll-bar@^2.3.3, react-remove-scroll-bar@^2.3.7:
+react-remove-scroll-bar@^2.3.7:
   version "2.3.8"
   resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz#99c20f908ee467b385b68a3469b4a3e750012223"
   integrity sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==
@@ -13485,7 +13475,7 @@ react-router@5.3.4:
     tiny-invariant "^1.0.2"
     tiny-warning "^1.0.0"
 
-react-style-singleton@^2.2.1, react-style-singleton@^2.2.2, react-style-singleton@^2.2.3:
+react-style-singleton@^2.2.2, react-style-singleton@^2.2.3:
   version "2.2.3"
   resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.3.tgz#4265608be69a4d70cfe3047f2c6c88b2c3ace388"
   integrity sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==
@@ -15448,14 +15438,14 @@ url-parse@^1.4.3, url-parse@^1.5.3:
     querystringify "^2.1.1"
     requires-port "^1.0.0"
 
-use-callback-ref@^1.3.0, use-callback-ref@^1.3.3:
+use-callback-ref@^1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/use-callback-ref/-/use-callback-ref-1.3.3.tgz#98d9fab067075841c5b2c6852090d5d0feabe2bf"
   integrity sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==
   dependencies:
     tslib "^2.0.0"
 
-use-sidecar@^1.1.2, use-sidecar@^1.1.3:
+use-sidecar@^1.1.3:
   version "1.1.3"
   resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.3.tgz#10e7fd897d130b896e2c546c63a5e8233d00efdb"
   integrity sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==