From 6e98568e5b7a5ad70641c8139d7bc6990fabdf34 Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Mon, 24 Mar 2025 00:49:13 +0530 Subject: [PATCH] API importer for Notion (#8710) --- .env.sample | 4 + app/components/Dialogs.tsx | 5 +- app/components/WebsocketProvider.tsx | 25 +- app/hooks/usePolicy.ts | 5 +- app/menus/ImportMenu.tsx | 62 + app/models/Import.ts | 54 + app/models/base/Model.ts | 28 +- app/scenes/Settings/Import.tsx | 242 +- .../Settings/components/ImportListItem.tsx | 148 + app/stores/DialogsStore.ts | 4 + app/stores/ImportsStore.ts | 25 + app/stores/RootStore.ts | 3 + app/utils/ApiClient.ts | 5 + app/utils/PluginManager.ts | 11 + app/utils/errors.ts | 2 + package.json | 2 + plugins/notion/client/Imports.tsx | 88 + .../notion/client/components/ImportDialog.tsx | 78 + plugins/notion/client/index.tsx | 19 + plugins/notion/plugin.json | 5 + plugins/notion/server/api/notion.ts | 105 + plugins/notion/server/api/schema.ts | 25 + plugins/notion/server/env.ts | 19 + plugins/notion/server/index.ts | 26 + plugins/notion/server/notion.ts | 284 + .../processors/NotionImportsProcessor.ts | 71 + .../server/tasks/NotionAPIImportTask.ts | 144 + .../server/utils/NotionConverter.test.ts | 15 + .../notion/server/utils/NotionConverter.ts | 584 ++ .../NotionConverter.test.ts.snap | 1985 +++++ plugins/notion/shared/NotionUtils.ts | 58 + plugins/notion/shared/types.ts | 18 + .../server/tasks/DeliverWebhookTask.ts | 6 + server/commands/documentCreator.ts | 5 +- server/errors.ts | 6 + .../20250306181804-create-imports.js | 93 + .../20250310043011-create-import-tasks.js | 67 + ...piImportId-to-collections-and-documents.js | 59 + server/models/Collection.ts | 9 + server/models/Document.ts | 8 + server/models/Import.ts | 87 + server/models/ImportTask.ts | 58 + server/models/index.ts | 4 + server/policies/fileOperation.ts | 2 +- server/policies/import.ts | 31 + server/policies/index.ts | 1 + server/presenters/import.ts | 19 + server/presenters/index.ts | 2 + server/queues/processors/BaseProcessor.ts | 11 + server/queues/processors/ImportsProcessor.ts | 529 ++ .../queues/processors/WebsocketsProcessor.ts | 19 +- server/queues/processors/index.ts | 4 +- server/queues/tasks/APIImportTask.ts | 391 + server/queues/tasks/BaseTask.ts | 11 + server/queues/tasks/CleanupOldImportsTask.ts | 84 + .../queues/tasks/ErrorTimedOutImportsTask.ts | 89 + .../tasks/UploadAttachmentsForImportTask.ts | 69 + .../api/fileOperations/fileOperations.ts | 7 +- server/routes/api/imports/imports.test.ts | 308 + server/routes/api/imports/imports.ts | 185 + server/routes/api/imports/index.ts | 1 + server/routes/api/imports/schema.ts | 69 + server/routes/api/index.ts | 2 + server/services/worker.ts | 10 + server/test/factories.ts | 39 + server/test/fixtures/notion-page.json | 6753 +++++++++++++++++ server/types.ts | 14 +- server/utils/indexing.ts | 2 +- shared/i18n/locales/en_US/translation.json | 15 +- shared/schema.ts | 59 + shared/types.ts | 33 + shared/utils/ProsemirrorHelper.ts | 40 + shared/validations.ts | 5 + yarn.lock | 54 +- 74 files changed, 13259 insertions(+), 150 deletions(-) create mode 100644 app/menus/ImportMenu.tsx create mode 100644 app/models/Import.ts create mode 100644 app/scenes/Settings/components/ImportListItem.tsx create mode 100644 app/stores/ImportsStore.ts create mode 100644 plugins/notion/client/Imports.tsx create mode 100644 plugins/notion/client/components/ImportDialog.tsx create mode 100644 plugins/notion/client/index.tsx create mode 100644 plugins/notion/plugin.json create mode 100644 plugins/notion/server/api/notion.ts create mode 100644 plugins/notion/server/api/schema.ts create mode 100644 plugins/notion/server/env.ts create mode 100644 plugins/notion/server/index.ts create mode 100644 plugins/notion/server/notion.ts create mode 100644 plugins/notion/server/processors/NotionImportsProcessor.ts create mode 100644 plugins/notion/server/tasks/NotionAPIImportTask.ts create mode 100644 plugins/notion/server/utils/NotionConverter.test.ts create mode 100644 plugins/notion/server/utils/NotionConverter.ts create mode 100644 plugins/notion/server/utils/__snapshots__/NotionConverter.test.ts.snap create mode 100644 plugins/notion/shared/NotionUtils.ts create mode 100644 plugins/notion/shared/types.ts create mode 100644 server/migrations/20250306181804-create-imports.js create mode 100644 server/migrations/20250310043011-create-import-tasks.js create mode 100644 server/migrations/20250314105847-add-apiImportId-to-collections-and-documents.js create mode 100644 server/models/Import.ts create mode 100644 server/models/ImportTask.ts create mode 100644 server/policies/import.ts create mode 100644 server/presenters/import.ts create mode 100644 server/queues/processors/ImportsProcessor.ts create mode 100644 server/queues/tasks/APIImportTask.ts create mode 100644 server/queues/tasks/CleanupOldImportsTask.ts create mode 100644 server/queues/tasks/ErrorTimedOutImportsTask.ts create mode 100644 server/queues/tasks/UploadAttachmentsForImportTask.ts create mode 100644 server/routes/api/imports/imports.test.ts create mode 100644 server/routes/api/imports/imports.ts create mode 100644 server/routes/api/imports/index.ts create mode 100644 server/routes/api/imports/schema.ts create mode 100644 server/test/fixtures/notion-page.json create mode 100644 shared/schema.ts diff --git a/.env.sample b/.env.sample index 51046501da..8f2e4910e8 100644 --- a/.env.sample +++ b/.env.sample @@ -147,6 +147,10 @@ DISCORD_SERVER_ID= # DISCORD_SERVER_ID and DISCORD_SERVER_ROLES must be set together. DISCORD_SERVER_ROLES= +# –––––––––––––– IMPORTS –––––––––––––– +NOTION_CLIENT_ID= +NOTION_CLIENT_SECRET= + # –––––––––––––––– OPTIONAL –––––––––––––––– # Base64 encoded private key and certificate for HTTPS termination. This is only diff --git a/app/components/Dialogs.tsx b/app/components/Dialogs.tsx index e7f4d05589..d1af42168c 100644 --- a/app/components/Dialogs.tsx +++ b/app/components/Dialogs.tsx @@ -23,7 +23,10 @@ function Dialogs() { key={id} isOpen={modal.isOpen} fullscreen={modal.fullscreen ?? false} - onRequestClose={() => dialogs.closeModal(id)} + onRequestClose={() => { + modal.onClose?.(); + dialogs.closeModal(id); + }} title={modal.title} style={modal.style} > diff --git a/app/components/WebsocketProvider.tsx b/app/components/WebsocketProvider.tsx index 03d007e6bb..4a4b0405da 100644 --- a/app/components/WebsocketProvider.tsx +++ b/app/components/WebsocketProvider.tsx @@ -6,7 +6,11 @@ import * as React from "react"; import { withTranslation, WithTranslation } from "react-i18next"; import { io, Socket } from "socket.io-client"; import { toast } from "sonner"; -import { FileOperationState, FileOperationType } from "@shared/types"; +import { + FileOperationState, + FileOperationType, + ImportState, +} from "@shared/types"; import RootStore from "~/stores/RootStore"; import Collection from "~/models/Collection"; import Comment from "~/models/Comment"; @@ -15,6 +19,7 @@ import FileOperation from "~/models/FileOperation"; import Group from "~/models/Group"; import GroupMembership from "~/models/GroupMembership"; import GroupUser from "~/models/GroupUser"; +import Import from "~/models/Import"; import Membership from "~/models/Membership"; import Notification from "~/models/Notification"; import Pin from "~/models/Pin"; @@ -100,6 +105,7 @@ class WebsocketProvider extends React.Component { subscriptions, fileOperations, notifications, + imports, } = this.props; const currentUserId = auth?.user?.id; @@ -620,6 +626,23 @@ class WebsocketProvider extends React.Component { } ); + this.socket.on("imports.create", (event: PartialExcept) => { + imports.add(event); + }); + + this.socket.on("imports.update", (event: PartialExcept) => { + imports.add(event); + + if ( + event.state === ImportState.Completed && + event.createdBy?.id === auth.user?.id + ) { + toast.success(event.name, { + description: this.props.t("Your import completed"), + }); + } + }); + this.socket.on( "subscriptions.create", (event: PartialExcept) => { diff --git a/app/hooks/usePolicy.ts b/app/hooks/usePolicy.ts index 86052f7c3d..4bb9e8b86b 100644 --- a/app/hooks/usePolicy.ts +++ b/app/hooks/usePolicy.ts @@ -18,6 +18,7 @@ export default function usePolicy(entity?: string | Model | null) { ? entity : entity.id : ""; + const policy = policies.get(entityId); React.useEffect(() => { if ( @@ -28,11 +29,11 @@ export default function usePolicy(entity?: string | Model | null) { ) { // The policy for this model is missing and we have an authenticated session, attempt to // reload relationships for this model. - if (!policies.get(entity.id) && user) { + if (!policy && user) { void entity.loadRelations(); } } - }, [policies, user, entity]); + }, [policies, policy, user, entity]); return policies.abilities(entityId); } diff --git a/app/menus/ImportMenu.tsx b/app/menus/ImportMenu.tsx new file mode 100644 index 0000000000..3b55e6fe40 --- /dev/null +++ b/app/menus/ImportMenu.tsx @@ -0,0 +1,62 @@ +import { observer } from "mobx-react"; +import { CrossIcon, TrashIcon } from "outline-icons"; +import * as React from "react"; +import { useTranslation } from "react-i18next"; +import { useMenuState } from "reakit/Menu"; +import Import from "~/models/Import"; +import ContextMenu from "~/components/ContextMenu"; +import OverflowMenuButton from "~/components/ContextMenu/OverflowMenuButton"; +import Template from "~/components/ContextMenu/Template"; +import usePolicy from "~/hooks/usePolicy"; +import { MenuItem } from "~/types"; + +type Props = { + /** Import to which actions will be applied. */ + importModel: Import; + /** Callback to handle import cancellation. */ + onCancel: () => Promise; + /** Callback to handle import deletion. */ + onDelete: () => Promise; +}; + +export const ImportMenu = observer( + ({ importModel, onCancel, onDelete }: Props) => { + const { t } = useTranslation(); + const can = usePolicy(importModel); + const menu = useMenuState({ + modal: true, + }); + + const items = React.useMemo( + () => + [ + { + type: "button", + title: t("Cancel"), + visible: can.cancel, + icon: , + dangerous: true, + onClick: onCancel, + }, + { + type: "button", + title: t("Delete"), + visible: can.delete, + icon: , + dangerous: true, + onClick: onDelete, + }, + ] satisfies MenuItem[], + [t, can.delete, can.cancel, onCancel, onDelete] + ); + + return ( + <> + + +