From 50759d40e8b7f1513c52a70e51e69dfe25217c58 Mon Sep 17 00:00:00 2001 From: Hemachandar <132386067+hmacr@users.noreply.github.com> Date: Sun, 11 Jan 2026 07:49:33 +0530 Subject: [PATCH] feat: Option to export nested documents (#9679) * add migration file * documents.export API * download dialog * file ops list item * export task * download modal styling * cleanup * lint * Restore individual download actions --------- Co-authored-by: Tom Moor --- app/actions/definitions/documents.tsx | 116 ++++++----- app/components/DocumentDownload.tsx | 191 ++++++++++++++++++ app/models/Document.ts | 11 +- app/models/FileOperation.ts | 2 + .../components/FileOperationListItem.tsx | 3 +- ...9063818-add-documentId-to-fileOperation.js | 19 ++ server/models/FileOperation.ts | 33 ++- server/presenters/fileOperation.ts | 6 +- server/queues/tasks/ExportDocumentTreeTask.ts | 54 ++++- server/queues/tasks/ExportHTMLZipTask.ts | 21 +- server/queues/tasks/ExportJSONTask.ts | 9 +- server/queues/tasks/ExportMarkdownZipTask.ts | 21 +- server/queues/tasks/ExportTask.ts | 136 ++++++++----- server/routes/api/documents/documents.ts | 71 ++++++- server/routes/api/documents/schema.ts | 4 +- shared/constants.ts | 7 +- shared/editor/extensions/Mermaid.ts | 18 +- shared/editor/lib/isCode.ts | 5 +- shared/i18n/locales/en_US/translation.json | 26 ++- 19 files changed, 614 insertions(+), 139 deletions(-) create mode 100644 app/components/DocumentDownload.tsx create mode 100644 server/migrations/20250719063818-add-documentId-to-fileOperation.js diff --git a/app/actions/definitions/documents.tsx b/app/actions/definitions/documents.tsx index b8d0ed20d7..979ef413c9 100644 --- a/app/actions/definitions/documents.tsx +++ b/app/actions/definitions/documents.tsx @@ -46,6 +46,7 @@ import DocumentPublish from "~/scenes/DocumentPublish"; import DeleteDocumentsInTrash from "~/scenes/Trash/components/DeleteDocumentsInTrash"; import ConfirmationDialog from "~/components/ConfirmationDialog"; import DocumentCopy from "~/components/DocumentCopy"; +import { DocumentDownload } from "~/components/DocumentDownload"; import MarkdownIcon from "~/components/Icons/MarkdownIcon"; import { getHeaderExpandedKey } from "~/components/Sidebar/components/Header"; import DocumentTemplatizeDialog from "~/components/TemplatizeDialog"; @@ -60,7 +61,6 @@ import { DocumentSection, TrashSection, } from "~/actions/sections"; -import env from "~/env"; import { setPersistedState } from "~/hooks/usePersistedState"; import history from "~/utils/history"; import { @@ -78,6 +78,7 @@ import capitalize from "lodash/capitalize"; import CollectionIcon from "~/components/Icons/CollectionIcon"; import type { Action, ActionGroup, ActionSeparator } from "~/types"; import lazyWithRetry from "~/utils/lazyWithRetry"; +import env from "~/env"; const Insights = lazyWithRetry( () => import("~/scenes/Document/components/Insights") @@ -518,13 +519,40 @@ export const shareDocument = createAction({ }, }); -export const downloadDocumentAsHTML = createAction({ - name: ({ t }) => t("HTML"), - analyticsName: "Download document as HTML", +export const downloadDocument = createAction({ + name: ({ t, isMenu }) => (isMenu ? t("Download") : t("Download document")), + analyticsName: "Download document", section: ActiveDocumentSection, - keywords: "html export", icon: , - iconInContextMenu: false, + keywords: "export md markdown html", + visible: ({ activeDocumentId, stores }) => + !!activeDocumentId && stores.policies.abilities(activeDocumentId).download, + perform: ({ activeDocumentId, t, stores }) => { + if (!activeDocumentId) { + return; + } + + const document = stores.documents.get(activeDocumentId); + invariant(document, "Document must exist"); + + stores.dialogs.openModal({ + title: t("Download document"), + content: ( + + ), + }); + }, +}); + +export const downloadDocumentAsMarkdown = createAction({ + name: ({ t }) => t("Downloas as Markdown"), + analyticsName: "Download document as Markdown", + section: ActiveDocumentSection, + keywords: "md markdown export", + icon: , visible: ({ activeDocumentId, stores }) => !!activeDocumentId && stores.policies.abilities(activeDocumentId).download, perform: async ({ activeDocumentId, stores }) => { @@ -533,70 +561,59 @@ export const downloadDocumentAsHTML = createAction({ } const document = stores.documents.get(activeDocumentId); - await document?.download(ExportContentType.Html); + await document?.download({ + contentType: ExportContentType.Markdown, + includeChildDocuments: false, + }); + }, +}); + +export const downloadDocumentAsHTML = createAction({ + name: ({ t }) => t("Download as HTML"), + analyticsName: "Download document as HTML", + section: ActiveDocumentSection, + keywords: "xml html export", + icon: , + visible: ({ activeDocumentId, stores }) => + !!activeDocumentId && stores.policies.abilities(activeDocumentId).download, + perform: async ({ activeDocumentId, stores }) => { + if (!activeDocumentId) { + return; + } + + const document = stores.documents.get(activeDocumentId); + await document?.download({ + contentType: ExportContentType.Html, + includeChildDocuments: false, + }); }, }); export const downloadDocumentAsPDF = createAction({ - name: ({ t }) => t("PDF"), + name: ({ t }) => t("Download as PDF"), analyticsName: "Download document as PDF", section: ActiveDocumentSection, - keywords: "export", + keywords: "pdf export", icon: , - iconInContextMenu: false, visible: ({ activeDocumentId, stores }) => !!( activeDocumentId && stores.policies.abilities(activeDocumentId).download && env.PDF_EXPORT_ENABLED ), - perform: ({ activeDocumentId, t, stores }) => { - if (!activeDocumentId) { - return; - } - - const id = toast.loading(`${t("Exporting")}…`); - const document = stores.documents.get(activeDocumentId); - return document - ?.download(ExportContentType.Pdf) - .finally(() => id && toast.dismiss(id)); - }, -}); - -export const downloadDocumentAsMarkdown = createAction({ - name: ({ t }) => t("Markdown"), - analyticsName: "Download document as Markdown", - section: ActiveDocumentSection, - keywords: "md markdown export", - icon: , - iconInContextMenu: false, - visible: ({ activeDocumentId, stores }) => - !!activeDocumentId && stores.policies.abilities(activeDocumentId).download, perform: async ({ activeDocumentId, stores }) => { if (!activeDocumentId) { return; } const document = stores.documents.get(activeDocumentId); - await document?.download(ExportContentType.Markdown); + await document?.download({ + contentType: ExportContentType.Pdf, + includeChildDocuments: false, + }); }, }); -export const downloadDocument = createActionWithChildren({ - name: ({ t, isMenu }) => (isMenu ? t("Download") : t("Download document")), - analyticsName: "Download document", - section: ActiveDocumentSection, - icon: , - keywords: "export", - visible: ({ activeDocumentId, stores }) => - !!activeDocumentId && stores.policies.abilities(activeDocumentId).download, - children: [ - downloadDocumentAsHTML, - downloadDocumentAsPDF, - downloadDocumentAsMarkdown, - ], -}); - export const copyDocumentAsMarkdown = createAction({ name: ({ t }) => t("Copy as Markdown"), section: ActiveDocumentSection, @@ -1440,6 +1457,9 @@ export const rootDocumentActions = [ deleteDocument, importDocument, downloadDocument, + downloadDocumentAsMarkdown, + downloadDocumentAsHTML, + downloadDocumentAsPDF, copyDocumentLink, copyDocumentShareLink, copyDocumentAsMarkdown, diff --git a/app/components/DocumentDownload.tsx b/app/components/DocumentDownload.tsx new file mode 100644 index 0000000000..bbc6e3609c --- /dev/null +++ b/app/components/DocumentDownload.tsx @@ -0,0 +1,191 @@ +import { observer } from "mobx-react"; +import { useCallback, useMemo, useState } from "react"; +import { Trans, useTranslation } from "react-i18next"; +import { toast } from "sonner"; +import styled from "styled-components"; +import { ExportContentType, NotificationEventType } from "@shared/types"; +import type Document from "~/models/Document"; +import ConfirmationDialog from "~/components/ConfirmationDialog"; +import Flex from "~/components/Flex"; +import Text from "~/components/Text"; +import env from "~/env"; +import useCurrentUser from "~/hooks/useCurrentUser"; +import history from "~/utils/history"; +import { settingsPath } from "~/utils/routeHelpers"; +import usePolicy from "~/hooks/usePolicy"; +import useCurrentTeam from "~/hooks/useCurrentTeam"; + +type Props = { + document: Document; + onSubmit: () => void; +}; + +export const DocumentDownload = observer(({ document, onSubmit }: Props) => { + const { t } = useTranslation(); + const user = useCurrentUser(); + const team = useCurrentTeam(); + const hasChildDocuments = !!document.childDocuments.length; + const can = usePolicy(team); + + const [contentType, setContentType] = useState( + ExportContentType.Markdown + ); + const [includeChildDocuments, setIncludeChildDocuments] = + useState(hasChildDocuments); + + const handleContentTypeChange = useCallback( + (ev: React.ChangeEvent) => { + setContentType(ev.target.value as ExportContentType); + }, + [] + ); + + const handleIncludeChildDocumentsChange = useCallback( + (ev: React.ChangeEvent) => { + setIncludeChildDocuments(ev.target.checked); + }, + [] + ); + + const handleSubmit = useCallback(async () => { + await document.download({ + contentType, + includeChildDocuments, + }); + + if (includeChildDocuments) { + const options = can.createExport + ? { + description: t( + `Your file will be available in {{ location }} soon`, + { + location: `"${t("Settings")} > ${t("Export")}"`, + } + ), + action: { + label: t("View"), + onClick: () => { + history.push(settingsPath("export")); + }, + }, + } + : { + description: t( + `A link to your file will be sent through email soon` + ), + }; + toast.success(t("Export started"), options); + } + + onSubmit(); + }, [t, document, contentType, includeChildDocuments, onSubmit]); + + const items = useMemo(() => { + const radioItems = [ + { + title: "Markdown", + description: t( + "A file containing the selected documents in Markdown format." + ), + value: ExportContentType.Markdown, + }, + { + title: "HTML", + description: t( + "A file containing the selected documents in HTML format." + ), + value: ExportContentType.Html, + }, + ]; + + if (env.PDF_EXPORT_ENABLED) { + radioItems.push({ + title: "PDF", + description: t( + "A file containing the selected documents in PDF format." + ), + value: ExportContentType.Pdf, + }); + } + + return radioItems; + }, [t, includeChildDocuments]); + + return ( + + + {items.map((item) => ( + + ))} + + {hasChildDocuments && ( + <> +
+ + + )} + + ); +}); + +const Option = styled.label` + display: flex; + align-items: baseline; + gap: 16px; + + p { + margin: 0; + } +`; + +const StyledInput = styled.input` + position: relative; + top: 1.5px; +`; diff --git a/app/models/Document.ts b/app/models/Document.ts index dc63a3bdcb..ea65acae43 100644 --- a/app/models/Document.ts +++ b/app/models/Document.ts @@ -702,14 +702,21 @@ export default class Document extends ArchivableModel implements Searchable { ); } - download = (contentType: ExportContentType) => + download = ({ + contentType, + includeChildDocuments, + }: { + contentType: ExportContentType; + includeChildDocuments?: boolean; + }) => client.post( `/documents.export`, { id: this.id, + includeChildDocuments: includeChildDocuments ?? false, }, { - download: true, + ...(includeChildDocuments ? {} : { download: true }), headers: { accept: contentType, }, diff --git a/app/models/FileOperation.ts b/app/models/FileOperation.ts index e833c1e520..36ac362999 100644 --- a/app/models/FileOperation.ts +++ b/app/models/FileOperation.ts @@ -21,6 +21,8 @@ class FileOperation extends Model { collectionId: string | null; + documentId: string | null; + @observable size: number; diff --git a/app/scenes/Settings/components/FileOperationListItem.tsx b/app/scenes/Settings/components/FileOperationListItem.tsx index b652b20ac4..7a139fc847 100644 --- a/app/scenes/Settings/components/FileOperationListItem.tsx +++ b/app/scenes/Settings/components/FileOperationListItem.tsx @@ -57,7 +57,8 @@ const FileOperationListItem = ({ fileOperation }: Props) => { const format = formatMapping[fileOperation.format]; const title = fileOperation.type === FileOperationType.Import || - fileOperation.collectionId + fileOperation.collectionId || + fileOperation.documentId ? fileOperation.name : t("All collections"); diff --git a/server/migrations/20250719063818-add-documentId-to-fileOperation.js b/server/migrations/20250719063818-add-documentId-to-fileOperation.js new file mode 100644 index 0000000000..1bd5f9cc63 --- /dev/null +++ b/server/migrations/20250719063818-add-documentId-to-fileOperation.js @@ -0,0 +1,19 @@ +"use strict"; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + async up(queryInterface, Sequelize) { + await queryInterface.addColumn("file_operations", "documentId", { + type: Sequelize.UUID, + allowNull: true, + onDelete: "cascade", + references: { + model: "documents", + }, + }); + }, + + async down(queryInterface, Sequelize) { + await queryInterface.removeColumn("file_operations", "documentId"); + }, +}; diff --git a/server/models/FileOperation.ts b/server/models/FileOperation.ts index 8ffd001c1c..1cb9ef7d9b 100644 --- a/server/models/FileOperation.ts +++ b/server/models/FileOperation.ts @@ -13,14 +13,17 @@ import { Table, DataType, } from "sequelize-typescript"; +import { v4 as uuidv4 } from "uuid"; import type { CollectionPermission, FileOperationFormat } from "@shared/types"; import { FileOperationState, FileOperationType } from "@shared/types"; import FileStorage from "@server/storage/files"; import Collection from "./Collection"; +import Document from "./Document"; import Team from "./Team"; import User from "./User"; import ParanoidModel from "./base/ParanoidModel"; import Fix from "./decorators/Fix"; +import { Buckets } from "./helpers/AttachmentHelper"; export type FileOperationOptions = { includeAttachments?: boolean; @@ -38,6 +41,13 @@ export type FileOperationOptions = { { model: Collection, as: "collection", + required: false, + paranoid: false, + }, + { + model: Document.scope("withDrafts"), + as: "document", + required: false, paranoid: false, }, ], @@ -130,12 +140,19 @@ class FileOperation extends ParanoidModel< teamId: string; @BelongsTo(() => Collection, "collectionId") - collection: Collection; + collection: Collection | null; @ForeignKey(() => Collection) @Column(DataType.UUID) collectionId?: string | null; + @BelongsTo(() => Document, "documentId") + document: Document | null; + + @ForeignKey(() => Document) + @Column(DataType.UUID) + documentId?: string | null; + /** * Count the number of export file operations for a given team after a point * in time. @@ -159,6 +176,20 @@ class FileOperation extends ParanoidModel< }, }); } + + static getExportKey({ + name, + teamId, + format, + }: { + name: string; + teamId: string; + format: FileOperationFormat; + }) { + return `${ + Buckets.uploads + }/${teamId}/${uuidv4()}/${name}-export.${format.replace(/outline-/, "")}.zip`; + } } export default FileOperation; diff --git a/server/presenters/fileOperation.ts b/server/presenters/fileOperation.ts index b709019c3c..5436159bdf 100644 --- a/server/presenters/fileOperation.ts +++ b/server/presenters/fileOperation.ts @@ -7,11 +7,15 @@ export default function presentFileOperation(data: FileOperation) { id: data.id, type: data.type, format: data.format, - name: data.collection?.name || path.basename(data.key || ""), + name: + data.collection?.name || + data.document?.titleWithDefault || + path.basename(data.key || ""), state: data.state, error: data.error, size: data.size, collectionId: data.collectionId, + documentId: data.documentId, user: presentUser(data.user), createdAt: data.createdAt, updatedAt: data.updatedAt, diff --git a/server/queues/tasks/ExportDocumentTreeTask.ts b/server/queues/tasks/ExportDocumentTreeTask.ts index 037c2ef049..77aa236d36 100644 --- a/server/queues/tasks/ExportDocumentTreeTask.ts +++ b/server/queues/tasks/ExportDocumentTreeTask.ts @@ -22,7 +22,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { * @param pathInZip The path in the zip to add the document to * @param format The format to export in */ - protected async addDocumentToArchive({ + protected async processDocument({ zip, pathInZip, documentId, @@ -154,7 +154,7 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { const documentId = path[0].replace("/doc/", ""); const pathInZip = path[1]; - await this.addDocumentToArchive({ + await this.processDocument({ zip, pathInZip, documentId, @@ -169,6 +169,56 @@ export default abstract class ExportDocumentTreeTask extends ExportTask { return await ZipHelper.toTmpFile(zip); } + protected async addDocumentToArchive({ + document, + format, + documentStructure, + zip, + }: { + document: Document; + format: FileOperationFormat; + documentStructure: NavigationNode[]; + zip: JSZip; + }) { + const pathMap = new Map(); + + const extension = format === FileOperationFormat.HTMLZip ? "html" : "md"; + const rootFolderName = serializeFilename(document.titleWithDefault); + + // entry for root document + pathMap.set(document.path, `${rootFolderName}.${extension}`); + + this.addDocumentTreeToPathMap( + pathMap, + documentStructure, + serializeFilename(document.titleWithDefault), + format + ); + + Logger.debug( + "task", + `Start adding ${Object.values(pathMap).length} documents to archive` + ); + + for (const entry of pathMap) { + const documentId = entry[0].replace("/doc/", ""); + const pathInZip = entry[1]; + + await this.processDocument({ + zip, + pathInZip, + documentId, + includeAttachments: true, + format, + pathMap, + }); + } + + Logger.debug("task", "Completed adding documents to archive"); + + return await ZipHelper.toTmpFile(zip); + } + /** * Generates a map of document urls to their path in the zip file. * diff --git a/server/queues/tasks/ExportHTMLZipTask.ts b/server/queues/tasks/ExportHTMLZipTask.ts index f3a3f0c770..72d6d90b3b 100644 --- a/server/queues/tasks/ExportHTMLZipTask.ts +++ b/server/queues/tasks/ExportHTMLZipTask.ts @@ -1,10 +1,15 @@ import JSZip from "jszip"; +import type { NavigationNode } from "@shared/types"; import { FileOperationFormat } from "@shared/types"; import type { Collection, FileOperation } from "@server/models"; +import type { Document } from "@server/models"; import ExportDocumentTreeTask from "./ExportDocumentTreeTask"; export default class ExportHTMLZipTask extends ExportDocumentTreeTask { - public async export(collections: Collection[], fileOperation: FileOperation) { + public async exportCollections( + collections: Collection[], + fileOperation: FileOperation + ) { const zip = new JSZip(); return await this.addCollectionsToArchive( @@ -14,4 +19,18 @@ export default class ExportHTMLZipTask extends ExportDocumentTreeTask { fileOperation.options?.includeAttachments ?? true ); } + + public async exportDocument( + document: Document, + documentStructure: NavigationNode[] + ): Promise { + const zip = new JSZip(); + + return await this.addDocumentToArchive({ + document, + documentStructure, + format: FileOperationFormat.HTMLZip, + zip, + }); + } } diff --git a/server/queues/tasks/ExportJSONTask.ts b/server/queues/tasks/ExportJSONTask.ts index 3b14f45608..c2bfe27ca6 100644 --- a/server/queues/tasks/ExportJSONTask.ts +++ b/server/queues/tasks/ExportJSONTask.ts @@ -15,7 +15,10 @@ import packageJson from "../../../package.json"; import ExportTask from "./ExportTask"; export default class ExportJSONTask extends ExportTask { - public async export(collections: Collection[], fileOperation: FileOperation) { + public async exportCollections( + collections: Collection[], + fileOperation: FileOperation + ) { const zip = new JSZip(); // serial to avoid overloading, slow and steady wins the race @@ -170,4 +173,8 @@ export default class ExportJSONTask extends ExportTask { : JSON.stringify(output) ); } + + public async exportDocument(): Promise { + throw new Error("JSON export unsupported for individual document."); + } } diff --git a/server/queues/tasks/ExportMarkdownZipTask.ts b/server/queues/tasks/ExportMarkdownZipTask.ts index 9f3bee0aad..a3913328ef 100644 --- a/server/queues/tasks/ExportMarkdownZipTask.ts +++ b/server/queues/tasks/ExportMarkdownZipTask.ts @@ -1,10 +1,15 @@ import JSZip from "jszip"; +import type { NavigationNode } from "@shared/types"; import { FileOperationFormat } from "@shared/types"; import type { Collection, FileOperation } from "@server/models"; +import type { Document } from "@server/models"; import ExportDocumentTreeTask from "./ExportDocumentTreeTask"; export default class ExportMarkdownZipTask extends ExportDocumentTreeTask { - public async export(collections: Collection[], fileOperation: FileOperation) { + public async exportCollections( + collections: Collection[], + fileOperation: FileOperation + ) { const zip = new JSZip(); return await this.addCollectionsToArchive( @@ -14,4 +19,18 @@ export default class ExportMarkdownZipTask extends ExportDocumentTreeTask { fileOperation.options?.includeAttachments ); } + + public async exportDocument( + document: Document, + documentStructure: NavigationNode[] + ): Promise { + const zip = new JSZip(); + + return await this.addDocumentToArchive({ + document, + documentStructure, + format: FileOperationFormat.MarkdownZip, + zip, + }); + } } diff --git a/server/queues/tasks/ExportTask.ts b/server/queues/tasks/ExportTask.ts index 6c8fe50e09..b3f7e0d798 100644 --- a/server/queues/tasks/ExportTask.ts +++ b/server/queues/tasks/ExportTask.ts @@ -1,6 +1,11 @@ import fs from "fs-extra"; import truncate from "lodash/truncate"; -import { FileOperationState, NotificationEventType } from "@shared/types"; +import type { + NavigationNode} from "@shared/types"; +import { + FileOperationState, + NotificationEventType, +} from "@shared/types"; import { bytesToHumanReadable } from "@shared/utils/files"; import ExportFailureEmail from "@server/emails/templates/ExportFailureEmail"; import ExportSuccessEmail from "@server/emails/templates/ExportSuccessEmail"; @@ -10,6 +15,7 @@ import Logger from "@server/logging/Logger"; import { Attachment, Collection, + Document, Event, FileOperation, Team, @@ -19,7 +25,6 @@ import fileOperationPresenter from "@server/presenters/fileOperation"; import FileStorage from "@server/storage/files"; import { BaseTask, TaskPriority } from "./base/BaseTask"; import { Op } from "sequelize"; -import type { WhereOptions } from "sequelize"; import { sequelizeReadOnly } from "@server/storage/database"; type Props = { @@ -45,49 +50,6 @@ export default abstract class ExportTask extends BaseTask { let filePath: string | undefined; try { - const where: WhereOptions = { - teamId: user.teamId, - }; - - if (!fileOperation.options?.includePrivate) { - where.permission = { - [Op.ne]: null, - }; - } - - if (fileOperation.collectionId) { - where.id = fileOperation.collectionId; - } else { - where.archivedAt = { - [Op.eq]: null, - }; - } - - const collections = await Collection.scope( - "withDocumentStructure" - ).findAll({ where }); - - if (!fileOperation.collectionId) { - const totalAttachmentsSize = await Attachment.getTotalSizeForTeam( - sequelizeReadOnly, - user.teamId - ); - - if ( - fileOperation.options?.includeAttachments && - env.MAXIMUM_EXPORT_SIZE && - totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE - ) { - throw ValidationError( - `${bytesToHumanReadable( - totalAttachmentsSize - )} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable( - env.MAXIMUM_EXPORT_SIZE - )}.` - ); - } - } - Logger.info("task", `ExportTask processing data for ${fileOperationId}`, { options: fileOperation.options, }); @@ -96,7 +58,7 @@ export default abstract class ExportTask extends BaseTask { state: FileOperationState.Creating, }); - filePath = await this.export(collections, fileOperation); + filePath = await this.loadDataAndExport(fileOperation, user); Logger.info("task", `ExportTask uploading data for ${fileOperationId}`); @@ -151,17 +113,97 @@ export default abstract class ExportTask extends BaseTask { } } + public async loadDataAndExport( + fileOperation: FileOperation, + user: User + ): Promise { + if (fileOperation.documentId) { + const document = await Document.findByPk(fileOperation.documentId!, { + include: { + model: Collection.scope("withDocumentStructure"), + as: "collection", + }, + rejectOnEmpty: true, + }); + + const documentStructure = document.collection?.getDocumentTree( + document.id + ); + + if (!documentStructure) { + throw new Error("Document not found in collection tree"); + } + + return this.exportDocument(document, documentStructure.children ?? []); + } + + // ensure attachment size is within limits + if (!fileOperation.collectionId) { + const totalAttachmentsSize = await Attachment.getTotalSizeForTeam( + sequelizeReadOnly, + user.teamId + ); + + if ( + fileOperation.options?.includeAttachments && + env.MAXIMUM_EXPORT_SIZE && + totalAttachmentsSize > env.MAXIMUM_EXPORT_SIZE + ) { + throw ValidationError( + `${bytesToHumanReadable( + totalAttachmentsSize + )} of attachments in workspace is larger than maximum export size of ${bytesToHumanReadable( + env.MAXIMUM_EXPORT_SIZE + )}.` + ); + } + } + + const where = fileOperation.collectionId + ? { + teamId: user.teamId, + id: fileOperation.collectionId, + } + : { + teamId: user.teamId, + archivedAt: { + [Op.ne]: null, + }, + }; + + const collections = await Collection.scope("withDocumentStructure").findAll( + { + where, + } + ); + + return this.exportCollections(collections, fileOperation); + } + /** * Transform the data in all of the passed collections into a single Buffer. * * @param collections The collections to export * @returns A promise that resolves to a temporary file path */ - protected abstract export( + protected abstract exportCollections( collections: Collection[], fileOperation: FileOperation ): Promise; + /** + * Transform the data in the document into a single Buffer. + * + * @param document The document to export + * @param documentStructure Structure of document's children + * @param fileOperation File operation associated with the export + * @returns A promise that resolves to a temporary file path + */ + protected abstract exportDocument( + document: Document, + documentStructure: NavigationNode[] + ): Promise; + /** * Update the state of the underlying FileOperation in the database and send * an event to the client. diff --git a/server/routes/api/documents/documents.ts b/server/routes/api/documents/documents.ts index fdb14a2fa7..755a61760a 100644 --- a/server/routes/api/documents/documents.ts +++ b/server/routes/api/documents/documents.ts @@ -15,7 +15,13 @@ import type { Order, ScopeOptions, WhereOptions } from "sequelize"; import { Op, Sequelize } from "sequelize"; import { randomUUID } from "crypto"; import type { NavigationNode } from "@shared/types"; -import { StatusFilter, UserRole } from "@shared/types"; +import { + FileOperationFormat, + FileOperationState, + FileOperationType, + StatusFilter, + UserRole, +} from "@shared/types"; import { subtractDate } from "@shared/utils/date"; import slugify from "@shared/utils/slugify"; import documentCreator from "@server/commands/documentCreator"; @@ -52,6 +58,7 @@ import { Group, GroupUser, GroupMembership, + FileOperation, } from "@server/models"; import AttachmentHelper from "@server/models/helpers/AttachmentHelper"; import { DocumentHelper } from "@server/models/helpers/DocumentHelper"; @@ -66,6 +73,7 @@ import { presentUser, presentGroupMembership, presentGroup, + presentFileOperation, } from "@server/presenters"; import type { DocumentImportTaskResponse } from "@server/queues/tasks/DocumentImportTask"; import DocumentImportTask from "@server/queues/tasks/DocumentImportTask"; @@ -744,7 +752,7 @@ router.post( auth(), validate(T.DocumentsExportSchema), async (ctx: APIContext) => { - const { id } = ctx.input.body; + const { id, includeChildDocuments } = ctx.input.body; const { user } = ctx.state.auth; const accept = ctx.request.headers["accept"]; @@ -755,20 +763,67 @@ router.post( includeState: !accept?.includes("text/markdown"), }); + authorize(user, "download", document); + + const format = accept?.includes("text/html") + ? FileOperationFormat.HTMLZip + : accept?.includes("text/markdown") + ? FileOperationFormat.MarkdownZip + : accept?.includes("application/pdf") + ? FileOperationFormat.PDF + : null; + + if (format === FileOperationFormat.PDF) { + throw IncorrectEditionError( + "PDF export is not available in the community edition" + ); + } + + if (includeChildDocuments) { + if (!format) { + throw InvalidRequestError( + "format needed for exporting nested documents" + ); + } + + const fileOperation = await FileOperation.createWithCtx(ctx, { + type: FileOperationType.Export, + state: FileOperationState.Creating, + format, + key: FileOperation.getExportKey({ + name: document.titleWithDefault, + teamId: document.teamId, + format, + }), + url: null, + size: 0, + documentId: document.id, + userId: user.id, + teamId: document.teamId, + }); + + fileOperation.user = user; + fileOperation.document = document; + + ctx.body = { + success: true, + data: { + fileOperation: presentFileOperation(fileOperation), + }, + }; + return; + } + let contentType: string; let content: string; - if (accept?.includes("text/html")) { + if (format === FileOperationFormat.HTMLZip) { contentType = "text/html"; content = await DocumentHelper.toHTML(document, { centered: true, includeMermaid: true, }); - } else if (accept?.includes("application/pdf")) { - throw IncorrectEditionError( - "PDF export is not available in the community edition" - ); - } else if (accept?.includes("text/markdown")) { + } else if (format === FileOperationFormat.MarkdownZip) { contentType = "text/markdown"; content = DocumentHelper.toMarkdown(document); } else { diff --git a/server/routes/api/documents/schema.ts b/server/routes/api/documents/schema.ts index bf2d94229d..3483b65ee7 100644 --- a/server/routes/api/documents/schema.ts +++ b/server/routes/api/documents/schema.ts @@ -152,7 +152,9 @@ export const DocumentsInfoSchema = BaseSchema.extend({ export type DocumentsInfoReq = z.infer; export const DocumentsExportSchema = BaseSchema.extend({ - body: BaseIdSchema, + body: BaseIdSchema.extend({ + includeChildDocuments: z.boolean().default(false), + }), }); export type DocumentsExportReq = z.infer; diff --git a/shared/constants.ts b/shared/constants.ts index 96bbbb8977..5ae59e784f 100644 --- a/shared/constants.ts +++ b/shared/constants.ts @@ -1,5 +1,10 @@ import type { TeamPreferences, UserPreferences } from "./types"; -import { TOCPosition, TeamPreference, UserPreference, EmailDisplay } from "./types"; +import { + TOCPosition, + TeamPreference, + UserPreference, + EmailDisplay, +} from "./types"; export const MAX_AVATAR_DISPLAY = 6; diff --git a/shared/editor/extensions/Mermaid.ts b/shared/editor/extensions/Mermaid.ts index 5dd9283205..33cbdd895d 100644 --- a/shared/editor/extensions/Mermaid.ts +++ b/shared/editor/extensions/Mermaid.ts @@ -168,9 +168,7 @@ function getNewState({ const decorations: Decoration[] = []; // Find all blocks that represent Mermaid diagrams (supports both "mermaid" and "mermaidjs") - const blocks = findBlockNodes(doc).filter( - (item) => isMermaid(item.node) - ); + const blocks = findBlockNodes(doc).filter((item) => isMermaid(item.node)); blocks.forEach((block) => { const existingDecorations = pluginState.decorationSet.find( @@ -272,9 +270,7 @@ export default function Mermaid({ !mermaidMeta ) { const codeBlock = findParentNode(isCode)(state.selection); - let isEditing = - codeBlock && - isMermaid(codeBlock.node); + let isEditing = codeBlock && isMermaid(codeBlock.node); if (isEditing && codeBlock && !transaction.docChanged) { const decorations = nextPluginState.decorationSet.find( @@ -401,10 +397,7 @@ export default function Mermaid({ ); const nextBlock = $pos.nodeAfter; - if ( - nextBlock && - isMermaid(nextBlock) - ) { + if (nextBlock && isMermaid(nextBlock)) { view.dispatch( view.state.tr .setSelection( @@ -426,10 +419,7 @@ export default function Mermaid({ ); const prevBlock = $pos.nodeBefore; - if ( - prevBlock && - isMermaid(prevBlock) - ) { + if (prevBlock && isMermaid(prevBlock)) { view.dispatch( view.state.tr .setSelection( diff --git a/shared/editor/lib/isCode.ts b/shared/editor/lib/isCode.ts index 7bec47de37..ca1dc8c628 100644 --- a/shared/editor/lib/isCode.ts +++ b/shared/editor/lib/isCode.ts @@ -11,5 +11,8 @@ export function isCode(node: Node) { * @returns true if the node is a Mermaid code block. */ export function isMermaid(node: Node) { - return isCode(node) && (node.attrs.language === "mermaid" || node.attrs.language === "mermaidjs"); + return ( + isCode(node) && + (node.attrs.language === "mermaid" || node.attrs.language === "mermaidjs") + ); } diff --git a/shared/i18n/locales/en_US/translation.json b/shared/i18n/locales/en_US/translation.json index 03d32cd05a..557a5a8bff 100644 --- a/shared/i18n/locales/en_US/translation.json +++ b/shared/i18n/locales/en_US/translation.json @@ -62,12 +62,11 @@ "Unpublished {{ documentName }}": "Unpublished {{ documentName }}", "Subscription inherited from collection": "Subscription inherited from collection", "Share this document": "Share this document", - "HTML": "HTML", - "PDF": "PDF", - "Exporting": "Exporting", - "Markdown": "Markdown", "Download": "Download", "Download document": "Download document", + "Downloas as Markdown": "Downloas as Markdown", + "Download as HTML": "Download as HTML", + "Download as PDF": "Download as PDF", "Copy as Markdown": "Copy as Markdown", "Markdown copied to clipboard": "Markdown copied to clipboard", "Copy as text": "Copy as text", @@ -144,6 +143,10 @@ "New Application": "New Application", "This version of the document was deleted": "This version of the document was deleted", "Link copied": "Link copied", + "HTML": "HTML", + "PDF": "PDF", + "Exporting": "Exporting", + "Markdown": "Markdown", "Download revision": "Download revision", "Dark": "Dark", "Light": "Light", @@ -230,6 +233,16 @@ "Include nested documents": "Include nested documents", "Copy to {{ location }}": "Copy to {{ location }}", "Copying": "Copying", + "Your file will be available in {{ location }} soon": "Your file will be available in {{ location }} soon", + "View": "View", + "A link to your file will be sent through email soon": "A link to your file will be sent through email soon", + "Export started": "Export started", + "A file containing the selected documents in Markdown format.": "A file containing the selected documents in Markdown format.", + "A file containing the selected documents in HTML format.": "A file containing the selected documents in HTML format.", + "A file containing the selected documents in PDF format.": "A file containing the selected documents in PDF format.", + "Include child documents": "Include child documents", + "When selected, exporting the document {{documentName}} may take some time.": "When selected, exporting the document {{documentName}} may take some time.", + "You will receive an email when it's complete.": "You will receive an email when it's complete.", "Search collections & documents": "Search collections & documents", "No results found": "No results found", "Document options": "Document options", @@ -296,15 +309,10 @@ "{{userName}} published": "{{userName}} published", "{{userName}} unpublished": "{{userName}} unpublished", "{{userName}} moved": "{{userName}} moved", - "Your file will be available in {{ location }} soon": "Your file will be available in {{ location }} soon", - "View": "View", - "A link to your file will be sent through email soon": "A link to your file will be sent through email soon", - "Export started": "Export started", "A ZIP file containing the images, and documents in the Markdown format.": "A ZIP file containing the images, and documents in the Markdown format.", "A ZIP file containing the images, and documents as HTML files.": "A ZIP file containing the images, and documents as HTML files.", "Structured data that can be used to transfer data to another compatible {{ appName }} instance.": "Structured data that can be used to transfer data to another compatible {{ appName }} instance.", "Exporting the collection {{collectionName}} may take some time.": "Exporting the collection {{collectionName}} may take some time.", - "You will receive an email when it's complete.": "You will receive an email when it's complete.", "Include attachments": "Include attachments", "Including uploaded images and files in the exported data": "Including uploaded images and files in the exported data", "Include private collections": "Include private collections",