use maxStateLength for text

Based on feedback, removed the validation behavior selection from the import dialog.
The backend still supports all three behaviors (Skip, Truncate, Abort) but now
defaults to Truncate behavior without exposing the option in the UI.

This makes the import dialog cleaner and simpler for users while still providing
the improved validation handling behind the scenes.

server/commands/documentImporter.ts

@@ -16,6 +16,7 @@ type Props = {
   fileName: string;
   content: Buffer | string;
   ctx: APIContext;
+  skipValidation?: boolean;
 };
 
 async function documentImporter({
@@ -24,6 +25,7 @@ async function documentImporter({
   content,
   user,
   ctx,
+  skipValidation = false,
 }: Props): Promise<{
   icon?: string;
   text: string;
@@ -63,7 +65,7 @@ async function documentImporter({
   text = await TextHelper.replaceImagesWithAttachments(ctx, text, user);
 
   // Sanity check – text cannot possibly be longer than state so if it is, we can short-circuit here
-  if (text.length > DocumentValidation.maxStateLength) {
+  if (!skipValidation && text.length > DocumentValidation.maxStateLength) {
     throw InvalidRequestError(
       `The document "${title}" is too large to import, please reduce the length and try again`
     );
@@ -75,7 +77,7 @@ async function documentImporter({
   const ydoc = ProsemirrorHelper.toYDoc(text);
   const state = ProsemirrorHelper.toState(ydoc);
 
-  if (state.length > DocumentValidation.maxStateLength) {
+  if (!skipValidation && state.length > DocumentValidation.maxStateLength) {
     throw InvalidRequestError(
       `The document "${title}" is too large to import, please reduce the length and try again`
     );

server/queues/tasks/ImportJSONTask.ts

@@ -125,7 +125,10 @@ export default class ImportJSONTask extends ImportTask {
       try {
         item = JSON.parse(await fs.readFile(node.path, "utf8"));
       } catch (err) {
-        throw new Error(`Could not parse ${node.path}. ${err.message}`);
+        Logger.warn(
+          `Skipping unparseable collection file: ${node.path}. ${err.message}`
+        );
+        continue;
       }
 
       const collectionId = uuidv4();

server/queues/tasks/ImportMarkdownZipTask.ts

@@ -95,6 +95,7 @@ export default class ImportMarkdownZipTask extends ImportTask {
                     : await fs.readFile(child.path, "utf8"),
                 user,
                 ctx: createContext({ user, transaction }),
+                skipValidation: true,
               })
           );
 

server/queues/tasks/ImportTask.ts

@@ -11,7 +11,7 @@ import {
   FileOperationState,
   ProsemirrorData,
 } from "@shared/types";
-import { CollectionValidation } from "@shared/validations";
+import { CollectionValidation, DocumentValidation } from "@shared/validations";
 import attachmentCreator from "@server/commands/attachmentCreator";
 import documentCreator from "@server/commands/documentCreator";
 import { createContext } from "@server/context";
@@ -423,14 +423,18 @@ export default abstract class ImportTask extends BaseTask<Props> {
           collections.set(item.id, collection);
 
           // Documents
-          for (const item of data.documents.filter(
+          for (const docItem of data.documents.filter(
             (d) => d.collectionId === collection.id
           )) {
             Logger.debug(
               "task",
-              `ImportTask persisting document ${item.title} (${item.id})`
+              `ImportTask persisting document ${docItem.title} (${docItem.id})`
             );
-            let text = item.text;
+
+            // Apply validation behavior
+            const processedDocument = this.preprocessDocument(docItem);
+
+            let text = processedDocument.text;
 
             // Check all of the attachments we've created against urls in the text
             // and replace them out with attachment redirect urls before saving.
@@ -452,28 +456,32 @@ export default abstract class ImportTask extends BaseTask<Props> {
 
             const document = await documentCreator({
               sourceMetadata: {
-                fileName: path.basename(item.path),
-                mimeType: item.mimeType,
-                externalId: item.externalId,
-                createdByName: item.createdByName,
+                fileName: path.basename(processedDocument.path),
+                mimeType: processedDocument.mimeType,
+                externalId: processedDocument.externalId,
+                createdByName: processedDocument.createdByName,
               },
-              id: item.id,
-              title: item.title,
-              urlId: item.urlId,
+              id: processedDocument.id,
+              title: processedDocument.title,
+              urlId: processedDocument.urlId,
               text,
-              content: item.data,
-              icon: item.icon,
-              color: item.color,
-              collectionId: item.collectionId,
-              createdAt: item.createdAt,
-              updatedAt: item.updatedAt ?? item.createdAt,
-              publishedAt: item.updatedAt ?? item.createdAt ?? new Date(),
-              parentDocumentId: item.parentDocumentId,
+              content: processedDocument.data,
+              icon: processedDocument.icon,
+              color: processedDocument.color,
+              collectionId: processedDocument.collectionId,
+              createdAt: processedDocument.createdAt,
+              updatedAt:
+                processedDocument.updatedAt ?? processedDocument.createdAt,
+              publishedAt:
+                processedDocument.updatedAt ??
+                processedDocument.createdAt ??
+                new Date(),
+              parentDocumentId: processedDocument.parentDocumentId,
               importId: fileOperation.id,
               user,
               ctx: createContext({ user, transaction }),
             });
-            documents.set(item.id, document);
+            documents.set(processedDocument.id, document);
 
             await collection.addDocumentToStructure(document, undefined, {
               transaction,
@@ -537,6 +545,50 @@ export default abstract class ImportTask extends BaseTask<Props> {
     };
   }
 
+  /**
+   * Preprocess a document by truncating the content that fail validation rules.
+   *
+   * @param document The document to preprocess
+   * @returns The processed document
+   */
+  private preprocessDocument(
+    document: StructuredImportData["documents"][number]
+  ): StructuredImportData["documents"][number] {
+    const titleTooLong =
+      document.title.length > DocumentValidation.maxTitleLength;
+    const textTooLong =
+      document.text.length > DocumentValidation.maxStateLength;
+
+    const hasValidationIssues = titleTooLong || textTooLong;
+
+    if (!hasValidationIssues) {
+      return document;
+    }
+
+    const processedDocument = { ...document };
+
+    if (titleTooLong) {
+      processedDocument.title = truncate(document.title, {
+        length: DocumentValidation.maxTitleLength,
+        omission: "...",
+      });
+    }
+
+    if (textTooLong) {
+      processedDocument.text = truncate(document.text, {
+        length: DocumentValidation.maxStateLength,
+        omission: "\n\n[Content truncated due to size limits]",
+      });
+    }
+
+    Logger.info(
+      "task",
+      `Truncated document "${document.title}" due to validation issues`
+    );
+
+    return processedDocument;
+  }
+
   /**
    * Job options such as priority and retry strategy, as defined by Bull.
    */