Stop writing text to revisions

server/commands/documentCollaborativeUpdater.ts

@@ -1,10 +1,8 @@
 import isEqual from "fast-deep-equal";
 import uniq from "lodash/uniq";
-import { Node } from "prosemirror-model";
 import { yDocToProsemirrorJSON } from "y-prosemirror";
 import * as Y from "yjs";
 import { ProsemirrorData } from "@shared/types";
-import { schema, serializer } from "@server/editor";
 import Logger from "@server/logging/Logger";
 import { Document, Event } from "@server/models";
 import { sequelize } from "@server/storage/database";
@@ -45,8 +43,6 @@ export default async function documentCollaborativeUpdater({
 
     const state = Y.encodeStateAsUpdate(ydoc);
     const content = yDocToProsemirrorJSON(ydoc, "default") as ProsemirrorData;
-    const node = Node.fromJSON(schema, content);
-    const text = serializer.serialize(node, undefined);
     const isUnchanged = isEqual(document.content, content);
     const lastModifiedById =
       sessionCollaboratorIds[sessionCollaboratorIds.length - 1] ??
@@ -72,7 +68,6 @@ export default async function documentCollaborativeUpdater({
 
     await document.update(
       {
-        text,
         content,
         state: Buffer.from(state),
         lastModifiedById,

server/commands/documentCreator.ts

@@ -1,8 +1,9 @@
 import { Optional } from "utility-types";
-import { ProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
+import { ProsemirrorHelper as SharedProsemirrorHelper } from "@shared/utils/ProsemirrorHelper";
 import { TextHelper } from "@shared/utils/TextHelper";
 import { Document, Event, User } from "@server/models";
 import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
+import { ProsemirrorHelper } from "@server/models/helpers/ProsemirrorHelper";
 import { APIContext } from "@server/types";
 
 type Props = Optional<
@@ -81,53 +82,58 @@ export default async function documentCreator({
     }
   }
 
-  const document = await Document.create(
-    {
-      id,
-      urlId,
-      parentDocumentId,
-      editorVersion,
-      collectionId,
-      teamId: user.teamId,
-      createdAt,
-      updatedAt: updatedAt ?? createdAt,
-      lastModifiedById: user.id,
-      createdById: user.id,
-      template,
-      templateId,
-      publishedAt,
-      importId,
-      sourceMetadata,
-      fullWidth: templateDocument ? templateDocument.fullWidth : fullWidth,
-      icon: templateDocument ? templateDocument.icon : icon,
-      color: templateDocument ? templateDocument.color : color,
-      title:
-        title ??
-        (templateDocument
-          ? template
-            ? templateDocument.title
-            : TextHelper.replaceTemplateVariables(templateDocument.title, user)
-          : ""),
-      text:
-        text ??
-        (templateDocument
-          ? template
-            ? templateDocument.text
-            : TextHelper.replaceTemplateVariables(templateDocument.text, user)
-          : ""),
-      content: templateDocument
-        ? ProsemirrorHelper.replaceTemplateVariables(
-            await DocumentHelper.toJSON(templateDocument),
-            user
-          )
-        : content,
-      state,
-    },
-    {
-      silent: !!createdAt,
-      transaction,
-    }
-  );
+  const titleWithReplacements =
+    title ??
+    (templateDocument
+      ? template
+        ? templateDocument.title
+        : TextHelper.replaceTemplateVariables(templateDocument.title, user)
+      : "");
+
+  const contentWithReplacements = text
+    ? ProsemirrorHelper.toProsemirror(text).toJSON()
+    : templateDocument
+    ? template
+      ? templateDocument.content
+      : SharedProsemirrorHelper.replaceTemplateVariables(
+          await DocumentHelper.toJSON(templateDocument),
+          user
+        )
+    : content;
+
+  const document = Document.build({
+    id,
+    urlId,
+    parentDocumentId,
+    editorVersion,
+    collectionId,
+    teamId: user.teamId,
+    createdAt,
+    updatedAt: updatedAt ?? createdAt,
+    lastModifiedById: user.id,
+    createdById: user.id,
+    template,
+    templateId,
+    publishedAt,
+    importId,
+    sourceMetadata,
+    fullWidth: fullWidth ?? templateDocument?.fullWidth,
+    icon: icon ?? templateDocument?.icon,
+    color: color ?? templateDocument?.color,
+    title: titleWithReplacements,
+    content: contentWithReplacements,
+    state,
+  });
+
+  document.text = DocumentHelper.toMarkdown(document, {
+    includeTitle: false,
+  });
+
+  await document.save({
+    silent: !!createdAt,
+    transaction,
+  });
+
   await Event.create(
     {
       name: "documents.create",

server/commands/documentDuplicator.ts

@@ -52,7 +52,6 @@ export default async function documentDuplicator({
       DocumentHelper.toProsemirror(document),
       ["comment"]
     ),
-    text: document.text,
     ...sharedProperties,
   });
 
@@ -86,7 +85,6 @@ export default async function documentDuplicator({
           DocumentHelper.toProsemirror(childDocument),
           ["comment"]
         ),
-        text: childDocument.text,
         ...sharedProperties,
       });
 

server/models/Document.ts

@@ -830,7 +830,9 @@ class Document extends ArchivableModel<
     }
 
     this.content = revision.content;
-    this.text = revision.text;
+    this.text = DocumentHelper.toMarkdown(revision, {
+      includeTitle: false,
+    });
     this.title = revision.title;
     this.icon = revision.icon;
     this.color = revision.color;

server/models/Revision.test.ts

@@ -16,6 +16,5 @@ describe("#findLatest", () => {
     await Revision.createFromDocument(document);
     const revision = await Revision.findLatest(document.id);
     expect(revision?.title).toBe("Changed 2");
-    expect(revision?.text).toBe("Content");
   });
 });

server/models/Revision.ts

@@ -69,8 +69,9 @@ class Revision extends IdModel<
   /**
    * The content of the revision as Markdown.
    *
-   * @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if exporting lossy markdown.
-   * This column will be removed in a future migration.
+   * @deprecated Use `content` instead, or `DocumentHelper.toMarkdown` if
+   * exporting lossy markdown. This column will be removed in a future migration
+   * and is no longer being written.
    */
   @Column(DataType.TEXT)
   text: string;
@@ -134,7 +135,6 @@ class Revision extends IdModel<
   static buildFromDocument(document: Document) {
     return this.build({
       title: document.title,
-      text: document.text,
       icon: document.icon,
       color: document.color,
       content: document.content,

server/models/helpers/DocumentHelper.tsx

@@ -147,10 +147,15 @@ export class DocumentHelper {
    * Returns the document as Markdown. This is a lossy conversion and should only be used for export.
    *
    * @param document The document or revision to convert
+   * @param options Options for the conversion
    * @returns The document title and content as a Markdown string
    */
   static toMarkdown(
-    document: Document | Revision | Collection | ProsemirrorData
+    document: Document | Revision | Collection | ProsemirrorData,
+    options?: {
+      /** Whether to include the document title (default: true) */
+      includeTitle?: boolean;
+    }
   ) {
     const text = serializer
       .serialize(DocumentHelper.toProsemirror(document))
@@ -165,7 +170,10 @@ export class DocumentHelper {
       return text;
     }
 
-    if (document instanceof Document || document instanceof Revision) {
+    if (
+      (document instanceof Document || document instanceof Revision) &&
+      options?.includeTitle !== false
+    ) {
       const iconType = determineIconType(document.icon);
 
       const title = `${iconType === IconType.Emoji ? document.icon + " " : ""}${

server/models/helpers/ProseMirrorHelper.test.ts

@@ -3,7 +3,7 @@ import { MentionType, ProsemirrorData } from "@shared/types";
 import { buildProseMirrorDoc } from "@server/test/factories";
 import { MentionAttrs, ProsemirrorHelper } from "./ProsemirrorHelper";
 
-describe("ProseMirrorHelper", () => {
+describe("ProsemirrorHelper", () => {
   describe("getNodeForMentionEmail", () => {
     it("should return the paragraph node", () => {
       const mentionAttrs: MentionAttrs = {

server/models/helpers/ProsemirrorHelper.tsx

@@ -118,10 +118,13 @@ export class ProsemirrorHelper {
   /**
    * Converts a plain object into a Prosemirror Node.
    *
-   * @param data The object to parse
+   * @param data The ProsemirrorData object or string to parse.
    * @returns The content as a Prosemirror Node
    */
-  static toProsemirror(data: ProsemirrorData) {
+  static toProsemirror(data: ProsemirrorData | string) {
+    if (typeof data === "string") {
+      return parser.parse(data);
+    }
     return Node.fromJSON(schema, data);
   }
 

server/presenters/document.ts

@@ -42,7 +42,7 @@ async function presentDocument(
 
   const text =
     !asData || options?.includeText
-      ? document.text || DocumentHelper.toMarkdown(data)
+      ? DocumentHelper.toMarkdown(data, { includeTitle: false })
       : undefined;
 
   const res: Record<string, any> = {

server/queues/processors/RevisionsProcessor.ts

@@ -2,6 +2,7 @@ import isEqual from "fast-deep-equal";
 import revisionCreator from "@server/commands/revisionCreator";
 import { Revision, Document, User } from "@server/models";
 import { DocumentEvent, RevisionEvent, Event } from "@server/types";
+import DocumentUpdateTextTask from "../tasks/DocumentUpdateTextTask";
 import BaseProcessor from "./BaseProcessor";
 
 export default class RevisionsProcessor extends BaseProcessor {
@@ -36,6 +37,8 @@ export default class RevisionsProcessor extends BaseProcessor {
           return;
         }
 
+        await DocumentUpdateTextTask.schedule(event);
+
         const user = await User.findByPk(event.actorId, {
           paranoid: false,
           rejectOnEmpty: true,

server/queues/tasks/DocumentUpdateTextTask.ts

@@ -0,0 +1,18 @@
+import { Node } from "prosemirror-model";
+import { schema, serializer } from "@server/editor";
+import { Document } from "@server/models";
+import { DocumentEvent } from "@server/types";
+import BaseTask from "./BaseTask";
+
+export default class DocumentUpdateTextTask extends BaseTask<DocumentEvent> {
+  public async perform(event: DocumentEvent) {
+    const document = await Document.findByPk(event.documentId);
+    if (!document?.content) {
+      return;
+    }
+
+    const node = Node.fromJSON(schema, document.content);
+    document.text = serializer.serialize(node);
+    await document.save({ silent: true });
+  }
+}

server/routes/api/documents/documents.test.ts

@@ -8,6 +8,7 @@ import {
 } from "@shared/types";
 import { TextHelper } from "@shared/utils/TextHelper";
 import { createContext } from "@server/context";
+import { parser } from "@server/editor";
 import {
   Document,
   View,
@@ -3257,21 +3258,26 @@ describe("#documents.restore", () => {
       teamId: user.teamId,
     });
     const revision = await Revision.createFromDocument(document);
-    const previousText = revision.text;
+    const previous = revision.content;
     const revisionId = revision.id;
+
     // update the document contents
-    document.text = "UPDATED";
+    document.content = parser.parse("updated")?.toJSON();
     await document.save();
+
     const res = await server.post("/api/documents.restore", {
       body: {
         token: user.getJwtToken(),
         id: document.id,
         revisionId,
       },
+      headers: {
+        "x-api-version": 3,
+      },
     });
     const body = await res.json();
     expect(res.status).toEqual(200);
-    expect(body.data.text).toEqual(previousText);
+    expect(body.data.data).toEqual(previous);
   });
 
   it("should not allow restoring a revision in another document", async () => {