perf: Presenting lists of imported documents causes database lockup (#11591)

server/presenters/document.ts

@@ -1,6 +1,8 @@
+import { Op } from "sequelize";
 import { Hour } from "@shared/utils/time";
 import { traceFunction } from "@server/logging/tracing";
 import type { Document } from "@server/models";
+import FileOperation from "@server/models/FileOperation";
 import { DocumentHelper } from "@server/models/helpers/DocumentHelper";
 import type { APIContext } from "@server/types";
 import presentUser from "./user";
@@ -89,7 +91,7 @@ async function presentDocument(
   }
 
   if (!options.isPublic) {
-    const source = await document.$get("import");
+    const source = document.import ?? (await document.$get("import"));
 
     res.tasks = document.tasks;
     res.isCollectionDeleted = await document.isCollectionDeleted();
@@ -118,3 +120,43 @@ async function presentDocument(
 export default traceFunction({
   spanName: "presenters",
 })(presentDocument);
+
+/**
+ * Batch-present multiple documents, fetching all related FileOperation records
+ * in a single query instead of one per document.
+ *
+ * @param ctx the API context.
+ * @param documents the documents to present.
+ * @param options presentation options forwarded to presentDocument.
+ * @returns array of presented document objects.
+ */
+export async function presentDocuments(
+  ctx: APIContext | undefined,
+  documents: Document[],
+  options?: Options | null
+) {
+  const opts = { isPublic: false, ...options };
+
+  if (!opts.isPublic) {
+    const importIds = documents
+      .filter((doc) => doc.sourceMetadata && doc.importId)
+      .map((doc) => doc.importId!);
+
+    if (importIds.length > 0) {
+      const sources = await FileOperation.unscoped().findAll({
+        where: { id: { [Op.in]: importIds } },
+      });
+      const sourceMap = new Map(sources.map((s) => [s.id, s]));
+
+      for (const doc of documents) {
+        if (doc.importId) {
+          doc.import = sourceMap.get(doc.importId) ?? null;
+        }
+      }
+    }
+  }
+
+  return Promise.all(
+    documents.map((document) => presentDocument(ctx, document, opts))
+  );
+}

server/presenters/index.ts

@@ -4,7 +4,7 @@ import presentAuthenticationProvider from "./authenticationProvider";
 import presentAvailableTeam from "./availableTeam";
 import presentCollection from "./collection";
 import presentComment from "./comment";
-import presentDocument from "./document";
+import presentDocument, { presentDocuments } from "./document";
 import presentEvent from "./event";
 import presentFileOperation from "./fileOperation";
 import presentGroup from "./group";
@@ -39,6 +39,7 @@ export {
   presentCollection,
   presentComment,
   presentDocument,
+  presentDocuments,
   presentEvent,
   presentFileOperation,
   presentGroup,

server/routes/api/documents/documents.ts

@@ -69,6 +69,7 @@ import { TextHelper } from "@server/models/helpers/TextHelper";
 import { authorize, cannot } from "@server/policies";
 import {
   presentDocument,
+  presentDocuments,
   presentPolicies,
   presentTemplate,
   presentMembership,
@@ -307,9 +308,7 @@ router.post(
       Document.count({ where }),
     ]);
 
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
     const policies = presentPolicies(user, documents);
 
     ctx.body = {
@@ -366,9 +365,7 @@ router.post(
       limit: ctx.state.pagination.limit,
     });
 
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
     const policies = presentPolicies(user, documents);
 
     ctx.body = {
@@ -425,9 +422,7 @@ router.post(
       offset: ctx.state.pagination.offset,
       limit: ctx.state.pagination.limit,
     });
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
     const policies = presentPolicies(user, documents);
 
     ctx.body = {
@@ -475,9 +470,7 @@ router.post(
       document.views = [view];
       return document;
     });
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
     const policies = presentPolicies(user, documents);
 
     ctx.body = {
@@ -534,9 +527,7 @@ router.post(
       offset: ctx.state.pagination.offset,
       limit: ctx.state.pagination.limit,
     });
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
     const policies = presentPolicies(user, documents);
 
     ctx.body = {
@@ -1033,9 +1024,7 @@ router.post(
       direction: direction as DirectionFilter,
     });
     const policies = presentPolicies(user, documents);
-    const data = await Promise.all(
-      documents.map((document) => presentDocument(ctx, document))
-    );
+    const data = await presentDocuments(ctx, documents);
 
     ctx.body = {
       pagination: ctx.state.pagination,
@@ -1380,9 +1369,7 @@ router.post(
 
     ctx.body = {
       data: {
-        documents: await Promise.all(
-          response.map((document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, response),
       },
       policies: presentPolicies(user, response),
     };
@@ -1435,9 +1422,7 @@ router.post(
 
     ctx.body = {
       data: {
-        documents: await Promise.all(
-          documents.map((doc) => presentDocument(ctx, doc))
-        ),
+        documents: await presentDocuments(ctx, documents),
         // Included for backwards compatibility
         collections: [],
       },

server/routes/api/groupMemberships/groupMemberships.ts

@@ -5,7 +5,7 @@ import auth from "@server/middlewares/authentication";
 import validate from "@server/middlewares/validate";
 import { Document, GroupMembership } from "@server/models";
 import {
-  presentDocument,
+  presentDocuments,
   presentGroup,
   presentGroupMembership,
   presentPolicies,
@@ -83,9 +83,7 @@ router.post(
       data: {
         groups: await Promise.all(groups.map(presentGroup)),
         groupMemberships: memberships.map(presentGroupMembership),
-        documents: await Promise.all(
-          documents.map((document: Document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
       },
       policies,
     };

server/routes/api/pins/pins.ts

@@ -8,7 +8,7 @@ import { Collection, Document, Pin } from "@server/models";
 import { authorize } from "@server/policies";
 import {
   presentPin,
-  presentDocument,
+  presentDocuments,
   presentPolicies,
 } from "@server/presenters";
 import type { APIContext } from "@server/types";
@@ -138,9 +138,7 @@ router.post(
       pagination: ctx.state.pagination,
       data: {
         pins: pins.map(presentPin),
-        documents: await Promise.all(
-          documents.map((document: Document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
       },
       policies,
     };

server/routes/api/relationships/relationships.ts

@@ -5,7 +5,7 @@ import { Document, Relationship } from "@server/models";
 import { authorize } from "@server/policies";
 import {
   presentRelationship,
-  presentDocument,
+  presentDocuments,
   presentPolicies,
 } from "@server/presenters";
 import type { APIContext } from "@server/types";
@@ -45,9 +45,7 @@ router.post(
     ctx.body = {
       data: {
         relationship: presentRelationship(relationship),
-        documents: await Promise.all(
-          documents.map((doc: Document) => presentDocument(ctx, doc))
-        ),
+        documents: await presentDocuments(ctx, documents),
       },
       policies: presentPolicies(user, documents),
     };
@@ -85,9 +83,7 @@ router.post(
       pagination: ctx.state.pagination,
       data: {
         relationships: relationships.map(presentRelationship),
-        documents: await Promise.all(
-          documents.map((document: Document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
         policies: presentPolicies(user, documents),
       },
       policies,

server/routes/api/stars/stars.ts

@@ -8,7 +8,7 @@ import { Document, Star, Collection } from "@server/models";
 import { authorize } from "@server/policies";
 import {
   presentStar,
-  presentDocument,
+  presentDocuments,
   presentPolicies,
 } from "@server/presenters";
 import type { APIContext } from "@server/types";
@@ -109,9 +109,7 @@ router.post(
       pagination: ctx.state.pagination,
       data: {
         stars: stars.map(presentStar),
-        documents: await Promise.all(
-          documents.map((document: Document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
       },
       policies,
     };

server/routes/api/suggestions/suggestions.ts

@@ -7,7 +7,11 @@ import validate from "@server/middlewares/validate";
 import { Group, User } from "@server/models";
 import SearchHelper from "@server/models/helpers/SearchHelper";
 import { can } from "@server/policies";
-import { presentDocument, presentGroup, presentUser } from "@server/presenters";
+import {
+  presentDocuments,
+  presentGroup,
+  presentUser,
+} from "@server/presenters";
 import type { APIContext } from "@server/types";
 import pagination from "../middlewares/pagination";
 import * as T from "./schema";
@@ -76,9 +80,7 @@ router.post(
     ctx.body = {
       pagination: ctx.state.pagination,
       data: {
-        documents: await Promise.all(
-          documents.map((document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
         users: users.map((user) =>
           presentUser(user, {
             includeEmail: !!can(actor, "readEmail", user),

server/routes/api/userMemberships/userMemberships.ts

@@ -6,7 +6,7 @@ import validate from "@server/middlewares/validate";
 import { Document, Event, UserMembership } from "@server/models";
 import { authorize } from "@server/policies";
 import {
-  presentDocument,
+  presentDocuments,
   presentMembership,
   presentPolicies,
 } from "@server/presenters";
@@ -55,9 +55,7 @@ router.post(
       pagination: ctx.state.pagination,
       data: {
         memberships: memberships.map(presentMembership),
-        documents: await Promise.all(
-          documents.map((document: Document) => presentDocument(ctx, document))
-        ),
+        documents: await presentDocuments(ctx, documents),
       },
       policies,
     };