feat: Add email headers to enhance threading (#7477)

* feat: Add email headers to enhance threading

* tsc

* review

* change comment mentioned email subject

* paginated load

* rename util method

* add tests

* group events

* test for unsupported notification

* typo

* review

server/emails/mailer.tsx

@@ -14,6 +14,8 @@ type SendMailOptions = {
   to: string;
   fromName?: string;
   replyTo?: string;
+  messageId?: string;
+  references?: string[];
   subject: string;
   previewText?: string;
   text: string;
@@ -113,6 +115,11 @@ export class Mailer {
   `;
   };
 
+  /**
+   *
+   * @param data Email headers and body
+   * @returns Message ID header from SMTP server
+   */
   sendMail = async (data: SendMailOptions): Promise<void> => {
     const { transporter } = this;
 
@@ -152,6 +159,9 @@ export class Mailer {
           : env.SMTP_FROM_EMAIL,
         replyTo: data.replyTo ?? env.SMTP_REPLY_EMAIL ?? env.SMTP_FROM_EMAIL,
         to: data.to,
+        messageId: data.messageId,
+        references: data.references,
+        inReplyTo: data.references?.at(-1),
         subject: data.subject,
         html,
         text: data.text,

server/emails/templates/BaseEmail.tsx

@@ -7,6 +7,7 @@ import Notification from "@server/models/Notification";
 import { taskQueue } from "@server/queues";
 import { TaskPriority } from "@server/queues/tasks/BaseTask";
 import { NotificationMetadata } from "@server/types";
+import { getEmailMessageId } from "@server/utils/emails";
 
 export interface EmailProps {
   to: string | null;
@@ -101,11 +102,21 @@ export default abstract class BaseEmail<
       return;
     }
 
+    const messageId = notification
+      ? getEmailMessageId(notification.id)
+      : undefined;
+
+    const references = notification
+      ? await Notification.emailReferences(notification)
+      : undefined;
+
     try {
       await mailer.sendMail({
         to: this.props.to,
         fromName: this.fromName?.(data),
         subject: this.subject(data),
+        messageId,
+        references,
         previewText: this.preview(data),
         component: (
           <>

server/emails/templates/CommentMentionedEmail.tsx

@@ -92,8 +92,8 @@ export default class CommentMentionedEmail extends BaseEmail<
     );
   }
 
-  protected subject({ actorName, document }: Props) {
-    return `${actorName} mentioned you in “${document.title}”`;
+  protected subject({ document }: Props) {
+    return `Mentioned you in “${document.title}”`;
   }
 
   protected preview({ actorName }: Props): string {

server/migrations/20240828081032-add-notifications-index.js

@@ -0,0 +1,12 @@
+"use strict";
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+  async up(queryInterface, Sequelize) {
+    await queryInterface.addIndex("notifications", ["documentId", "userId"]);
+  },
+
+  async down(queryInterface, Sequelize) {
+    await queryInterface.removeIndex("notifications", ["documentId", "userId"]);
+  },
+};

server/models/Notification.test.ts

@@ -0,0 +1,397 @@
+import { NotificationEventType } from "@shared/types";
+import {
+  buildCollection,
+  buildComment,
+  buildDocument,
+  buildNotification,
+  buildTeam,
+  buildUser,
+} from "@server/test/factories";
+import {
+  getEmailMessageId,
+  MaxMessagesInEmailThread,
+} from "@server/utils/emails";
+import Notification from "./Notification";
+
+describe("Notification", () => {
+  describe("emailReferences", () => {
+    it("should return no references for an unsupported notification", async () => {
+      const team = await buildTeam();
+      const user = await buildUser({ teamId: team.id });
+      const collection = await buildCollection({
+        userId: user.id,
+        teamId: team.id,
+      });
+      const document = await buildDocument({
+        collectionId: collection.id,
+        userId: user.id,
+        teamId: team.id,
+      });
+
+      const notification = await buildNotification({
+        event: NotificationEventType.AddUserToDocument,
+        documentId: document.id,
+        userId: user.id,
+        teamId: team.id,
+      });
+      const references = await Notification.emailReferences(notification);
+
+      expect(references).toBeUndefined();
+    });
+
+    it("should return no references for a new notification", async () => {
+      const team = await buildTeam();
+      const user = await buildUser({ teamId: team.id });
+      const collection = await buildCollection({
+        userId: user.id,
+        teamId: team.id,
+      });
+      const document = await buildDocument({
+        collectionId: collection.id,
+        userId: user.id,
+        teamId: team.id,
+      });
+
+      const notification = await buildNotification({
+        event: NotificationEventType.UpdateDocument,
+        documentId: document.id,
+        userId: user.id,
+        teamId: team.id,
+      });
+      const references = await Notification.emailReferences(notification);
+
+      expect(references).toBeUndefined();
+    });
+
+    describe("should return references from last thread for current notification", () => {
+      it("only one thread available", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const pastNotifications = await Notification.bulkCreate(
+          [...Array(2)].map(() => ({
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          }))
+        );
+
+        const notification = await buildNotification({
+          event: NotificationEventType.UpdateDocument,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(2);
+
+        const expectedReferences = pastNotifications.map((notif) =>
+          getEmailMessageId(notif.id)
+        );
+
+        expect(references).toEqual(expectedReferences);
+      });
+
+      it("multiple threads available", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const pastNotifications = await Notification.bulkCreate(
+          [...Array(105)].map(() => ({
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          }))
+        );
+
+        const notification = await buildNotification({
+          event: NotificationEventType.UpdateDocument,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(5);
+
+        const expectedReferences = pastNotifications
+          .slice(MaxMessagesInEmailThread)
+          .map((notif) => getEmailMessageId(notif.id));
+
+        expect(references).toEqual(expectedReferences);
+      });
+    });
+
+    describe("should return references from consolidated events", () => {
+      it("document edits", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const pastNotifications = await Notification.bulkCreate([
+          {
+            event: NotificationEventType.PublishDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          ...[...Array(2)].map(() => ({
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          })),
+          ...[...Array(2)].map(() => ({
+            event: NotificationEventType.CreateComment,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          })),
+        ]);
+
+        const notification = await buildNotification({
+          event: NotificationEventType.UpdateDocument,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(3);
+
+        const expectedReferences = pastNotifications
+          .filter(
+            (notif) =>
+              notif.event === NotificationEventType.PublishDocument ||
+              notif.event === NotificationEventType.UpdateDocument
+          )
+          .map((notif) => getEmailMessageId(notif.id));
+
+        expect(references).toEqual(expectedReferences);
+      });
+
+      it("comment creation", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const pastNotifications = await Notification.bulkCreate([
+          {
+            event: NotificationEventType.PublishDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          ...[...Array(2)].map(() => ({
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          })),
+          ...[...Array(2)].map(() => ({
+            event: NotificationEventType.CreateComment,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          })),
+        ]);
+
+        const notification = await buildNotification({
+          event: NotificationEventType.CreateComment,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(2);
+
+        const expectedReferences = pastNotifications
+          .filter(
+            (notif) => notif.event === NotificationEventType.CreateComment
+          )
+          .map((notif) => getEmailMessageId(notif.id));
+
+        expect(references).toEqual(expectedReferences);
+      });
+
+      it("document mentions", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const comment = await buildComment({
+          documentId: document.id,
+          userId: user.id,
+        });
+        const pastNotifications = await Notification.bulkCreate([
+          {
+            event: NotificationEventType.PublishDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.CreateComment,
+            commentId: comment.id,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.MentionedInComment,
+            commentId: comment.id,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.MentionedInDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+        ]);
+
+        const notification = await buildNotification({
+          event: NotificationEventType.MentionedInDocument,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(2);
+
+        const expectedReferences = pastNotifications
+          .filter(
+            (notif) =>
+              notif.event === NotificationEventType.MentionedInDocument ||
+              notif.event === NotificationEventType.MentionedInComment
+          )
+          .map((notif) => getEmailMessageId(notif.id));
+
+        expect(references).toEqual(expectedReferences);
+      });
+
+      it("comment mentions", async () => {
+        const team = await buildTeam();
+        const user = await buildUser({ teamId: team.id });
+        const collection = await buildCollection({
+          userId: user.id,
+          teamId: team.id,
+        });
+        const document = await buildDocument({
+          collectionId: collection.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const comment = await buildComment({
+          documentId: document.id,
+          userId: user.id,
+        });
+        const pastNotifications = await Notification.bulkCreate([
+          {
+            event: NotificationEventType.PublishDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.UpdateDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.CreateComment,
+            commentId: comment.id,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.MentionedInComment,
+            commentId: comment.id,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+          {
+            event: NotificationEventType.MentionedInDocument,
+            documentId: document.id,
+            userId: user.id,
+            teamId: team.id,
+          },
+        ]);
+
+        const notification = await buildNotification({
+          event: NotificationEventType.MentionedInComment,
+          documentId: document.id,
+          userId: user.id,
+          teamId: team.id,
+        });
+        const references = await Notification.emailReferences(notification);
+
+        expect(references?.length).toEqual(2);
+
+        const expectedReferences = pastNotifications
+          .filter(
+            (notif) =>
+              notif.event === NotificationEventType.MentionedInComment ||
+              notif.event === NotificationEventType.MentionedInDocument
+          )
+          .map((notif) => getEmailMessageId(notif.id));
+
+        expect(references).toEqual(expectedReferences);
+      });
+    });
+  });
+});

server/models/Notification.ts

@@ -1,9 +1,11 @@
 import crypto from "crypto";
+import chunk from "lodash/chunk";
 import type {
   InferAttributes,
   InferCreationAttributes,
   SaveOptions,
 } from "sequelize";
+import { Op } from "sequelize";
 import {
   Table,
   ForeignKey,
@@ -22,6 +24,12 @@ import {
 import { NotificationEventType } from "@shared/types";
 import env from "@server/env";
 import Model from "@server/models/base/Model";
+import {
+  getEmailMessageId,
+  getEmailThreadEventGroup,
+  isEmailThreadSupportedNotification,
+  MaxMessagesInEmailThread,
+} from "@server/utils/emails";
 import Collection from "./Collection";
 import Comment from "./Comment";
 import Document from "./Document";
@@ -218,6 +226,65 @@ class Notification extends Model<
   public get pixelUrl() {
     return `${env.URL}/api/notifications.pixel?token=${this.pixelToken}&id=${this.id}`;
   }
+
+  /**
+   * Returns the past message ids which are used to setup the thread chain in email clients.
+   *
+   * @param notification Notification for which the past notifications are fetched - used for determining the properties that form a thread.
+   * @returns An array of email message ids that form a thread.
+   */
+  public static async emailReferences(
+    notification: Notification
+  ): Promise<string[] | undefined> {
+    if (!isEmailThreadSupportedNotification(notification.event)) {
+      return;
+    }
+
+    const events = getEmailThreadEventGroup(notification.event);
+
+    if (!events) {
+      return;
+    }
+
+    const prevNotifications: Notification[] = [];
+
+    await this.findAllInBatches<Notification>(
+      {
+        attributes: ["id"],
+        where: {
+          id: {
+            [Op.ne]: notification.id,
+          },
+          event: {
+            [Op.in]: events,
+          },
+          documentId: notification.documentId,
+          userId: notification.userId,
+        },
+        order: [["createdAt", "ASC"]],
+        offset: 0,
+        batchLimit: 100,
+      },
+      async (notifications) => void prevNotifications.push(...notifications)
+    );
+
+    const emailThreads = chunk(prevNotifications, MaxMessagesInEmailThread);
+    const lastThread = emailThreads.at(-1);
+
+    // Don't return anything if there are no past notifications (or) the limit is reached.
+    // This will start a new thread in the email clients.
+    // Also ensures we don't face header limit errors.
+    if (
+      !lastThread ||
+      lastThread.length === 0 ||
+      lastThread.length === MaxMessagesInEmailThread
+    ) {
+      return;
+    }
+
+    // Return references from the last thread.
+    return lastThread.map((notif) => getEmailMessageId(notif.id));
+  }
 }
 
 export default Notification;

server/utils/emails.ts

@@ -0,0 +1,44 @@
+import { NotificationEventType } from "@shared/types";
+import { getBaseDomain } from "@shared/utils/domains";
+
+const Domain = getBaseDomain();
+
+const EmailThreadSupportedNotifications = [
+  NotificationEventType.PublishDocument,
+  NotificationEventType.UpdateDocument,
+  NotificationEventType.MentionedInDocument,
+  NotificationEventType.CreateComment,
+  NotificationEventType.MentionedInComment,
+];
+
+// Gmail creates a new thread for every 100 messages.
+export const MaxMessagesInEmailThread = 100;
+
+export const isEmailThreadSupportedNotification = (
+  event: NotificationEventType
+) => EmailThreadSupportedNotifications.includes(event);
+
+export const getEmailThreadEventGroup = (
+  event: NotificationEventType
+): NotificationEventType[] | undefined => {
+  switch (event) {
+    case NotificationEventType.PublishDocument:
+    case NotificationEventType.UpdateDocument:
+      return [
+        NotificationEventType.PublishDocument,
+        NotificationEventType.UpdateDocument,
+      ];
+    case NotificationEventType.MentionedInDocument:
+    case NotificationEventType.MentionedInComment:
+      return [
+        NotificationEventType.MentionedInDocument,
+        NotificationEventType.MentionedInComment,
+      ];
+    case NotificationEventType.CreateComment:
+      return [NotificationEventType.CreateComment];
+    default:
+      return;
+  }
+};
+
+export const getEmailMessageId = (text: string) => `<${text}@${Domain}>`;