test

closes #4045

.env.sample

@@ -174,6 +174,6 @@ DEFAULT_LANGUAGE=en_US
 # Optionally enable rate limiter at application web server
 RATE_LIMITER_ENABLED=true
 
-# Configure default throttling paramaters for rate limiter
-RATE_LIMITER_REQUESTS=5000
+# Configure default throttling parameters for rate limiter
+RATE_LIMITER_REQUESTS=1000
 RATE_LIMITER_DURATION_WINDOW=60

.jestconfig.json

@@ -12,6 +12,9 @@
       },
       "setupFiles": [
         "<rootDir>/__mocks__/console.js",
+        "<rootDir>/server/test/env.ts"
+      ],
+      "setupFilesAfterEnv": [
         "<rootDir>/server/test/setup.ts"
       ],
       "testEnvironment": "node",

app/actions/definitions/documents.tsx

@@ -11,6 +11,8 @@ import {
   ImportIcon,
   PinIcon,
   SearchIcon,
+  UnsubscribeIcon,
+  SubscribeIcon,
   MoveIcon,
   TrashIcon,
   CrossIcon,
@@ -115,12 +117,74 @@ export const unstarDocument = createAction({
   },
 });
 
-export const downloadDocument = createAction({
-  name: ({ t, isContextMenu }) =>
-    isContextMenu ? t("Download") : t("Download document"),
+export const subscribeDocument = createAction({
+  name: ({ t }) => t("Subscribe"),
   section: DocumentSection,
+  icon: <SubscribeIcon />,
+  visible: ({ activeDocumentId, stores }) => {
+    if (!activeDocumentId) {
+      return false;
+    }
+
+    const document = stores.documents.get(activeDocumentId);
+
+    return (
+      !document?.isSubscribed &&
+      stores.policies.abilities(activeDocumentId).subscribe
+    );
+  },
+  perform: ({ activeDocumentId, stores, t }) => {
+    if (!activeDocumentId) {
+      return;
+    }
+
+    const document = stores.documents.get(activeDocumentId);
+
+    document?.subscribe();
+
+    stores.toasts.showToast(t("Subscribed to document notifications"), {
+      type: "success",
+    });
+  },
+});
+
+export const unsubscribeDocument = createAction({
+  name: ({ t }) => t("Unsubscribe"),
+  section: DocumentSection,
+  icon: <UnsubscribeIcon />,
+  visible: ({ activeDocumentId, stores }) => {
+    if (!activeDocumentId) {
+      return false;
+    }
+
+    const document = stores.documents.get(activeDocumentId);
+
+    return (
+      !!document?.isSubscribed &&
+      stores.policies.abilities(activeDocumentId).unsubscribe
+    );
+  },
+  perform: ({ activeDocumentId, stores, currentUserId, t }) => {
+    if (!activeDocumentId || !currentUserId) {
+      return;
+    }
+
+    const document = stores.documents.get(activeDocumentId);
+
+    document?.unsubscribe(currentUserId);
+
+    stores.toasts.showToast(t("Unsubscribed from document notifications"), {
+      type: "success",
+    });
+  },
+});
+
+export const downloadDocumentAsHTML = createAction({
+  name: ({ t }) => t("HTML"),
+  section: DocumentSection,
+  keywords: "html export",
   icon: <DownloadIcon />,
-  keywords: "export",
+  iconInContextMenu: false,
   visible: ({ activeDocumentId, stores }) =>
     !!activeDocumentId && stores.policies.abilities(activeDocumentId).download,
   perform: ({ activeDocumentId, stores }) => {
@@ -129,10 +193,37 @@ export const downloadDocument = createAction({
     }
 
     const document = stores.documents.get(activeDocumentId);
-    document?.download();
+    document?.download("text/html");
   },
 });
 
+export const downloadDocumentAsMarkdown = createAction({
+  name: ({ t }) => t("Markdown"),
+  section: DocumentSection,
+  keywords: "md markdown export",
+  icon: <DownloadIcon />,
+  iconInContextMenu: false,
+  visible: ({ activeDocumentId, stores }) =>
+    !!activeDocumentId && stores.policies.abilities(activeDocumentId).download,
+  perform: ({ activeDocumentId, stores }) => {
+    if (!activeDocumentId) {
+      return;
+    }
+
+    const document = stores.documents.get(activeDocumentId);
+    document?.download("text/markdown");
+  },
+});
+
+export const downloadDocument = createAction({
+  name: ({ t, isContextMenu }) =>
+    isContextMenu ? t("Download") : t("Download document"),
+  section: DocumentSection,
+  icon: <DownloadIcon />,
+  keywords: "export",
+  children: [downloadDocumentAsHTML, downloadDocumentAsMarkdown],
+});
+
 export const duplicateDocument = createAction({
   name: ({ t, isContextMenu }) =>
     isContextMenu ? t("Duplicate") : t("Duplicate document"),
@@ -471,6 +562,8 @@ export const rootDocumentActions = [
   downloadDocument,
   starDocument,
   unstarDocument,
+  subscribeDocument,
+  unsubscribeDocument,
   duplicateDocument,
   moveDocument,
   permanentlyDeleteDocument,

app/components/CollectionDescription.tsx

@@ -25,7 +25,7 @@ function CollectionDescription({ collection }: Props) {
   const [isExpanded, setExpanded] = React.useState(false);
   const [isEditing, setEditing] = React.useState(false);
   const [isDirty, setDirty] = React.useState(false);
-  const can = usePolicy(collection.id);
+  const can = usePolicy(collection);
 
   const handleStartEditing = React.useCallback(() => {
     setEditing(true);

app/components/DocumentListItem.tsx

@@ -49,8 +49,8 @@ function DocumentListItem(
   ref: React.RefObject<HTMLAnchorElement>
 ) {
   const { t } = useTranslation();
-  const currentUser = useCurrentUser();
-  const currentTeam = useCurrentTeam();
+  const user = useCurrentUser();
+  const team = useCurrentTeam();
   const [menuOpen, handleMenuOpen, handleMenuClose] = useBoolean();
 
   const {
@@ -70,7 +70,7 @@ function DocumentListItem(
     !!document.title.toLowerCase().includes(highlight.toLowerCase());
   const canStar =
     !document.isDraft && !document.isArchived && !document.isTemplate;
-  const can = usePolicy(currentTeam.id);
+  const can = usePolicy(team);
   const canCollection = usePolicy(document.collectionId);
 
   return (
@@ -96,7 +96,7 @@ function DocumentListItem(
             highlight={highlight}
             dir={document.dir}
           />
-          {document.isBadgedNew && document.createdBy.id !== currentUser.id && (
+          {document.isBadgedNew && document.createdBy.id !== user.id && (
             <Badge yellow>{t("New")}</Badge>
           )}
           {canStar && (

app/components/Editor.tsx

@@ -1,12 +1,12 @@
 import { formatDistanceToNow } from "date-fns";
 import { deburr, sortBy } from "lodash";
+import { observer } from "mobx-react";
 import { DOMParser as ProsemirrorDOMParser } from "prosemirror-model";
 import { TextSelection } from "prosemirror-state";
 import * as React from "react";
 import { mergeRefs } from "react-merge-refs";
 import { Optional } from "utility-types";
 import insertFiles from "@shared/editor/commands/insertFiles";
-import embeds from "@shared/editor/embeds";
 import { Heading } from "@shared/editor/lib/getHeadings";
 import { getDataTransferFiles } from "@shared/utils/files";
 import parseDocumentSlug from "@shared/utils/parseDocumentSlug";
@@ -18,6 +18,7 @@ import ErrorBoundary from "~/components/ErrorBoundary";
 import HoverPreview from "~/components/HoverPreview";
 import type { Props as EditorProps, Editor as SharedEditor } from "~/editor";
 import useDictionary from "~/hooks/useDictionary";
+import useEmbeds from "~/hooks/useEmbeds";
 import useStores from "~/hooks/useStores";
 import useToasts from "~/hooks/useToasts";
 import { NotFoundError } from "~/utils/errors";
@@ -58,6 +59,7 @@ function Editor(props: Props, ref: React.RefObject<SharedEditor> | null) {
   const { documents } = useStores();
   const { showToast } = useToasts();
   const dictionary = useDictionary();
+  const embeds = useEmbeds(!shareId);
   const [
     activeLinkEvent,
     setActiveLinkEvent,
@@ -310,4 +312,4 @@ function Editor(props: Props, ref: React.RefObject<SharedEditor> | null) {
   );
 }
 
-export default React.forwardRef(Editor);
+export default observer(React.forwardRef(Editor));

app/components/EventListItem.tsx

@@ -33,7 +33,7 @@ type Props = {
 const EventListItem = ({ event, latest, document, ...rest }: Props) => {
   const { t } = useTranslation();
   const location = useLocation();
-  const can = usePolicy(document.id);
+  const can = usePolicy(document);
   const opts = {
     userName: event.actor.name,
   };

app/components/InputSelectPermission.tsx

@@ -1,6 +1,7 @@
 import * as React from "react";
 import { useTranslation } from "react-i18next";
 import { $Diff } from "utility-types";
+import { CollectionPermission } from "@shared/types";
 import InputSelect, { Props, Option } from "./InputSelect";
 
 export default function InputSelectPermission(
@@ -31,11 +32,11 @@ export default function InputSelectPermission(
       options={[
         {
           label: t("View and edit"),
-          value: "read_write",
+          value: CollectionPermission.ReadWrite,
         },
         {
           label: t("View only"),
-          value: "read",
+          value: CollectionPermission.Read,
         },
         {
           label: t("No access"),

app/components/Sidebar/App.tsx

@@ -36,7 +36,7 @@ function AppSidebar() {
   const { documents } = useStores();
   const team = useCurrentTeam();
   const user = useCurrentUser();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   React.useEffect(() => {
     if (!user.isViewer) {

app/components/Sidebar/components/CollectionLink.tsx

@@ -44,7 +44,7 @@ const CollectionLink: React.FC<Props> = ({
   const { dialogs, documents, collections } = useStores();
   const [menuOpen, handleMenuOpen, handleMenuClose] = useBoolean();
   const [isEditing, setIsEditing] = React.useState(false);
-  const canUpdate = usePolicy(collection.id).update;
+  const canUpdate = usePolicy(collection).update;
   const { t } = useTranslation();
   const history = useHistory();
   const inStarredSection = useStarredContext();

app/components/Sidebar/components/CollectionLinkChildren.tsx

@@ -25,7 +25,7 @@ function CollectionLinkChildren({
   expanded,
   prefetchDocument,
 }: Props) {
-  const can = usePolicy(collection.id);
+  const can = usePolicy(collection);
   const { showToast } = useToasts();
   const manualSort = collection.sort.field === "index";
   const { documents } = useStores();

app/components/Sidebar/components/DraggableCollectionLink.tsx

@@ -39,7 +39,7 @@ function DraggableCollectionLink({
   const [expanded, setExpanded] = React.useState(
     collection.id === ui.activeCollectionId && !locationStateStarred
   );
-  const can = usePolicy(collection.id);
+  const can = usePolicy(collection);
   const belowCollectionIndex = belowCollection ? belowCollection.index : null;
 
   // Drop to reorder collection

app/components/SocketProvider.tsx

@@ -1,391 +0,0 @@
-import invariant from "invariant";
-import { find } from "lodash";
-import { observable } from "mobx";
-import { observer } from "mobx-react";
-import * as React from "react";
-import { io, Socket } from "socket.io-client";
-import RootStore from "~/stores/RootStore";
-import withStores from "~/components/withStores";
-import { AuthorizationError, NotFoundError } from "~/utils/errors";
-import { getVisibilityListener, getPageVisible } from "~/utils/pageVisibility";
-
-type SocketWithAuthentication = Socket & {
-  authenticated?: boolean;
-};
-
-export const SocketContext = React.createContext<SocketWithAuthentication | null>(
-  null
-);
-
-type Props = RootStore;
-
-@observer
-class SocketProvider extends React.Component<Props> {
-  @observable
-  socket: SocketWithAuthentication | null;
-
-  componentDidMount() {
-    this.createConnection();
-    document.addEventListener(getVisibilityListener(), this.checkConnection);
-  }
-
-  componentWillUnmount() {
-    if (this.socket) {
-      this.socket.authenticated = false;
-      this.socket.disconnect();
-    }
-
-    document.removeEventListener(getVisibilityListener(), this.checkConnection);
-  }
-
-  checkConnection = () => {
-    if (this.socket?.disconnected && getPageVisible()) {
-      // null-ifying this reference is important, do not remove. Without it
-      // references to old sockets are potentially held in context
-      this.socket.close();
-      this.socket = null;
-      this.createConnection();
-    }
-  };
-
-  createConnection = () => {
-    this.socket = io(window.location.origin, {
-      path: "/realtime",
-      transports: ["websocket"],
-      reconnectionDelay: 1000,
-      reconnectionDelayMax: 30000,
-    });
-    invariant(this.socket, "Socket should be defined");
-
-    this.socket.authenticated = false;
-    const {
-      auth,
-      toasts,
-      documents,
-      collections,
-      groups,
-      pins,
-      stars,
-      memberships,
-      policies,
-      presence,
-      views,
-      fileOperations,
-    } = this.props;
-    if (!auth.token) {
-      return;
-    }
-
-    this.socket.on("connect", () => {
-      // immediately send current users token to the websocket backend where it
-      // is verified, if all goes well an 'authenticated' message will be
-      // received in response
-      this.socket?.emit("authentication", {
-        token: auth.token,
-      });
-    });
-
-    this.socket.on("disconnect", () => {
-      // when the socket is disconnected we need to clear all presence state as
-      // it's no longer reliable.
-      presence.clear();
-    });
-
-    // on reconnection, reset the transports option, as the Websocket
-    // connection may have failed (caused by proxy, firewall, browser, ...)
-    this.socket.io.on("reconnect_attempt", () => {
-      if (this.socket) {
-        this.socket.io.opts.transports = auth?.team?.domain
-          ? ["websocket"]
-          : ["websocket", "polling"];
-      }
-    });
-
-    this.socket.on("authenticated", () => {
-      if (this.socket) {
-        this.socket.authenticated = true;
-      }
-    });
-
-    this.socket.on("unauthorized", (err: Error) => {
-      if (this.socket) {
-        this.socket.authenticated = false;
-      }
-      toasts.showToast(err.message, {
-        type: "error",
-      });
-      throw err;
-    });
-
-    this.socket.on("entities", async (event: any) => {
-      if (event.documentIds) {
-        for (const documentDescriptor of event.documentIds) {
-          const documentId = documentDescriptor.id;
-          let document = documents.get(documentId) || {};
-
-          if (event.event === "documents.delete") {
-            const document = documents.get(documentId);
-
-            if (document) {
-              document.deletedAt = documentDescriptor.updatedAt;
-            }
-
-            policies.remove(documentId);
-            continue;
-          }
-
-          // if we already have the latest version (it was us that performed
-          // the change) then we don't need to update anything either.
-          // @ts-expect-error ts-migrate(2339) FIXME: Property 'title' does not exist on type '{}'.
-          const { title, updatedAt } = document;
-
-          if (updatedAt === documentDescriptor.updatedAt) {
-            continue;
-          }
-
-          // otherwise, grab the latest version of the document
-          try {
-            document = await documents.fetch(documentId, {
-              force: true,
-            });
-          } catch (err) {
-            if (
-              err instanceof AuthorizationError ||
-              err instanceof NotFoundError
-            ) {
-              documents.remove(documentId);
-              return;
-            }
-          }
-
-          // if the title changed then we need to update the collection also
-          // @ts-expect-error ts-migrate(2339) FIXME: Property 'title' does not exist on type '{}'.
-          if (title !== document.title) {
-            if (!event.collectionIds) {
-              event.collectionIds = [];
-            }
-
-            const existing = find(event.collectionIds, {
-              // @ts-expect-error ts-migrate(2339) FIXME: Property 'collectionId' does not exist on type '{}... Remove this comment to see the full error message
-              id: document.collectionId,
-            });
-
-            if (!existing) {
-              event.collectionIds.push({
-                // @ts-expect-error ts-migrate(2339) FIXME: Property 'collectionId' does not exist on type '{}... Remove this comment to see the full error message
-                id: document.collectionId,
-              });
-            }
-          }
-        }
-      }
-
-      if (event.collectionIds) {
-        for (const collectionDescriptor of event.collectionIds) {
-          const collectionId = collectionDescriptor.id;
-          const collection = collections.get(collectionId);
-
-          if (event.event === "collections.delete") {
-            if (collection) {
-              collection.deletedAt = collectionDescriptor.updatedAt;
-            }
-
-            const deletedDocuments = documents.inCollection(collectionId);
-            deletedDocuments.forEach((doc) => {
-              doc.deletedAt = collectionDescriptor.updatedAt;
-              policies.remove(doc.id);
-            });
-            documents.removeCollectionDocuments(collectionId);
-            memberships.removeCollectionMemberships(collectionId);
-            collections.remove(collectionId);
-            policies.remove(collectionId);
-            continue;
-          }
-
-          // if we already have the latest version (it was us that performed
-          // the change) then we don't need to update anything either.
-
-          if (collection?.updatedAt === collectionDescriptor.updatedAt) {
-            continue;
-          }
-
-          try {
-            await collections.fetch(collectionId, {
-              force: true,
-            });
-          } catch (err) {
-            if (
-              err instanceof AuthorizationError ||
-              err instanceof NotFoundError
-            ) {
-              documents.removeCollectionDocuments(collectionId);
-              memberships.removeCollectionMemberships(collectionId);
-              collections.remove(collectionId);
-              policies.remove(collectionId);
-              return;
-            }
-          }
-        }
-      }
-
-      if (event.groupIds) {
-        for (const groupDescriptor of event.groupIds) {
-          const groupId = groupDescriptor.id;
-          const group = groups.get(groupId) || {};
-          // if we already have the latest version (it was us that performed
-          // the change) then we don't need to update anything either.
-          // @ts-expect-error ts-migrate(2339) FIXME: Property 'updatedAt' does not exist on type '{}'.
-          const { updatedAt } = group;
-
-          if (updatedAt === groupDescriptor.updatedAt) {
-            continue;
-          }
-
-          try {
-            await groups.fetch(groupId, {
-              force: true,
-            });
-          } catch (err) {
-            if (
-              err instanceof AuthorizationError ||
-              err instanceof NotFoundError
-            ) {
-              groups.remove(groupId);
-            }
-          }
-        }
-      }
-
-      if (event.teamIds) {
-        await auth.fetch();
-      }
-    });
-
-    this.socket.on("pins.create", (event: any) => {
-      pins.add(event);
-    });
-
-    this.socket.on("pins.update", (event: any) => {
-      pins.add(event);
-    });
-
-    this.socket.on("pins.delete", (event: any) => {
-      pins.remove(event.modelId);
-    });
-
-    this.socket.on("stars.create", (event: any) => {
-      stars.add(event);
-    });
-
-    this.socket.on("stars.update", (event: any) => {
-      stars.add(event);
-    });
-
-    this.socket.on("stars.delete", (event: any) => {
-      stars.remove(event.modelId);
-    });
-
-    this.socket.on("documents.permanent_delete", (event: any) => {
-      documents.remove(event.documentId);
-    });
-
-    // received when a user is given access to a collection
-    // if the user is us then we go ahead and load the collection from API.
-    this.socket.on("collections.add_user", (event: any) => {
-      if (auth.user && event.userId === auth.user.id) {
-        collections.fetch(event.collectionId, {
-          force: true,
-        });
-      }
-
-      // Document policies might need updating as the permission changes
-      documents.inCollection(event.collectionId).forEach((document) => {
-        policies.remove(document.id);
-      });
-    });
-
-    // received when a user is removed from having access to a collection
-    // to keep state in sync we must update our UI if the user is us,
-    // or otherwise just remove any membership state we have for that user.
-    this.socket.on("collections.remove_user", (event: any) => {
-      if (auth.user && event.userId === auth.user.id) {
-        collections.remove(event.collectionId);
-        memberships.removeCollectionMemberships(event.collectionId);
-        documents.removeCollectionDocuments(event.collectionId);
-      } else {
-        memberships.remove(`${event.userId}-${event.collectionId}`);
-      }
-    });
-
-    this.socket.on("collections.update_index", (event: any) => {
-      const collection = collections.get(event.collectionId);
-
-      if (collection) {
-        collection.updateIndex(event.index);
-      }
-    });
-
-    this.socket.on("fileOperations.create", async (event: any) => {
-      const user = auth.user;
-      if (user) {
-        fileOperations.add({ ...event, user });
-      }
-    });
-
-    this.socket.on("fileOperations.update", async (event: any) => {
-      const user = auth.user;
-      if (user) {
-        fileOperations.add({ ...event, user });
-      }
-    });
-
-    // received a message from the API server that we should request
-    // to join a specific room. Forward that to the ws server.
-    this.socket.on("join", (event: any) => {
-      this.socket?.emit("join", event);
-    });
-
-    // received a message from the API server that we should request
-    // to leave a specific room. Forward that to the ws server.
-    this.socket.on("leave", (event: any) => {
-      this.socket?.emit("leave", event);
-    });
-
-    // received whenever we join a document room, the payload includes
-    // userIds that are present/viewing and those that are editing.
-    this.socket.on("document.presence", (event: any) => {
-      presence.init(event.documentId, event.userIds, event.editingIds);
-    });
-
-    // received whenever a new user joins a document room, aka they
-    // navigate to / start viewing a document
-    this.socket.on("user.join", (event: any) => {
-      presence.touch(event.documentId, event.userId, event.isEditing);
-      views.touch(event.documentId, event.userId);
-    });
-
-    // received whenever a new user leaves a document room, aka they
-    // navigate away / stop viewing a document
-    this.socket.on("user.leave", (event: any) => {
-      presence.leave(event.documentId, event.userId);
-      views.touch(event.documentId, event.userId);
-    });
-
-    // received when another client in a document room wants to change
-    // or update it's presence. Currently the only property is whether
-    // the client is in editing state or not.
-    this.socket.on("user.presence", (event: any) => {
-      presence.touch(event.documentId, event.userId, event.isEditing);
-    });
-  };
-
-  render() {
-    return (
-      <SocketContext.Provider value={this.socket}>
-        {this.props.children}
-      </SocketContext.Provider>
-    );
-  }
-}
-
-export default withStores(SocketProvider);

app/components/Theme.tsx

@@ -2,10 +2,10 @@ import { observer } from "mobx-react";
 import * as React from "react";
 import { ThemeProvider } from "styled-components";
 import { breakpoints } from "@shared/styles";
+import GlobalStyles from "@shared/styles/globals";
 import { dark, light, lightMobile, darkMobile } from "@shared/styles/theme";
 import useMediaQuery from "~/hooks/useMediaQuery";
 import useStores from "~/hooks/useStores";
-import GlobalStyles from "~/styles/globals";
 
 const Theme: React.FC = ({ children }) => {
   const { ui } = useStores();

app/components/WebsocketProvider.tsx

@@ -0,0 +1,448 @@
+import invariant from "invariant";
+import { find } from "lodash";
+import { action, observable } from "mobx";
+import { observer } from "mobx-react";
+import * as React from "react";
+import { io, Socket } from "socket.io-client";
+import RootStore from "~/stores/RootStore";
+import Collection from "~/models/Collection";
+import Document from "~/models/Document";
+import FileOperation from "~/models/FileOperation";
+import Group from "~/models/Group";
+import Pin from "~/models/Pin";
+import Star from "~/models/Star";
+import Subscription from "~/models/Subscription";
+import Team from "~/models/Team";
+import withStores from "~/components/withStores";
+import {
+  PartialWithId,
+  WebsocketCollectionUpdateIndexEvent,
+  WebsocketCollectionUserEvent,
+  WebsocketEntitiesEvent,
+  WebsocketEntityDeletedEvent,
+} from "~/types";
+import { AuthorizationError, NotFoundError } from "~/utils/errors";
+import { getVisibilityListener, getPageVisible } from "~/utils/pageVisibility";
+
+type SocketWithAuthentication = Socket & {
+  authenticated?: boolean;
+};
+
+export const WebsocketContext = React.createContext<SocketWithAuthentication | null>(
+  null
+);
+
+type Props = RootStore;
+
+@observer
+class WebsocketProvider extends React.Component<Props> {
+  @observable
+  socket: SocketWithAuthentication | null;
+
+  componentDidMount() {
+    this.createConnection();
+    document.addEventListener(getVisibilityListener(), this.checkConnection);
+  }
+
+  componentWillUnmount() {
+    if (this.socket) {
+      this.socket.authenticated = false;
+      this.socket.disconnect();
+    }
+
+    document.removeEventListener(getVisibilityListener(), this.checkConnection);
+  }
+
+  checkConnection = () => {
+    if (this.socket?.disconnected && getPageVisible()) {
+      // null-ifying this reference is important, do not remove. Without it
+      // references to old sockets are potentially held in context
+      this.socket.close();
+      this.socket = null;
+      this.createConnection();
+    }
+  };
+
+  createConnection = () => {
+    this.socket = io(window.location.origin, {
+      path: "/realtime",
+      transports: ["websocket"],
+      reconnectionDelay: 1000,
+      reconnectionDelayMax: 30000,
+    });
+    invariant(this.socket, "Socket should be defined");
+
+    this.socket.authenticated = false;
+    const {
+      auth,
+      toasts,
+      documents,
+      collections,
+      groups,
+      pins,
+      stars,
+      memberships,
+      policies,
+      presence,
+      views,
+      subscriptions,
+      fileOperations,
+    } = this.props;
+    if (!auth.token) {
+      return;
+    }
+
+    this.socket.on("connect", () => {
+      // immediately send current users token to the websocket backend where it
+      // is verified, if all goes well an 'authenticated' message will be
+      // received in response
+      this.socket?.emit("authentication", {
+        token: auth.token,
+      });
+    });
+
+    this.socket.on("disconnect", () => {
+      // when the socket is disconnected we need to clear all presence state as
+      // it's no longer reliable.
+      presence.clear();
+    });
+
+    // on reconnection, reset the transports option, as the Websocket
+    // connection may have failed (caused by proxy, firewall, browser, ...)
+    this.socket.io.on("reconnect_attempt", () => {
+      if (this.socket) {
+        this.socket.io.opts.transports = auth?.team?.domain
+          ? ["websocket"]
+          : ["websocket", "polling"];
+      }
+    });
+
+    this.socket.on("authenticated", () => {
+      if (this.socket) {
+        this.socket.authenticated = true;
+      }
+    });
+
+    this.socket.on("unauthorized", (err: Error) => {
+      if (this.socket) {
+        this.socket.authenticated = false;
+      }
+      toasts.showToast(err.message, {
+        type: "error",
+      });
+      throw err;
+    });
+
+    this.socket.on(
+      "entities",
+      action(async (event: WebsocketEntitiesEvent) => {
+        if (event.documentIds) {
+          for (const documentDescriptor of event.documentIds) {
+            const documentId = documentDescriptor.id;
+            let document = documents.get(documentId);
+            const previousTitle = document?.title;
+
+            // if we already have the latest version (it was us that performed
+            // the change) then we don't need to update anything either.
+            if (document?.updatedAt === documentDescriptor.updatedAt) {
+              continue;
+            }
+
+            // otherwise, grab the latest version of the document
+            try {
+              document = await documents.fetch(documentId, {
+                force: true,
+              });
+            } catch (err) {
+              if (
+                err instanceof AuthorizationError ||
+                err instanceof NotFoundError
+              ) {
+                documents.remove(documentId);
+                return;
+              }
+            }
+
+            // if the title changed then we need to update the collection also
+            if (document && previousTitle !== document.title) {
+              if (!event.collectionIds) {
+                event.collectionIds = [];
+              }
+
+              const existing = find(event.collectionIds, {
+                id: document.collectionId,
+              });
+
+              if (!existing) {
+                event.collectionIds.push({
+                  id: document.collectionId,
+                });
+              }
+            }
+          }
+        }
+
+        if (event.collectionIds) {
+          for (const collectionDescriptor of event.collectionIds) {
+            const collectionId = collectionDescriptor.id;
+            const collection = collections.get(collectionId);
+
+            // if we already have the latest version (it was us that performed
+            // the change) then we don't need to update anything either.
+            if (collection?.updatedAt === collectionDescriptor.updatedAt) {
+              continue;
+            }
+
+            try {
+              await collections.fetch(collectionId, {
+                force: true,
+              });
+            } catch (err) {
+              if (
+                err instanceof AuthorizationError ||
+                err instanceof NotFoundError
+              ) {
+                documents.removeCollectionDocuments(collectionId);
+                memberships.removeCollectionMemberships(collectionId);
+                collections.remove(collectionId);
+                policies.remove(collectionId);
+                return;
+              }
+            }
+          }
+        }
+      })
+    );
+
+    this.socket.on(
+      "documents.update",
+      action(
+        (event: PartialWithId<Document> & { title: string; url: string }) => {
+          documents.add(event);
+
+          if (event.collectionId) {
+            const collection = collections.get(event.collectionId);
+            collection?.updateDocument(event);
+          }
+        }
+      )
+    );
+
+    this.socket.on(
+      "documents.archive",
+      action((event: PartialWithId<Document>) => {
+        documents.add(event);
+        policies.remove(event.id);
+
+        if (event.collectionId) {
+          const collection = collections.get(event.collectionId);
+          collection?.removeDocument(event.id);
+        }
+      })
+    );
+
+    this.socket.on(
+      "documents.delete",
+      action((event: PartialWithId<Document>) => {
+        documents.add(event);
+        policies.remove(event.id);
+
+        if (event.collectionId) {
+          const collection = collections.get(event.collectionId);
+          collection?.removeDocument(event.id);
+        }
+      })
+    );
+
+    this.socket.on(
+      "documents.permanent_delete",
+      (event: WebsocketEntityDeletedEvent) => {
+        documents.remove(event.modelId);
+      }
+    );
+
+    this.socket.on("groups.create", (event: PartialWithId<Group>) => {
+      groups.add(event);
+    });
+
+    this.socket.on("groups.update", (event: PartialWithId<Group>) => {
+      groups.add(event);
+    });
+
+    this.socket.on("groups.delete", (event: WebsocketEntityDeletedEvent) => {
+      groups.remove(event.modelId);
+    });
+
+    this.socket.on("collections.create", (event: PartialWithId<Collection>) => {
+      collections.add(event);
+    });
+
+    this.socket.on(
+      "collections.delete",
+      action((event: WebsocketEntityDeletedEvent) => {
+        const collectionId = event.modelId;
+        const deletedAt = new Date().toISOString();
+
+        const deletedDocuments = documents.inCollection(collectionId);
+        deletedDocuments.forEach((doc) => {
+          doc.deletedAt = deletedAt;
+          policies.remove(doc.id);
+        });
+        documents.removeCollectionDocuments(collectionId);
+        memberships.removeCollectionMemberships(collectionId);
+        collections.remove(collectionId);
+        policies.remove(collectionId);
+      })
+    );
+
+    this.socket.on("teams.update", (event: PartialWithId<Team>) => {
+      auth.updateTeam(event);
+    });
+
+    this.socket.on("pins.create", (event: PartialWithId<Pin>) => {
+      pins.add(event);
+    });
+
+    this.socket.on("pins.update", (event: PartialWithId<Pin>) => {
+      pins.add(event);
+    });
+
+    this.socket.on("pins.delete", (event: WebsocketEntityDeletedEvent) => {
+      pins.remove(event.modelId);
+    });
+
+    this.socket.on("stars.create", (event: PartialWithId<Star>) => {
+      stars.add(event);
+    });
+
+    this.socket.on("stars.update", (event: PartialWithId<Star>) => {
+      stars.add(event);
+    });
+
+    this.socket.on("stars.delete", (event: WebsocketEntityDeletedEvent) => {
+      stars.remove(event.modelId);
+    });
+
+    // received when a user is given access to a collection
+    // if the user is us then we go ahead and load the collection from API.
+    this.socket.on(
+      "collections.add_user",
+      action((event: WebsocketCollectionUserEvent) => {
+        if (auth.user && event.userId === auth.user.id) {
+          collections.fetch(event.collectionId, {
+            force: true,
+          });
+        }
+
+        // Document policies might need updating as the permission changes
+        documents.inCollection(event.collectionId).forEach((document) => {
+          policies.remove(document.id);
+        });
+      })
+    );
+
+    // received when a user is removed from having access to a collection
+    // to keep state in sync we must update our UI if the user is us,
+    // or otherwise just remove any membership state we have for that user.
+    this.socket.on(
+      "collections.remove_user",
+      action((event: WebsocketCollectionUserEvent) => {
+        if (auth.user && event.userId === auth.user.id) {
+          collections.remove(event.collectionId);
+          memberships.removeCollectionMemberships(event.collectionId);
+          documents.removeCollectionDocuments(event.collectionId);
+        } else {
+          memberships.remove(`${event.userId}-${event.collectionId}`);
+        }
+      })
+    );
+
+    this.socket.on(
+      "collections.update_index",
+      action((event: WebsocketCollectionUpdateIndexEvent) => {
+        const collection = collections.get(event.collectionId);
+
+        if (collection) {
+          collection.updateIndex(event.index);
+        }
+      })
+    );
+
+    this.socket.on(
+      "fileOperations.create",
+      (event: PartialWithId<FileOperation>) => {
+        fileOperations.add(event);
+      }
+    );
+
+    this.socket.on(
+      "fileOperations.update",
+      (event: PartialWithId<FileOperation>) => {
+        fileOperations.add(event);
+      }
+    );
+
+    this.socket.on(
+      "subscriptions.create",
+      (event: PartialWithId<Subscription>) => {
+        subscriptions.add(event);
+      }
+    );
+
+    this.socket.on(
+      "subscriptions.delete",
+      (event: WebsocketEntityDeletedEvent) => {
+        subscriptions.remove(event.modelId);
+      }
+    );
+
+    // received a message from the API server that we should request
+    // to join a specific room. Forward that to the ws server.
+    this.socket.on("join", (event: any) => {
+      this.socket?.emit("join", event);
+    });
+
+    // received a message from the API server that we should request
+    // to leave a specific room. Forward that to the ws server.
+    this.socket.on("leave", (event: any) => {
+      this.socket?.emit("leave", event);
+    });
+
+    // received whenever we join a document room, the payload includes
+    // userIds that are present/viewing and those that are editing.
+    this.socket.on("document.presence", (event: any) => {
+      presence.init(event.documentId, event.userIds, event.editingIds);
+    });
+
+    // received whenever a new user joins a document room, aka they
+    // navigate to / start viewing a document
+    this.socket.on("user.join", (event: any) => {
+      presence.touch(event.documentId, event.userId, event.isEditing);
+      views.touch(event.documentId, event.userId);
+    });
+
+    // received whenever a new user leaves a document room, aka they
+    // navigate away / stop viewing a document
+    this.socket.on("user.leave", (event: any) => {
+      presence.leave(event.documentId, event.userId);
+      views.touch(event.documentId, event.userId);
+    });
+
+    // received when another client in a document room wants to change
+    // or update it's presence. Currently the only property is whether
+    // the client is in editing state or not.
+    this.socket.on("user.presence", (event: any) => {
+      presence.touch(event.documentId, event.userId, event.isEditing);
+    });
+  };
+
+  render() {
+    return (
+      <WebsocketContext.Provider value={this.socket}>
+        {this.props.children}
+      </WebsocketContext.Provider>
+    );
+  }
+}
+
+export default withStores(WebsocketProvider);

app/editor/components/CommandMenu.tsx

@@ -7,9 +7,10 @@ import { Portal } from "react-portal";
 import { VisuallyHidden } from "reakit/VisuallyHidden";
 import styled from "styled-components";
 import insertFiles from "@shared/editor/commands/insertFiles";
+import { EmbedDescriptor } from "@shared/editor/embeds";
 import { CommandFactory } from "@shared/editor/lib/Extension";
 import filterExcessSeparators from "@shared/editor/lib/filterExcessSeparators";
-import { EmbedDescriptor, MenuItem } from "@shared/editor/types";
+import { MenuItem } from "@shared/editor/types";
 import { depths } from "@shared/styles";
 import { getEventFiles } from "@shared/utils/files";
 import { AttachmentValidation } from "@shared/validations";
@@ -427,10 +428,12 @@ class CommandMenu<T = MenuItem> extends React.Component<Props<T>, State> {
 
     for (const embed of embeds) {
       if (embed.title) {
-        embedItems.push({
-          ...embed,
-          name: "embed",
-        });
+        embedItems.push(
+          new EmbedDescriptor({
+            ...embed,
+            name: "embed",
+          })
+        );
       }
     }
 

app/editor/index.tsx

@@ -16,6 +16,8 @@ import { EditorState, Selection, Plugin, Transaction } from "prosemirror-state";
 import { Decoration, EditorView } from "prosemirror-view";
 import * as React from "react";
 import { DefaultTheme, ThemeProps } from "styled-components";
+import EditorContainer from "@shared/editor/components/Styles";
+import { EmbedDescriptor } from "@shared/editor/embeds";
 import Extension, { CommandFactory } from "@shared/editor/lib/Extension";
 import ExtensionManager from "@shared/editor/lib/ExtensionManager";
 import getHeadings from "@shared/editor/lib/getHeadings";
@@ -25,8 +27,10 @@ import Mark from "@shared/editor/marks/Mark";
 import Node from "@shared/editor/nodes/Node";
 import ReactNode from "@shared/editor/nodes/ReactNode";
 import fullExtensionsPackage from "@shared/editor/packages/full";
-import { EmbedDescriptor, EventType } from "@shared/editor/types";
+import { EventType } from "@shared/editor/types";
+import { IntegrationType } from "@shared/types";
 import EventEmitter from "@shared/utils/events";
+import Integration from "~/models/Integration";
 import Flex from "~/components/Flex";
 import { Dictionary } from "~/hooks/useDictionary";
 import Logger from "~/utils/Logger";
@@ -37,7 +41,6 @@ import EmojiMenu from "./components/EmojiMenu";
 import { SearchResult } from "./components/LinkEditor";
 import LinkToolbar from "./components/LinkToolbar";
 import SelectionToolbar from "./components/SelectionToolbar";
-import EditorContainer from "./components/Styles";
 import WithTheme from "./components/WithTheme";
 
 export { default as Extension } from "@shared/editor/lib/Extension";
@@ -110,6 +113,8 @@ export type Props = {
   onShowToast: (message: string) => void;
   className?: string;
   style?: React.CSSProperties;
+
+  embedIntegrations?: Integration<IntegrationType.Embed>[];
 };
 
 type State = {

app/hooks/useAuthorizedSettingsConfig.ts

@@ -9,12 +9,14 @@ import {
   LinkIcon,
   TeamIcon,
   BeakerIcon,
+  BuildingBlocksIcon,
   DownloadIcon,
   WebhooksIcon,
 } from "outline-icons";
 import React from "react";
 import { useTranslation } from "react-i18next";
 import Details from "~/scenes/Settings/Details";
+import Drawio from "~/scenes/Settings/Drawio";
 import Export from "~/scenes/Settings/Export";
 import Features from "~/scenes/Settings/Features";
 import Groups from "~/scenes/Settings/Groups";
@@ -67,7 +69,7 @@ type ConfigType = {
 
 const useAuthorizedSettingsConfig = () => {
   const team = useCurrentTeam();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
   const { t } = useTranslation();
 
   const config: ConfigType = React.useMemo(
@@ -170,6 +172,14 @@ const useAuthorizedSettingsConfig = () => {
         group: t("Integrations"),
         icon: WebhooksIcon,
       },
+      Drawio: {
+        name: t("Draw.io"),
+        path: "/settings/integrations/drawio",
+        component: Drawio,
+        enabled: can.update,
+        group: t("Integrations"),
+        icon: BuildingBlocksIcon,
+      },
       Slack: {
         name: "Slack",
         path: "/settings/integrations/slack",

app/hooks/useEmbeds.ts

@@ -0,0 +1,48 @@
+import { find } from "lodash";
+import * as React from "react";
+import embeds, { EmbedDescriptor } from "@shared/editor/embeds";
+import { IntegrationType } from "@shared/types";
+import Integration from "~/models/Integration";
+import Logger from "~/utils/Logger";
+import useStores from "./useStores";
+
+/**
+ * Hook to get all embed configuration for the current team
+ *
+ * @param loadIfMissing Should we load integration settings if they are not
+ * locally available
+ * @returns A list of embed descriptors
+ */
+export default function useEmbeds(loadIfMissing = false) {
+  const { integrations } = useStores();
+
+  React.useEffect(() => {
+    async function fetchEmbedIntegrations() {
+      try {
+        await integrations.fetchPage({
+          limit: 100,
+          type: IntegrationType.Embed,
+        });
+      } catch (err) {
+        Logger.error("Failed to fetch embed integrations", err);
+      }
+    }
+
+    !integrations.isLoaded && loadIfMissing && fetchEmbedIntegrations();
+  }, [integrations, loadIfMissing]);
+
+  return React.useMemo(
+    () =>
+      embeds.map((e) => {
+        const em: Integration<IntegrationType.Embed> | undefined = find(
+          integrations.orderedData,
+          (i) => i.service === e.component.name.toLowerCase()
+        );
+        return new EmbedDescriptor({
+          ...e,
+          settings: em?.settings,
+        });
+      }),
+    [integrations.orderedData]
+  );
+}

app/hooks/usePolicy.ts

@@ -1,12 +1,34 @@
+import * as React from "react";
+import BaseModel from "~/models/BaseModel";
 import useStores from "./useStores";
 
 /**
- * Quick access to retrieve the abilities of a policy for a given entity
+ * Retrieve the abilities of a policy for a given entity, if the policy is not
+ * located in the store, it will be fetched from the server.
  *
- * @param entityId The entity id
- * @returns The available abilities
+ * @param entity The model or model id
+ * @returns The policy for the model
  */
-export default function usePolicy(entityId: string) {
+export default function usePolicy(entity: string | BaseModel | undefined) {
   const { policies } = useStores();
+  const triggered = React.useRef(false);
+  const entityId = entity
+    ? typeof entity === "string"
+      ? entity
+      : entity.id
+    : "";
+
+  React.useEffect(() => {
+    if (entity && typeof entity !== "string") {
+      // The policy for this model is missing and we haven't tried to fetch it
+      // yet, go ahead and do that now. The force flag is needed otherwise the
+      // network request will be skipped due to the model existing in the store
+      if (!policies.get(entity.id) && !triggered.current) {
+        triggered.current = true;
+        void entity.store.fetch(entity.id, { force: true });
+      }
+    }
+  }, [policies, entity]);
+
   return policies.abilities(entityId);
 }

app/menus/CollectionMenu.tsx

@@ -187,8 +187,8 @@ function CollectionMenu({
   );
 
   const alphabeticalSort = collection.sort.field === "title";
-  const can = usePolicy(collection.id);
-  const canUserInTeam = usePolicy(team.id);
+  const can = usePolicy(collection);
+  const canUserInTeam = usePolicy(team);
   const items: MenuItem[] = React.useMemo(
     () => [
       {

app/menus/DocumentMenu.tsx

@@ -27,6 +27,8 @@ import { actionToMenuItem } from "~/actions";
 import {
   pinDocument,
   createTemplate,
+  subscribeDocument,
+  unsubscribeDocument,
   moveDocument,
   deleteDocument,
   permanentlyDeleteDocument,
@@ -123,7 +125,7 @@ function DocumentMenu({
   }, [menu]);
 
   const collection = collections.get(document.collectionId);
-  const can = usePolicy(document.id);
+  const can = usePolicy(document);
   const canViewHistory = can.read && !can.restore;
   const restoreItems = React.useMemo(
     () => [
@@ -250,9 +252,10 @@ function DocumentMenu({
                 ...restoreItems,
               ],
             },
-            actionToMenuItem(unstarDocument, context),
             actionToMenuItem(starDocument, context),
-            actionToMenuItem(pinDocument, context),
+            actionToMenuItem(unstarDocument, context),
+            actionToMenuItem(subscribeDocument, context),
+            actionToMenuItem(unsubscribeDocument, context),
             {
               type: "separator",
             },
@@ -284,11 +287,16 @@ function DocumentMenu({
             },
             actionToMenuItem(archiveDocument, context),
             actionToMenuItem(moveDocument, context),
+            actionToMenuItem(pinDocument, context),
+            {
+              type: "separator",
+            },
             actionToMenuItem(deleteDocument, context),
             actionToMenuItem(permanentlyDeleteDocument, context),
             {
               type: "separator",
             },
+            actionToMenuItem(downloadDocument, context),
             {
               type: "route",
               title: t("History"),
@@ -298,7 +306,6 @@ function DocumentMenu({
               visible: canViewHistory,
               icon: <HistoryIcon />,
             },
-            actionToMenuItem(downloadDocument, context),
             {
               type: "button",
               title: t("Print"),

app/menus/EmptyTrashMenu.tsx

@@ -1,97 +0,0 @@
-import { observer } from "mobx-react";
-import { TrashIcon } from "outline-icons";
-import * as React from "react";
-import { Trans, useTranslation } from "react-i18next";
-import { useHistory } from "react-router";
-import Document from "~/models/Document";
-import Button from "~/components/Button";
-import Flex from "~/components/Flex";
-import Modal from "~/components/Modal";
-import Text from "~/components/Text";
-import useStores from "~/hooks/useStores";
-import useToasts from "~/hooks/useToasts";
-
-function EmptyTrashMenu() {
-  const { t } = useTranslation();
-  const { documents } = useStores();
-  const [showModal, setShowModal] = React.useState(false);
-  const [isDeleting, setIsDeleting] = React.useState(false);
-  const { showToast } = useToasts();
-  const history = useHistory();
-
-  const [trashed, setTrashed] = React.useState<Document[]>([]);
-
-  React.useEffect(() => {
-    async function getTrashed() {
-      const trashedDocs = await documents.fetchDeleted();
-      setTrashed(trashedDocs);
-    }
-    getTrashed();
-  }, [documents]);
-
-  const handleSubmit = React.useCallback(
-    async (ev: React.SyntheticEvent) => {
-      ev.preventDefault();
-
-      try {
-        setIsDeleting(true);
-        await documents.emptyTrash();
-        // If all trashed documents were not removed.
-        // E.g. More trashed docs than `emptyTrash` limit.
-        const trashedDocs = await documents.fetchDeleted();
-        setTrashed(trashedDocs);
-        showToast(t("Trash permanently deleted"), {
-          type: "success",
-        });
-        history.push("/trash");
-        setShowModal(false);
-      } catch (err) {
-        showToast(err.message, {
-          type: "error",
-        });
-      } finally {
-        setIsDeleting(false);
-        setTrashed([]);
-      }
-    },
-    [documents, history, showToast, t]
-  );
-
-  if (!trashed.length) {
-    return <></>;
-  }
-
-  return (
-    <>
-      <Button icon={<TrashIcon />} onClick={() => setShowModal(true)}>
-        {t("Empty trash")}
-      </Button>
-      {showModal && (
-        <Modal
-          title={t("Empty trash")}
-          isOpen={showModal}
-          onRequestClose={() => setShowModal(false)}
-          isCentered
-        >
-          <Flex column>
-            <form onSubmit={handleSubmit}>
-              <Text type="secondary">
-                <Trans
-                  defaults="Are you sure you want to permanently delete all documents in the trash? This action cannot be undone."
-                  components={{
-                    em: <strong />,
-                  }}
-                />
-              </Text>
-              <Button type="submit" danger>
-                {isDeleting ? `${t("Deleting")}…` : t("I’m sure – Delete")}
-              </Button>
-            </form>
-          </Flex>
-        </Modal>
-      )}
-    </>
-  );
-}
-
-export default observer(EmptyTrashMenu);

app/menus/GroupMenu.tsx

@@ -24,7 +24,7 @@ function GroupMenu({ group, onMembers }: Props) {
   });
   const [editModalOpen, setEditModalOpen] = React.useState(false);
   const [deleteModalOpen, setDeleteModalOpen] = React.useState(false);
-  const can = usePolicy(group.id);
+  const can = usePolicy(group);
 
   return (
     <>

app/menus/NewDocumentMenu.tsx

@@ -27,7 +27,7 @@ function NewDocumentMenu() {
   const { t } = useTranslation();
   const team = useCurrentTeam();
   const { collections, policies } = useStores();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
   const items = React.useMemo(
     () =>
       collections.orderedData.reduce<MenuItem[]>((filtered, collection) => {

app/menus/NewTemplateMenu.tsx

@@ -22,7 +22,7 @@ function NewTemplateMenu() {
   const { t } = useTranslation();
   const team = useCurrentTeam();
   const { collections, policies } = useStores();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   const items = React.useMemo(
     () =>

app/models/Collection.ts

@@ -1,5 +1,6 @@
 import { trim } from "lodash";
 import { action, computed, observable } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import { sortNavigationNodes } from "@shared/utils/collections";
 import CollectionsStore from "~/stores/CollectionsStore";
 import Document from "~/models/Document";
@@ -39,7 +40,7 @@ export default class Collection extends ParanoidModel {
 
   @Field
   @observable
-  permission: "read" | "read_write" | void;
+  permission: CollectionPermission | void;
 
   @Field
   @observable
@@ -101,8 +102,14 @@ export default class Collection extends ParanoidModel {
     return sortNavigationNodes(this.documents, this.sort);
   }
 
+  /**
+   * Updates the document identified by the given id in the collection in memory.
+   * Does not update the document in the database.
+   *
+   * @param document The document properties stored in the collection
+   */
   @action
-  updateDocument(document: Document) {
+  updateDocument(document: Pick<Document, "id" | "title" | "url">) {
     const travelNodes = (nodes: NavigationNode[]) =>
       nodes.forEach((node) => {
         if (node.id === document.id) {
@@ -116,6 +123,27 @@ export default class Collection extends ParanoidModel {
     travelNodes(this.documents);
   }
 
+  /**
+   * Removes the document identified by the given id from the collection in
+   * memory. Does not remove the document from the database.
+   *
+   * @param documentId The id of the document to remove.
+   */
+  @action
+  removeDocument(documentId: string) {
+    this.documents = this.documents.filter(function f(node): boolean {
+      if (node.id === documentId) {
+        return false;
+      }
+
+      if (node.children) {
+        node.children = node.children.filter(f);
+      }
+
+      return true;
+    });
+  }
+
   @action
   updateIndex(index: string) {
     this.index = index;
@@ -188,7 +216,7 @@ export default class Collection extends ParanoidModel {
   };
 
   export = () => {
-    return client.get("/collections.export", {
+    return client.post("/collections.export", {
       id: this.id,
     });
   };

app/models/CollectionGroupMembership.ts

@@ -1,4 +1,5 @@
 import { computed } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import BaseModel from "./BaseModel";
 
 class CollectionGroupMembership extends BaseModel {
@@ -8,16 +9,11 @@ class CollectionGroupMembership extends BaseModel {
 
   collectionId: string;
 
-  permission: string;
+  permission: CollectionPermission;
 
   @computed
   get isEditor(): boolean {
-    return this.permission === "read_write";
-  }
-
-  @computed
-  get isMaintainer(): boolean {
-    return this.permission === "maintainer";
+    return this.permission === CollectionPermission.ReadWrite;
   }
 }
 

app/models/Document.ts

@@ -2,10 +2,10 @@ import { addDays, differenceInDays } from "date-fns";
 import { floor } from "lodash";
 import { action, autorun, computed, observable, set } from "mobx";
 import parseTitle from "@shared/utils/parseTitle";
-import unescape from "@shared/utils/unescape";
 import DocumentsStore from "~/stores/DocumentsStore";
 import User from "~/models/User";
 import type { NavigationNode } from "~/types";
+import { client } from "~/utils/ApiClient";
 import Storage from "~/utils/Storage";
 import ParanoidModel from "./ParanoidModel";
 import View from "./View";
@@ -155,6 +155,19 @@ export default class Document extends ParanoidModel {
     );
   }
 
+  /**
+   * Returns whether there is a subscription for this document in the store.
+   * Does not consider remote state.
+   *
+   * @returns True if there is a subscription, false otherwise.
+   */
+  @computed
+  get isSubscribed(): boolean {
+    return !!this.store.rootStore.subscriptions.orderedData.find(
+      (subscription) => subscription.documentId === this.id
+    );
+  }
+
   @computed
   get isArchived(): boolean {
     return !!this.archivedAt;
@@ -255,15 +268,15 @@ export default class Document extends ParanoidModel {
   };
 
   @action
-  pin = async (collectionId?: string) => {
-    await this.store.rootStore.pins.create({
+  pin = (collectionId?: string) => {
+    return this.store.rootStore.pins.create({
       documentId: this.id,
       ...(collectionId ? { collectionId } : {}),
     });
   };
 
   @action
-  unpin = async (collectionId?: string) => {
+  unpin = (collectionId?: string) => {
     const pin = this.store.rootStore.pins.orderedData.find(
       (pin) =>
         pin.documentId === this.id &&
@@ -271,19 +284,39 @@ export default class Document extends ParanoidModel {
           (!collectionId && !pin.collectionId))
     );
 
-    await pin?.delete();
+    return pin?.delete();
   };
 
   @action
-  star = async () => {
+  star = () => {
     return this.store.star(this);
   };
 
   @action
-  unstar = async () => {
+  unstar = () => {
     return this.store.unstar(this);
   };
 
+  /**
+   * Subscribes the current user to this document.
+   *
+   * @returns A promise that resolves when the subscription is created.
+   */
+  @action
+  subscribe = () => {
+    return this.store.subscribe(this);
+  };
+
+  /**
+   * Unsubscribes the current user to this document.
+   *
+   * @returns A promise that resolves when the subscription is destroyed.
+   */
+  @action
+  unsubscribe = (userId: string) => {
+    return this.store.unsubscribe(userId, this);
+  };
+
   @action
   view = () => {
     // we don't record views for documents in the trash
@@ -304,7 +337,7 @@ export default class Document extends ParanoidModel {
   };
 
   @action
-  templatize = async () => {
+  templatize = () => {
     return this.store.templatize(this.id);
   };
 
@@ -386,21 +419,18 @@ export default class Document extends ParanoidModel {
     };
   }
 
-  download = async () => {
-    // Ensure the document is upto date with latest server contents
-    await this.fetch();
-    const body = unescape(this.text);
-    const blob = new Blob([`# ${this.title}\n\n${body}`], {
-      type: "text/markdown",
-    });
-    const url = URL.createObjectURL(blob);
-    const a = document.createElement("a");
-    // Firefox support requires the anchor tag be in the DOM to trigger the dl
-    if (document.body) {
-      document.body.appendChild(a);
-    }
-    a.href = url;
-    a.download = `${this.titleWithDefault}.md`;
-    a.click();
+  download = async (contentType: "text/html" | "text/markdown") => {
+    await client.post(
+      `/documents.export`,
+      {
+        id: this.id,
+      },
+      {
+        download: true,
+        headers: {
+          accept: contentType,
+        },
+      }
+    );
   };
 }

app/models/Integration.ts

@@ -1,14 +1,9 @@
 import { observable } from "mobx";
+import type { IntegrationSettings } from "@shared/types";
 import BaseModel from "~/models/BaseModel";
 import Field from "./decorators/Field";
 
-type Settings = {
-  url: string;
-  channel: string;
-  channelId: string;
-};
-
-class Integration extends BaseModel {
+class Integration<T = unknown> extends BaseModel {
   id: string;
 
   type: string;
@@ -21,7 +16,7 @@ class Integration extends BaseModel {
   @observable
   events: string[];
 
-  settings: Settings;
+  settings: IntegrationSettings<T>;
 }
 
 export default Integration;

app/models/Membership.ts

@@ -1,4 +1,5 @@
 import { computed } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import BaseModel from "./BaseModel";
 
 class Membership extends BaseModel {
@@ -8,16 +9,11 @@ class Membership extends BaseModel {
 
   collectionId: string;
 
-  permission: string;
+  permission: CollectionPermission;
 
   @computed
   get isEditor(): boolean {
-    return this.permission === "read_write";
-  }
-
-  @computed
-  get isMaintainer(): boolean {
-    return this.permission === "maintainer";
+    return this.permission === CollectionPermission.ReadWrite;
   }
 }
 

app/models/Subscription.ts

@@ -0,0 +1,29 @@
+import { observable } from "mobx";
+import BaseModel from "./BaseModel";
+import Field from "./decorators/Field";
+
+/**
+ * A subscription represents a request for a user to receive notifications for
+ * a document.
+ */
+class Subscription extends BaseModel {
+  @Field
+  @observable
+  id: string;
+
+  /** The user subscribing */
+  userId: string;
+
+  /** The document being subscribed to */
+  documentId: string;
+
+  /** The event being subscribed to */
+  @Field
+  @observable
+  event: string;
+
+  createdAt: string;
+  updatedAt: string;
+}
+
+export default Subscription;

app/routes/authenticated.tsx

@@ -11,7 +11,7 @@ import Layout from "~/components/AuthenticatedLayout";
 import CenteredContent from "~/components/CenteredContent";
 import PlaceholderDocument from "~/components/PlaceholderDocument";
 import Route from "~/components/ProfiledRoute";
-import SocketProvider from "~/components/SocketProvider";
+import WebsocketProvider from "~/components/WebsocketProvider";
 import useCurrentTeam from "~/hooks/useCurrentTeam";
 import usePolicy from "~/hooks/usePolicy";
 import { matchDocumentSlug as slug } from "~/utils/routeHelpers";
@@ -64,10 +64,10 @@ const RedirectDocument = ({
 
 function AuthenticatedRoutes() {
   const team = useCurrentTeam();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   return (
-    <SocketProvider>
+    <WebsocketProvider>
       <Layout>
         <React.Suspense
           fallback={
@@ -116,7 +116,7 @@ function AuthenticatedRoutes() {
           </Switch>
         </React.Suspense>
       </Layout>
-    </SocketProvider>
+    </WebsocketProvider>
   );
 }
 

app/scenes/Collection/Actions.tsx

@@ -18,7 +18,7 @@ type Props = {
 
 function Actions({ collection }: Props) {
   const { t } = useTranslation();
-  const can = usePolicy(collection.id);
+  const can = usePolicy(collection);
 
   return (
     <>

app/scenes/Collection/Empty.tsx

@@ -20,7 +20,7 @@ type Props = {
 
 function EmptyCollection({ collection }: Props) {
   const { t } = useTranslation();
-  const can = usePolicy(collection.id);
+  const can = usePolicy(collection);
   const collectionName = collection ? collection.name : "";
 
   const [

app/scenes/CollectionNew.tsx

@@ -4,6 +4,7 @@ import { observer } from "mobx-react";
 import * as React from "react";
 import { withTranslation, Trans, WithTranslation } from "react-i18next";
 import { randomElement } from "@shared/random";
+import { CollectionPermission } from "@shared/types";
 import { colorPalette } from "@shared/utils/collections";
 import { CollectionValidation } from "@shared/validations";
 import RootStore from "~/stores/RootStore";
@@ -38,7 +39,7 @@ class CollectionNew extends React.Component<Props> {
   sharing = true;
 
   @observable
-  permission = "read_write";
+  permission = CollectionPermission.ReadWrite;
 
   @observable
   isSaving: boolean;
@@ -100,8 +101,8 @@ class CollectionNew extends React.Component<Props> {
     this.hasOpenedIconPicker = true;
   };
 
-  handlePermissionChange = (newPermission: string) => {
-    this.permission = newPermission;
+  handlePermissionChange = (permission: CollectionPermission) => {
+    this.permission = permission;
   };
 
   handleSharingChange = (ev: React.ChangeEvent<HTMLInputElement>) => {

app/scenes/CollectionPermissions/AddGroupsToCollection.tsx

@@ -59,7 +59,6 @@ class AddGroupsToCollection extends React.Component<Props> {
       this.props.collectionGroupMemberships.create({
         collectionId: this.props.collection.id,
         groupId: group.id,
-        permission: "read_write",
       });
       this.props.toasts.showToast(
         t("{{ groupName }} was added to the collection", {

app/scenes/CollectionPermissions/AddPeopleToCollection.tsx

@@ -57,7 +57,6 @@ class AddPeopleToCollection extends React.Component<Props> {
       this.props.memberships.create({
         collectionId: this.props.collection.id,
         userId: user.id,
-        permission: "read_write",
       });
       this.props.toasts.showToast(
         t("{{ userName }} was added to the collection", {

app/scenes/CollectionPermissions/components/CollectionGroupMemberListItem.tsx

@@ -1,6 +1,7 @@
 import * as React from "react";
 import { useTranslation } from "react-i18next";
 import styled from "styled-components";
+import { CollectionPermission } from "@shared/types";
 import CollectionGroupMembership from "~/models/CollectionGroupMembership";
 import Group from "~/models/Group";
 import GroupListItem from "~/components/GroupListItem";
@@ -10,7 +11,7 @@ import CollectionGroupMemberMenu from "~/menus/CollectionGroupMemberMenu";
 type Props = {
   group: Group;
   collectionGroupMembership: CollectionGroupMembership | null | undefined;
-  onUpdate: (permission: string) => void;
+  onUpdate: (permission: CollectionPermission) => void;
   onRemove: () => void;
 };
 
@@ -21,19 +22,6 @@ const CollectionGroupMemberListItem = ({
   onRemove,
 }: Props) => {
   const { t } = useTranslation();
-  const PERMISSIONS = React.useMemo(
-    () => [
-      {
-        label: t("View only"),
-        value: "read",
-      },
-      {
-        label: t("View and edit"),
-        value: "read_write",
-      },
-    ],
-    [t]
-  );
 
   return (
     <GroupListItem
@@ -43,7 +31,16 @@ const CollectionGroupMemberListItem = ({
         <>
           <Select
             label={t("Permissions")}
-            options={PERMISSIONS}
+            options={[
+              {
+                label: t("View only"),
+                value: CollectionPermission.Read,
+              },
+              {
+                label: t("View and edit"),
+                value: CollectionPermission.ReadWrite,
+              },
+            ]}
             value={
               collectionGroupMembership
                 ? collectionGroupMembership.permission

app/scenes/CollectionPermissions/components/MemberListItem.tsx

@@ -2,6 +2,7 @@ import { observer } from "mobx-react";
 import * as React from "react";
 import { Trans, useTranslation } from "react-i18next";
 import styled from "styled-components";
+import { CollectionPermission } from "@shared/types";
 import Membership from "~/models/Membership";
 import User from "~/models/User";
 import Avatar from "~/components/Avatar";
@@ -19,7 +20,7 @@ type Props = {
   canEdit: boolean;
   onAdd?: () => void;
   onRemove?: () => void;
-  onUpdate?: (permission: string) => void;
+  onUpdate?: (permission: CollectionPermission) => void;
 };
 
 const MemberListItem = ({
@@ -31,19 +32,6 @@ const MemberListItem = ({
   canEdit,
 }: Props) => {
   const { t } = useTranslation();
-  const PERMISSIONS = React.useMemo(
-    () => [
-      {
-        label: t("View only"),
-        value: "read",
-      },
-      {
-        label: t("View and edit"),
-        value: "read_write",
-      },
-    ],
-    [t]
-  );
 
   return (
     <ListItem
@@ -67,7 +55,16 @@ const MemberListItem = ({
           {onUpdate && (
             <Select
               label={t("Permissions")}
-              options={PERMISSIONS}
+              options={[
+                {
+                  label: t("View only"),
+                  value: CollectionPermission.Read,
+                },
+                {
+                  label: t("View and edit"),
+                  value: CollectionPermission.ReadWrite,
+                },
+              ]}
               value={membership ? membership.permission : undefined}
               onChange={onUpdate}
               disabled={!canEdit}

app/scenes/CollectionPermissions/index.tsx

@@ -3,6 +3,7 @@ import { PlusIcon } from "outline-icons";
 import * as React from "react";
 import { useTranslation, Trans } from "react-i18next";
 import styled from "styled-components";
+import { CollectionPermission } from "@shared/types";
 import Collection from "~/models/Collection";
 import Group from "~/models/Group";
 import User from "~/models/User";
@@ -151,7 +152,7 @@ function CollectionPermissions({ collection }: Props) {
   );
 
   const handleChangePermission = React.useCallback(
-    async (permission: string) => {
+    async (permission: CollectionPermission) => {
       try {
         await collection.save({
           permission,
@@ -218,9 +219,10 @@ function CollectionPermissions({ collection }: Props) {
             }}
           />
         )}
-        {collection.permission === "read" && (
+        {collection.permission === CollectionPermission.ReadWrite && (
           <Trans
-            defaults="Team members can view documents in the <em>{{ collectionName }}</em> collection by default."
+            defaults="Team members can view and edit documents in the <em>{{ collectionName }}</em> collection by
+          default."
             values={{
               collectionName,
             }}
@@ -229,10 +231,9 @@ function CollectionPermissions({ collection }: Props) {
             }}
           />
         )}
-        {collection.permission === "read_write" && (
+        {collection.permission === CollectionPermission.Read && (
           <Trans
-            defaults="Team members can view and edit documents in the <em>{{ collectionName }}</em> collection by
-          default."
+            defaults="Team members can view documents in the <em>{{ collectionName }}</em> collection by default."
             values={{
               collectionName,
             }}

app/scenes/Document/components/Contents.tsx

@@ -57,15 +57,17 @@ export default function Contents({ headings, isFullWidth }: Props) {
         <Heading>{t("Contents")}</Heading>
         {headings.length ? (
           <List>
-            {headings.map((heading) => (
-              <ListItem
-                key={heading.id}
-                level={heading.level - headingAdjustment}
-                active={activeSlug === heading.id}
-              >
-                <Link href={`#${heading.id}`}>{heading.title}</Link>
-              </ListItem>
-            ))}
+            {headings
+              .filter((heading) => heading.level < 4)
+              .map((heading) => (
+                <ListItem
+                  key={heading.id}
+                  level={heading.level - headingAdjustment}
+                  active={activeSlug === heading.id}
+                >
+                  <Link href={`#${heading.id}`}>{heading.title}</Link>
+                </ListItem>
+              ))}
           </List>
         ) : (
           <Empty>

app/scenes/Document/components/DataLoader.tsx

@@ -8,6 +8,7 @@ import ErrorOffline from "~/scenes/ErrorOffline";
 import usePolicy from "~/hooks/usePolicy";
 import useStores from "~/hooks/useStores";
 import { NavigationNode } from "~/types";
+import Logger from "~/utils/Logger";
 import { NotFoundError, OfflineError } from "~/utils/errors";
 import history from "~/utils/history";
 import { matchDocumentEdit } from "~/utils/routeHelpers";
@@ -41,7 +42,7 @@ type Props = RouteComponentProps<Params, StaticContext, LocationState> & {
 };
 
 function DataLoader({ match, children }: Props) {
-  const { ui, shares, documents, auth, revisions } = useStores();
+  const { ui, shares, documents, auth, revisions, subscriptions } = useStores();
   const { team } = auth;
   const [error, setError] = React.useState<Error | null>(null);
   const { revisionId, shareId, documentSlug } = match.params;
@@ -57,7 +58,7 @@ function DataLoader({ match, children }: Props) {
     : undefined;
   const isEditRoute = match.path === matchDocumentEdit;
   const isEditing = isEditRoute || !!auth.team?.collaborativeEditing;
-  const can = usePolicy(document ? document.id : "");
+  const can = usePolicy(document?.id);
   const location = useLocation<LocationState>();
 
   React.useEffect(() => {
@@ -86,6 +87,22 @@ function DataLoader({ match, children }: Props) {
     fetchRevision();
   }, [revisions, revisionId]);
 
+  React.useEffect(() => {
+    async function fetchSubscription() {
+      if (document?.id) {
+        try {
+          await subscriptions.fetchPage({
+            documentId: document.id,
+            event: "documents.update",
+          });
+        } catch (err) {
+          Logger.error("Failed to fetch subscriptions", err);
+        }
+      }
+    }
+    fetchSubscription();
+  }, [document?.id, subscriptions]);
+
   const onCreateLink = React.useCallback(
     async (title: string) => {
       if (!document) {

app/scenes/Document/components/Header.tsx

@@ -100,7 +100,7 @@ function DocumentHeader({
   }, [onSave]);
 
   const { isDeleted, isTemplate } = document;
-  const can = usePolicy(document.id);
+  const can = usePolicy(document?.id);
   const canToggleEmbeds = team?.documentEmbeds;
   const canEdit = can.update && !isEditing;
   const toc = (

app/scenes/Document/components/Loading.tsx

@@ -1,6 +1,5 @@
 import { Location } from "history";
 import * as React from "react";
-import { useTranslation } from "react-i18next";
 import CenteredContent from "~/components/CenteredContent";
 import PageTitle from "~/components/PageTitle";
 import PlaceholderDocument from "~/components/PlaceholderDocument";
@@ -11,12 +10,9 @@ type Props = {
 };
 
 export default function Loading({ location }: Props) {
-  const { t } = useTranslation();
   return (
     <Container column auto>
-      <PageTitle
-        title={location.state ? location.state.title : t("Untitled")}
-      />
+      {location.state?.title && <PageTitle title={location.state.title} />}
       <CenteredContent>
         <PlaceholderDocument />
       </CenteredContent>

app/scenes/Document/components/MultiplayerEditor.tsx

@@ -139,6 +139,9 @@ function MultiplayerEditor({ onSynced, ...props }: Props, ref: any) {
     });
 
     if (debug) {
+      provider.on("close", (ev: MessageEvent) =>
+        Logger.debug("collaboration", "close", ev)
+      );
       provider.on("message", (ev: MessageEvent) =>
         Logger.debug("collaboration", "incoming", {
           message: ev.message,

app/scenes/Document/components/SharePopover.tsx

@@ -45,7 +45,7 @@ function SharePopover({
   const timeout = React.useRef<ReturnType<typeof setTimeout>>();
   const buttonRef = React.useRef<HTMLButtonElement>(null);
   const can = usePolicy(share ? share.id : "");
-  const documentAbilities = usePolicy(document.id);
+  const documentAbilities = usePolicy(document);
   const canPublish =
     can.update &&
     !document.isTemplate &&

app/scenes/Document/components/SocketPresence.ts

@@ -1,6 +1,6 @@
 import * as React from "react";
 import { USER_PRESENCE_INTERVAL } from "@shared/constants";
-import { SocketContext } from "~/components/SocketProvider";
+import { WebsocketContext } from "~/components/WebsocketProvider";
 
 type Props = {
   documentId: string;
@@ -8,9 +8,9 @@ type Props = {
 };
 
 export default class SocketPresence extends React.Component<Props> {
-  static contextType = SocketContext;
+  static contextType = WebsocketContext;
 
-  previousContext: typeof SocketContext;
+  previousContext: typeof WebsocketContext;
 
   editingInterval: ReturnType<typeof setInterval>;
 

app/scenes/DocumentReparent.tsx

@@ -2,6 +2,7 @@ import { observer } from "mobx-react";
 import { useState } from "react";
 import * as React from "react";
 import { Trans, useTranslation } from "react-i18next";
+import { CollectionPermission } from "@shared/types";
 import Collection from "~/models/Collection";
 import Button from "~/components/Button";
 import Flex from "~/components/Flex";
@@ -38,8 +39,8 @@ function DocumentReparent({ collection, item, onSubmit, onCancel }: Props) {
   const { t } = useTranslation();
   const prevCollection = collections.get(item.collectionId);
   const accessMapping = {
-    read_write: t("view and edit access"),
-    read: t("view only access"),
+    [CollectionPermission.ReadWrite]: t("view and edit access"),
+    [CollectionPermission.Read]: t("view only access"),
     null: t("no access"),
   };
 

app/scenes/GroupMembers/GroupMembers.tsx

@@ -26,7 +26,7 @@ function GroupMembers({ group }: Props) {
   const { users, groupMemberships } = useStores();
   const { showToast } = useToasts();
   const { t } = useTranslation();
-  const can = usePolicy(group.id);
+  const can = usePolicy(group);
 
   const handleAddModal = (state: boolean) => {
     setAddModalOpen(state);

app/scenes/Home.tsx

@@ -30,7 +30,7 @@ function Home() {
     pins.fetchPage();
   }, [pins]);
 
-  const canManageTeam = usePolicy(team.id).manage;
+  const canManageTeam = usePolicy(team).manage;
 
   return (
     <Scene

app/scenes/Invite.tsx

@@ -56,7 +56,7 @@ function Invite({ onSubmit }: Props) {
   const team = useCurrentTeam();
   const { t } = useTranslation();
   const predictedDomain = user.email.split("@")[1];
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   const handleSubmit = React.useCallback(
     async (ev: React.SyntheticEvent) => {

app/scenes/Login/index.tsx

@@ -243,7 +243,7 @@ const CheckEmailIcon = styled(EmailIcon)`
 
 const Background = styled(Fade)`
   width: 100vw;
-  height: 100vh;
+  height: 100%;
   background: ${(props) => props.theme.background};
   display: flex;
 `;

app/scenes/Settings/Drawio.tsx

@@ -0,0 +1,101 @@
+import { head } from "lodash";
+import { observer } from "mobx-react";
+import { BuildingBlocksIcon } from "outline-icons";
+import * as React from "react";
+import { useForm } from "react-hook-form";
+import { useTranslation, Trans } from "react-i18next";
+import { IntegrationType } from "@shared/types";
+import Integration from "~/models/Integration";
+import Button from "~/components/Button";
+import Heading from "~/components/Heading";
+import { ReactHookWrappedInput as Input } from "~/components/Input";
+import Scene from "~/components/Scene";
+import Text from "~/components/Text";
+import useStores from "~/hooks/useStores";
+import useToasts from "~/hooks/useToasts";
+
+type FormData = {
+  url: string;
+};
+
+const SERVICE_NAME = "diagrams";
+
+function Drawio() {
+  const { integrations } = useStores();
+  const { t } = useTranslation();
+  const { showToast } = useToasts();
+
+  React.useEffect(() => {
+    integrations.fetchPage({
+      service: SERVICE_NAME,
+      type: IntegrationType.Embed,
+    });
+  }, [integrations]);
+
+  const integration = head(integrations.orderedData) as
+    | Integration<IntegrationType.Embed>
+    | undefined;
+
+  const { register, handleSubmit: formHandleSubmit, formState } = useForm<
+    FormData
+  >({
+    mode: "all",
+    defaultValues: {
+      url: integration?.settings.url,
+    },
+  });
+
+  const handleSubmit = React.useCallback(
+    async (data: FormData) => {
+      try {
+        await integrations.save({
+          id: integration?.id,
+          type: IntegrationType.Embed,
+          service: SERVICE_NAME,
+          settings: {
+            url: data.url,
+          },
+        });
+
+        showToast(t("Settings saved"), {
+          type: "success",
+        });
+      } catch (err) {
+        showToast(err.message, {
+          type: "error",
+        });
+      }
+    },
+    [integrations, integration, t, showToast]
+  );
+
+  return (
+    <Scene title="Draw.io" icon={<BuildingBlocksIcon color="currentColor" />}>
+      <Heading>Draw.io</Heading>
+
+      <Text type="secondary">
+        <Trans>
+          Add your self-hosted draw.io installation url here to enable automatic
+          embedding of diagrams within documents.
+        </Trans>
+        <form onSubmit={formHandleSubmit(handleSubmit)}>
+          <p>
+            <Input
+              label={t("Draw.io deployment")}
+              placeholder={"https://app.diagrams.net/"}
+              pattern="https?://.*"
+              {...register("url", {
+                required: true,
+              })}
+            />
+            <Button type="submit" disabled={formState.isSubmitting}>
+              {formState.isSubmitting ? `${t("Saving")}…` : t("Save")}
+            </Button>
+          </p>
+        </form>
+      </Text>
+    </Scene>
+  );
+}
+
+export default observer(Drawio);

app/scenes/Settings/Groups.tsx

@@ -24,7 +24,7 @@ function Groups() {
   const { t } = useTranslation();
   const { groups } = useStores();
   const team = useCurrentTeam();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
   const [
     newGroupModalOpen,
     handleNewGroupModalOpen,

app/scenes/Settings/Members.tsx

@@ -40,7 +40,7 @@ function Members() {
   const [data, setData] = React.useState<User[]>([]);
   const [totalPages, setTotalPages] = React.useState(0);
   const [userIds, setUserIds] = React.useState<string[]>([]);
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
   const query = params.get("query") || "";
   const filter = params.get("filter") || "";
   const sort = params.get("sort") || "name";

app/scenes/Settings/Shares.tsx

@@ -21,7 +21,7 @@ function Shares() {
   const { t } = useTranslation();
   const { shares, auth } = useStores();
   const canShareDocuments = auth.team && auth.team.sharing;
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
   const [isLoading, setIsLoading] = React.useState(false);
   const [data, setData] = React.useState<Share[]>([]);
   const [totalPages, setTotalPages] = React.useState(0);

app/scenes/Settings/Slack.tsx

@@ -3,6 +3,7 @@ import { observer } from "mobx-react";
 import * as React from "react";
 import { useTranslation, Trans } from "react-i18next";
 import styled from "styled-components";
+import { IntegrationType } from "@shared/types";
 import Collection from "~/models/Collection";
 import Integration from "~/models/Integration";
 import Button from "~/components/Button";
@@ -124,7 +125,9 @@ function Slack() {
                   <SlackListItem
                     key={integration.id}
                     collection={collection}
-                    integration={integration}
+                    integration={
+                      integration as Integration<IntegrationType.Post>
+                    }
                   />
                 );
               }

app/scenes/Settings/Tokens.tsx

@@ -23,7 +23,7 @@ function Tokens() {
   const { t } = useTranslation();
   const { apiKeys } = useStores();
   const [newModalOpen, handleNewModalOpen, handleNewModalClose] = useBoolean();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   return (
     <Scene

app/scenes/Settings/Webhooks.tsx

@@ -23,7 +23,7 @@ function Webhooks() {
   const { t } = useTranslation();
   const { webhookSubscriptions } = useStores();
   const [newModalOpen, handleNewModalOpen, handleNewModalClose] = useBoolean();
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   return (
     <Scene

app/scenes/Settings/components/SlackListItem.tsx

@@ -4,6 +4,7 @@ import * as React from "react";
 import { Trans, useTranslation } from "react-i18next";
 import { usePopoverState, PopoverDisclosure } from "reakit/Popover";
 import styled from "styled-components";
+import { IntegrationType } from "@shared/types";
 import Collection from "~/models/Collection";
 import Integration from "~/models/Integration";
 import Button from "~/components/Button";
@@ -17,7 +18,7 @@ import Text from "~/components/Text";
 import useToasts from "~/hooks/useToasts";
 
 type Props = {
-  integration: Integration;
+  integration: Integration<IntegrationType.Post>;
   collection: Collection;
 };
 

app/scenes/Settings/components/WebhookSubscriptionForm.tsx

@@ -31,8 +31,6 @@ const WEBHOOK_EVENTS = {
     "documents.archive",
     "documents.unarchive",
     "documents.restore",
-    "documents.star",
-    "documents.unstar",
     "documents.move",
     "documents.update",
     "documents.update.delayed",

app/scenes/Templates.tsx

@@ -21,7 +21,7 @@ function Templates(props: RouteComponentProps<{ sort: string }>) {
   const team = useCurrentTeam();
   const { fetchTemplates, templates, templatesAlphabetical } = documents;
   const { sort } = props.match.params;
-  const can = usePolicy(team.id);
+  const can = usePolicy(team);
 
   return (
     <Scene

app/scenes/Trash.tsx

@@ -2,33 +2,18 @@ import { observer } from "mobx-react";
 import { TrashIcon } from "outline-icons";
 import * as React from "react";
 import { useTranslation } from "react-i18next";
-import { Action } from "~/components/Actions";
 import Empty from "~/components/Empty";
 import Heading from "~/components/Heading";
 import PaginatedDocumentList from "~/components/PaginatedDocumentList";
 import Scene from "~/components/Scene";
 import Subheading from "~/components/Subheading";
-import useCurrentUser from "~/hooks/useCurrentUser";
 import useStores from "~/hooks/useStores";
-import EmptyTrashMenu from "~/menus/EmptyTrashMenu";
 
 function Trash() {
   const { t } = useTranslation();
   const { documents } = useStores();
-  const user = useCurrentUser();
-
   return (
-    <Scene
-      icon={<TrashIcon color="currentColor" />}
-      title={t("Trash")}
-      actions={
-        user.isAdmin && (
-          <Action>
-            <EmptyTrashMenu />
-          </Action>
-        )
-      }
-    >
+    <Scene icon={<TrashIcon color="currentColor" />} title={t("Trash")}>
       <Heading>{t("Trash")}</Heading>
       <PaginatedDocumentList
         documents={documents.deleted}

app/stores/AuthStore.ts

@@ -260,12 +260,6 @@ export default class AuthStore {
 
   @action
   logout = async (savePath = false) => {
-    if (!this.token) {
-      return;
-    }
-
-    client.post(`/auth.delete`);
-
     // if this logout was forced from an authenticated route then
     // save the current path so we can go back there once signed in
     if (savePath) {
@@ -276,10 +270,19 @@ export default class AuthStore {
       }
     }
 
+    // If there is no auth token stored there is nothing else to do
+    if (!this.token) {
+      return;
+    }
+
+    // invalidate authentication token on server
+    client.post(`/auth.delete`);
+
     // remove authentication token itself
     removeCookie("accessToken", {
       path: "/",
     });
+
     // remove session record on apex cookie
     const team = this.team;
 

app/stores/BaseStore.ts

@@ -5,12 +5,10 @@ import { Class } from "utility-types";
 import RootStore from "~/stores/RootStore";
 import BaseModel from "~/models/BaseModel";
 import Policy from "~/models/Policy";
-import { PaginationParams } from "~/types";
+import { PaginationParams, PartialWithId } from "~/types";
 import { client } from "~/utils/ApiClient";
 import { AuthorizationError, NotFoundError } from "~/utils/errors";
 
-type PartialWithId<T> = Partial<T> & { id: string };
-
 export enum RPCAction {
   Info = "info",
   List = "list",

app/stores/CollectionGroupMembershipsStore.ts

@@ -1,5 +1,6 @@
 import invariant from "invariant";
 import { action, runInAction } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import CollectionGroupMembership from "~/models/CollectionGroupMembership";
 import { PaginationParams } from "~/types";
 import { client } from "~/utils/ApiClient";
@@ -45,7 +46,7 @@ export default class CollectionGroupMembershipsStore extends BaseStore<
   }: {
     collectionId: string;
     groupId: string;
-    permission: string;
+    permission?: CollectionPermission;
   }) {
     const res = await client.post("/collections.add_group", {
       id: collectionId,

app/stores/CollectionsStore.ts

@@ -1,6 +1,7 @@
 import invariant from "invariant";
 import { concat, find, last } from "lodash";
 import { computed, action } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import Collection from "~/models/Collection";
 import { NavigationNode } from "~/types";
 import { client } from "~/utils/ApiClient";
@@ -171,8 +172,10 @@ export default class CollectionsStore extends BaseStore<Collection> {
 
   @computed
   get publicCollections() {
-    return this.orderedData.filter((collection) =>
-      ["read", "read_write"].includes(collection.permission || "")
+    return this.orderedData.filter(
+      (collection) =>
+        collection.permission &&
+        Object.values(CollectionPermission).includes(collection.permission)
     );
   }
 

app/stores/DocumentsStore.ts

@@ -381,7 +381,7 @@ export default class DocumentsStore extends BaseStore<Document> {
 
   @action
   searchTitles = async (query: string) => {
-    const res = await client.get("/documents.search_titles", {
+    const res = await client.post("/documents.search_titles", {
       query,
     });
     invariant(res?.data, "Search response should be available");
@@ -397,7 +397,7 @@ export default class DocumentsStore extends BaseStore<Document> {
     options: SearchParams
   ): Promise<SearchResult[]> => {
     const compactedOptions = omitBy(options, (o) => !o);
-    const res = await client.get("/documents.search", {
+    const res = await client.post("/documents.search", {
       ...compactedOptions,
       query,
     });
@@ -684,33 +684,6 @@ export default class DocumentsStore extends BaseStore<Document> {
     }
   }
 
-  @action
-  async emptyTrash() {
-    const res = await client.post("/documents.empty_trash");
-    invariant(res?.data, "Data should be available");
-    res.data.forEach(
-      ({
-        documentId,
-        collectionId,
-      }: {
-        documentId: string;
-        collectionId: string;
-      }) => {
-        this.remove(documentId);
-
-        const share = this.rootStore.shares.getByDocumentId(documentId);
-        if (share) {
-          this.rootStore.shares.remove(share.id);
-        }
-
-        const collection = this.rootStore.collections.data.get(collectionId);
-        if (collection) {
-          collection.refresh();
-        }
-      }
-    );
-  }
-
   @action
   archive = async (document: Document) => {
     const res = await client.post("/documents.archive", {
@@ -745,10 +718,7 @@ export default class DocumentsStore extends BaseStore<Document> {
       document.updateFromJson(res.data);
       this.addPolicies(res.policies);
     });
-
-    const collection = this.rootStore.collections.data.get(
-      document.collectionId
-    );
+    const collection = this.getCollectionForDocument(document);
     if (collection) {
       collection.refresh();
     }
@@ -770,20 +740,37 @@ export default class DocumentsStore extends BaseStore<Document> {
     }
   };
 
-  star = async (document: Document) => {
-    await this.rootStore.stars.create({
+  star = (document: Document) => {
+    return this.rootStore.stars.create({
       documentId: document.id,
     });
   };
 
-  unstar = async (document: Document) => {
+  unstar = (document: Document) => {
     const star = this.rootStore.stars.orderedData.find(
       (star) => star.documentId === document.id
     );
-    await star?.delete();
+    return star?.delete();
   };
 
-  getByUrl = (url = ""): Document | null | undefined => {
+  subscribe = (document: Document) => {
+    return this.rootStore.subscriptions.create({
+      documentId: document.id,
+      event: "documents.update",
+    });
+  };
+
+  unsubscribe = (userId: string, document: Document) => {
+    const subscription = this.rootStore.subscriptions.orderedData.find(
+      (subscription) =>
+        subscription.documentId === document.id &&
+        subscription.userId === userId
+    );
+
+    return subscription?.delete();
+  };
+
+  getByUrl = (url = ""): Document | undefined => {
     return find(this.orderedData, (doc) => url.endsWith(doc.urlId));
   };
 

app/stores/MembershipsStore.ts

@@ -1,5 +1,6 @@
 import invariant from "invariant";
 import { action, runInAction } from "mobx";
+import { CollectionPermission } from "@shared/types";
 import Membership from "~/models/Membership";
 import { PaginationParams } from "~/types";
 import { client } from "~/utils/ApiClient";
@@ -43,7 +44,7 @@ export default class MembershipsStore extends BaseStore<Membership> {
   }: {
     collectionId: string;
     userId: string;
-    permission: string;
+    permission?: CollectionPermission;
   }) {
     const res = await client.post("/collections.add_user", {
       id: collectionId,

app/stores/RootStore.ts

@@ -18,6 +18,7 @@ import RevisionsStore from "./RevisionsStore";
 import SearchesStore from "./SearchesStore";
 import SharesStore from "./SharesStore";
 import StarsStore from "./StarsStore";
+import SubscriptionsStore from "./SubscriptionsStore";
 import ToastsStore from "./ToastsStore";
 import UiStore from "./UiStore";
 import UsersStore from "./UsersStore";
@@ -45,6 +46,7 @@ export default class RootStore {
   shares: SharesStore;
   ui: UiStore;
   stars: StarsStore;
+  subscriptions: SubscriptionsStore;
   users: UsersStore;
   views: ViewsStore;
   toasts: ToastsStore;
@@ -72,6 +74,7 @@ export default class RootStore {
     this.searches = new SearchesStore(this);
     this.shares = new SharesStore(this);
     this.stars = new StarsStore(this);
+    this.subscriptions = new SubscriptionsStore(this);
     this.ui = new UiStore();
     this.users = new UsersStore(this);
     this.views = new ViewsStore(this);
@@ -99,6 +102,7 @@ export default class RootStore {
     this.searches.clear();
     this.shares.clear();
     this.stars.clear();
+    this.subscriptions.clear();
     this.fileOperations.clear();
     // this.ui omitted to keep ui settings between sessions
     this.users.clear();

app/stores/SubscriptionsStore.ts

@@ -0,0 +1,11 @@
+import Subscription from "~/models/Subscription";
+import BaseStore, { RPCAction } from "./BaseStore";
+import RootStore from "./RootStore";
+
+export default class SubscriptionsStore extends BaseStore<Subscription> {
+  actions = [RPCAction.List, RPCAction.Create, RPCAction.Delete];
+
+  constructor(rootStore: RootStore) {
+    super(rootStore, Subscription);
+  }
+}

app/types.ts

@@ -1,7 +1,12 @@
 import { Location, LocationDescriptor } from "history";
 import { TFunction } from "react-i18next";
 import RootStore from "~/stores/RootStore";
-import Document from "~/models/Document";
+import Document from "./models/Document";
+import FileOperation from "./models/FileOperation";
+import Pin from "./models/Pin";
+import Star from "./models/Star";
+
+export type PartialWithId<T> = Partial<T> & { id: string };
 
 export type MenuItemButton = {
   type: "button";
@@ -178,3 +183,34 @@ export type ToastOptions = {
     onClick: React.MouseEventHandler<HTMLSpanElement>;
   };
 };
+
+export type WebsocketEntityDeletedEvent = {
+  modelId: string;
+};
+
+export type WebsocketEntitiesEvent = {
+  documentIds: { id: string; updatedAt?: string }[];
+  collectionIds: { id: string; updatedAt?: string }[];
+  groupIds: { id: string; updatedAt?: string }[];
+  teamIds: string[];
+  event: string;
+};
+
+export type WebsocketCollectionUserEvent = {
+  collectionId: string;
+  userId: string;
+};
+
+export type WebsocketCollectionUpdateIndexEvent = {
+  collectionId: string;
+  index: string;
+};
+
+export type WebsocketEvent =
+  | PartialWithId<Pin>
+  | PartialWithId<Star>
+  | PartialWithId<FileOperation>
+  | WebsocketCollectionUserEvent
+  | WebsocketCollectionUpdateIndexEvent
+  | WebsocketEntityDeletedEvent
+  | WebsocketEntitiesEvent;

app/utils/ApiClient.ts

@@ -5,6 +5,7 @@ import queryString from "query-string";
 import EDITOR_VERSION from "@shared/editor/version";
 import stores from "~/stores";
 import isCloudHosted from "~/utils/isCloudHosted";
+import Logger from "./Logger";
 import download from "./download";
 import {
   AuthorizationError,
@@ -12,6 +13,7 @@ import {
   NetworkError,
   NotFoundError,
   OfflineError,
+  RateLimitExceededError,
   RequestError,
   ServiceUnavailableError,
   UpdateRequiredError,
@@ -23,6 +25,7 @@ type Options = {
 
 type FetchOptions = {
   download?: boolean;
+  headers?: Record<string, string>;
 };
 
 const fetchWithRetry = retry(fetch);
@@ -79,6 +82,7 @@ class ApiClient {
       "cache-control": "no-cache",
       "x-editor-version": EDITOR_VERSION,
       pragma: "no-cache",
+      ...options?.headers,
     };
 
     // for multipart forms or other non JSON requests fetch
@@ -181,7 +185,20 @@ class ApiClient {
       throw new ServiceUnavailableError(error.message);
     }
 
-    throw new RequestError(`Error ${response.status}: ${error.message}`);
+    if (response.status === 429) {
+      throw new RateLimitExceededError(
+        `Too many requests, try again in a minute.`
+      );
+    }
+
+    const err = new RequestError(`Error ${response.status}`);
+    Logger.error("Request failed", err, {
+      ...error,
+      url: urlToFetch,
+    });
+
+    // Still need to throw to trigger retry
+    throw err;
   };
 
   get = (

app/utils/download.ts

@@ -67,7 +67,6 @@ export default function download(
     if ("download" in a) {
       a.href = url;
       a.setAttribute("download", fn);
-      a.innerHTML = "downloading…";
       D.body && D.body.appendChild(a);
       setTimeout(function () {
         a.click();

app/utils/errors.ts

@@ -12,6 +12,8 @@ export class OfflineError extends ExtendableError {}
 
 export class ServiceUnavailableError extends ExtendableError {}
 
+export class RateLimitExceededError extends ExtendableError {}
+
 export class RequestError extends ExtendableError {}
 
 export class UpdateRequiredError extends ExtendableError {}

app/utils/sentry.ts

@@ -22,6 +22,7 @@ export function initSentry(history: History) {
       "NetworkError",
       "NotFoundError",
       "OfflineError",
+      "RateLimitExceededError",
       "ServiceUnavailableError",
       "UpdateRequiredError",
       "ChunkLoadError",

package.json

@@ -12,7 +12,7 @@
     "build": "yarn clean && yarn build:webpack && yarn build:i18n && yarn build:server",
     "start": "node ./build/server/index.js",
     "dev": "NODE_ENV=development yarn concurrently -n api,collaboration  -c \"blue,magenta\" \"node --inspect=0.0.0.0 build/server/index.js --services=collaboration,websockets,admin,web,worker\"",
-    "dev:watch": "nodemon --exec \"yarn build:server && yarn dev\" -e js,ts --ignore build/ --ignore app/ --ignore shared/editor",
+    "dev:watch": "nodemon --exec \"yarn build:server && yarn dev\" -e js,ts,tsx --ignore build/ --ignore app/ --ignore shared/editor",
     "lint": "eslint app server shared",
     "deploy": "git push heroku master",
     "prepare": "husky install",
@@ -20,6 +20,7 @@
     "heroku-postbuild": "yarn build:webpack && yarn build:server && yarn copy:i18n && yarn db:migrate",
     "sequelize:migrate": "sequelize db:migrate",
     "db:create-migration": "sequelize migration:create",
+    "db:create": "sequelize db:create",
     "db:migrate": "sequelize db:migrate",
     "db:rollback": "sequelize db:migrate:undo",
     "db:reset": "sequelize db:drop && sequelize db:create && sequelize db:migrate",
@@ -50,8 +51,8 @@
     "@babel/plugin-transform-regenerator": "^7.10.4",
     "@babel/preset-env": "^7.16.0",
     "@babel/preset-react": "^7.16.0",
-    "@bull-board/api": "^4.0.0",
-    "@bull-board/koa": "^4.0.0",
+    "@bull-board/api": "^4.2.2",
+    "@bull-board/koa": "^4.2.2",
     "@dnd-kit/core": "^4.0.3",
     "@dnd-kit/modifiers": "^4.0.0",
     "@dnd-kit/sortable": "^5.1.0",
@@ -66,7 +67,7 @@
     "@sentry/node": "^6.3.1",
     "@sentry/react": "^6.3.1",
     "@sentry/tracing": "^6.3.1",
-    "@theo.gravity/datadog-apm": "2.1.0",
+    "@theo.gravity/datadog-apm": "2.2.0",
     "@tippy.js/react": "^2.2.2",
     "@tommoor/remove-markdown": "^0.3.2",
     "@types/mermaid": "^8.2.9",
@@ -76,7 +77,7 @@
     "babel-plugin-styled-components": "^1.11.1",
     "babel-plugin-transform-class-properties": "^6.24.1",
     "body-scroll-lock": "^4.0.0-beta.0",
-    "bull": "^3.29.0",
+    "bull": "^4.8.5",
     "cancan": "3.1.0",
     "chalk": "^4.1.0",
     "class-validator": "^0.13.2",
@@ -105,8 +106,9 @@
     "i18next-http-backend": "^1.3.2",
     "immutable": "^4.0.0",
     "invariant": "^2.2.4",
-    "ioredis": "^4.28.0",
+    "ioredis": "^4.28.5",
     "is-printable-key-event": "^1.0.0",
+    "jsdom": "^20.0.0",
     "json-loader": "0.5.4",
     "jsonwebtoken": "^8.5.0",
     "jszip": "^3.10.0",
@@ -123,6 +125,7 @@
     "koa-send": "5.0.1",
     "koa-sslify": "2.1.2",
     "koa-static": "^4.0.1",
+    "koa-useragent": "^4.1.0",
     "lodash": "^4.17.21",
     "mammoth": "^1.4.19",
     "markdown-it": "^13.0.1",
@@ -134,8 +137,9 @@
     "mobx-react": "^6.3.1",
     "natural-sort": "^1.0.0",
     "node-fetch": "2.6.7",
+    "node-htmldiff": "^0.9.4",
     "nodemailer": "^6.6.1",
-    "outline-icons": "^1.43.1",
+    "outline-icons": "^1.44.0",
     "oy-vey": "^0.11.2",
     "passport": "^0.6.0",
     "passport-google-oauth2": "^0.2.0",
@@ -224,12 +228,13 @@
     "@pmmmwh/react-refresh-webpack-plugin": "^0.5.4",
     "@relative-ci/agent": "^3.0.0",
     "@types/body-scroll-lock": "^3.1.0",
-    "@types/bull": "^3.15.5",
+    "@types/bull": "^3.15.9",
     "@types/crypto-js": "^4.1.0",
     "@types/datadog-metrics": "^0.6.2",
     "@types/emoji-regex": "^9.2.0",
     "@types/enzyme": "^3.10.10",
     "@types/enzyme-adapter-react-16": "^1.0.6",
+    "@types/express-useragent": "^1.0.2",
     "@types/formidable": "^2.0.5",
     "@types/fs-extra": "^9.0.13",
     "@types/fuzzy-search": "^2.1.2",
@@ -246,6 +251,7 @@
     "@types/koa-router": "^7.4.4",
     "@types/koa-sslify": "^2.1.0",
     "@types/koa-static": "^4.0.2",
+    "@types/koa-useragent": "^2.1.2",
     "@types/markdown-it": "^12.2.3",
     "@types/markdown-it-container": "^2.0.4",
     "@types/markdown-it-emoji": "^2.0.2",
@@ -279,6 +285,7 @@
     "@types/react-table": "^7.7.9",
     "@types/react-virtualized-auto-sizer": "^1.0.1",
     "@types/react-window": "^1.8.5",
+    "@types/redis-info": "^3.0.0",
     "@types/refractor": "^3.0.2",
     "@types/semver": "^7.3.10",
     "@types/sequelize": "^4.28.10",

server/RateLimiter.ts

@@ -1,7 +1,9 @@
-import { RateLimiterRedis } from "rate-limiter-flexible";
+import {
+  IRateLimiterStoreOptions,
+  RateLimiterRedis,
+} from "rate-limiter-flexible";
 import env from "@server/env";
 import Redis from "@server/redis";
-import { RateLimiterConfig } from "@server/types";
 
 export default class RateLimiter {
   constructor() {
@@ -11,6 +13,7 @@ export default class RateLimiter {
   static readonly RATE_LIMITER_REDIS_KEY_PREFIX = "rl";
 
   static readonly rateLimiterMap = new Map<string, RateLimiterRedis>();
+
   static readonly defaultRateLimiter = new RateLimiterRedis({
     storeClient: Redis.defaultClient,
     points: env.RATE_LIMITER_REQUESTS,
@@ -22,7 +25,7 @@ export default class RateLimiter {
     return this.rateLimiterMap.get(path) || this.defaultRateLimiter;
   }
 
-  static setRateLimiter(path: string, config: RateLimiterConfig): void {
+  static setRateLimiter(path: string, config: IRateLimiterStoreOptions): void {
     const rateLimiter = new RateLimiterRedis(config);
     this.rateLimiterMap.set(path, rateLimiter);
   }
@@ -31,3 +34,29 @@ export default class RateLimiter {
     return this.rateLimiterMap.has(path);
   }
 }
+
+/**
+ * Re-useable configuration for rate limiter middleware.
+ */
+export const RateLimiterStrategy = {
+  /** Allows five requests per minute, per IP address */
+  FivePerMinute: {
+    duration: 60,
+    requests: 5,
+  },
+  /** Allows ten requests per minute, per IP address */
+  TenPerMinute: {
+    duration: 60,
+    requests: 10,
+  },
+  /** Allows ten requests per hour, per IP address */
+  TenPerHour: {
+    duration: 3600,
+    requests: 10,
+  },
+  /** Allows five requests per hour, per IP address */
+  FivePerHour: {
+    duration: 3600,
+    requests: 5,
+  },
+};

server/__mocks__/RateLimiter.ts

@@ -1,16 +0,0 @@
-export default class MockRateLimiter {
-  static getRateLimiter() {
-    return {
-      points: 100,
-      consume: jest.fn(),
-    };
-  }
-
-  static setRateLimiter() {
-    //
-  }
-
-  static hasRateLimiter() {
-    return false;
-  }
-}

server/commands/documentLoader.ts

@@ -13,6 +13,7 @@ type Props = {
   id?: string;
   shareId?: string;
   user?: User;
+  includeState?: boolean;
 };
 
 type Result = {
@@ -25,6 +26,7 @@ export default async function loadDocument({
   id,
   shareId,
   user,
+  includeState,
 }: Props): Promise<Result> {
   let document;
   let collection;
@@ -97,10 +99,6 @@ export default async function loadDocument({
     const canReadDocument = user && can(user, "read", document);
 
     if (canReadDocument) {
-      await share.update({
-        lastAccessedAt: new Date(),
-      });
-
       // Cannot use document.collection here as it does not include the
       // documentStructure by default through the relationship.
       collection = await Collection.findByPk(document.collectionId);
@@ -156,14 +154,11 @@ export default async function loadDocument({
     if (!team.sharing) {
       throw AuthorizationError();
     }
-
-    await share.update({
-      lastAccessedAt: new Date(),
-    });
   } else {
     document = await Document.findByPk(id as string, {
       userId: user ? user.id : undefined,
       paranoid: false,
+      includeState,
     });
 
     if (!document) {

server/commands/documentUpdater.ts

@@ -1,5 +1,6 @@
 import { Transaction } from "sequelize";
 import { Event, Document, User } from "@server/models";
+import DocumentHelper from "@server/models/helpers/DocumentHelper";
 
 type Props = {
   /** The user updating the document */
@@ -62,7 +63,7 @@ export default async function documentUpdater({
   }
   if (text !== undefined) {
     if (user.team?.collaborativeEditing) {
-      document.updateFromMarkdown(text, append);
+      document = DocumentHelper.applyMarkdownToDocument(document, text, append);
     } else if (append) {
       document.text += text;
     } else {

server/commands/subscriptionCreator.test.ts

@@ -0,0 +1,275 @@
+import { sequelize } from "@server/database/sequelize";
+import { Subscription, Event } from "@server/models";
+import { buildDocument, buildUser } from "@server/test/factories";
+import { getTestDatabase } from "@server/test/support";
+import subscriptionCreator from "./subscriptionCreator";
+import subscriptionDestroyer from "./subscriptionDestroyer";
+
+const db = getTestDatabase();
+
+beforeEach(db.flush);
+afterAll(db.disconnect);
+
+describe("subscriptionCreator", () => {
+  const ip = "127.0.0.1";
+  const subscribedEvent = "documents.update";
+
+  it("should create a subscription", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    const event = await Event.findOne();
+
+    expect(subscription.documentId).toEqual(document.id);
+    expect(subscription.userId).toEqual(user.id);
+    expect(event?.name).toEqual("subscriptions.create");
+    expect(event?.modelId).toEqual(subscription.id);
+    expect(event?.actorId).toEqual(subscription.userId);
+    expect(event?.userId).toEqual(subscription.userId);
+    expect(event?.documentId).toEqual(subscription.documentId);
+  });
+
+  it("should not create another subscription if one already exists", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription0 = await Subscription.create({
+      userId: user.id,
+      documentId: document.id,
+      event: subscribedEvent,
+    });
+
+    const subscription1 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    expect(subscription0.event).toEqual(subscribedEvent);
+    expect(subscription1.event).toEqual(subscribedEvent);
+
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription1.userId).toEqual(user.id);
+
+    // Primary concern
+    expect(subscription0.id).toEqual(subscription1.id);
+
+    // Edge cases
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription1.documentId).toEqual(document.id);
+
+    expect(subscription0.userId).toEqual(subscription1.userId);
+    expect(subscription0.documentId).toEqual(subscription1.documentId);
+  });
+
+  it("should enable subscription by overriding one that exists in disabled state", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription0 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    await sequelize.transaction(async (transaction) =>
+      subscriptionDestroyer({
+        user,
+        subscription: subscription0,
+        ip,
+        transaction,
+      })
+    );
+
+    expect(subscription0.id).toBeDefined();
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.deletedAt).toBeDefined();
+
+    const subscription1 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    const events = await Event.count();
+
+    // 3 events. 1 create, 1 destroy and 1 re-create.
+    expect(events).toEqual(3);
+
+    expect(subscription0.id).toEqual(subscription1.id);
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription1.documentId).toEqual(document.id);
+    expect(subscription1.userId).toEqual(user.id);
+    expect(subscription0.id).toEqual(subscription1.id);
+    expect(subscription0.userId).toEqual(subscription1.userId);
+    expect(subscription0.documentId).toEqual(subscription1.documentId);
+  });
+
+  it("should fetch already enabled subscription on create request", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription0 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    const subscription1 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    // Should emit 1 event instead of 2.
+    const events = await Event.count();
+    expect(events).toEqual(1);
+
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription1.documentId).toEqual(document.id);
+    expect(subscription1.userId).toEqual(user.id);
+    expect(subscription0.id).toEqual(subscription1.id);
+    expect(subscription0.userId).toEqual(subscription1.userId);
+    expect(subscription0.documentId).toEqual(subscription1.documentId);
+  });
+
+  it("should emit event when re-creating subscription", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription0 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    await sequelize.transaction(async (transaction) =>
+      subscriptionDestroyer({
+        user,
+        subscription: subscription0,
+        ip,
+        transaction,
+      })
+    );
+
+    expect(subscription0.id).toBeDefined();
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.deletedAt).toBeDefined();
+
+    const subscription1 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    // Should emit 3 events.
+    // 2 create, 1 destroy.
+    const events = await Event.findAll();
+    expect(events.length).toEqual(3);
+
+    expect(events[0].name).toEqual("subscriptions.create");
+    expect(events[0].documentId).toEqual(document.id);
+    expect(events[1].name).toEqual("subscriptions.delete");
+    expect(events[1].documentId).toEqual(document.id);
+    expect(events[2].name).toEqual("subscriptions.create");
+    expect(events[2].documentId).toEqual(document.id);
+
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription1.documentId).toEqual(document.id);
+    expect(subscription1.userId).toEqual(user.id);
+    expect(subscription0.id).toEqual(subscription1.id);
+    expect(subscription0.userId).toEqual(subscription1.userId);
+    expect(subscription0.documentId).toEqual(subscription1.documentId);
+  });
+
+  it("should fetch deletedAt column with paranoid option", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription0 = await sequelize.transaction(async (transaction) =>
+      subscriptionCreator({
+        user,
+        documentId: document.id,
+        event: subscribedEvent,
+        ip,
+        transaction,
+      })
+    );
+
+    const events = await Event.count();
+    expect(events).toEqual(1);
+
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription0.userId).toEqual(user.id);
+    expect(subscription0.documentId).toEqual(document.id);
+    expect(subscription0.deletedAt).toEqual(null);
+  });
+});

server/commands/subscriptionCreator.ts

@@ -0,0 +1,74 @@
+import { Transaction } from "sequelize";
+import { Subscription, Event, User } from "@server/models";
+
+type Props = {
+  /** The user creating the subscription */
+  user: User;
+  /** The document to subscribe to */
+  documentId?: string;
+  /** Event to subscribe to */
+  event: string;
+  /** The IP address of the incoming request */
+  ip: string;
+  /** Whether the subscription should be restored if it exists in a deleted state  */
+  resubscribe?: boolean;
+  transaction: Transaction;
+};
+
+/**
+ * This command creates a subscription of a user to a document.
+ *
+ * @returns The subscription that was created
+ */
+export default async function subscriptionCreator({
+  user,
+  documentId,
+  event,
+  ip,
+  resubscribe = true,
+  transaction,
+}: Props): Promise<Subscription> {
+  const [subscription, created] = await Subscription.findOrCreate({
+    where: {
+      userId: user.id,
+      documentId,
+      event,
+    },
+    transaction,
+    // Previous subscriptions are soft-deleted, we want to know about them here
+    paranoid: false,
+  });
+
+  // If the subscription was deleted, then just restore the existing row.
+  if (subscription.deletedAt && resubscribe) {
+    subscription.restore({ transaction });
+
+    await Event.create(
+      {
+        name: "subscriptions.create",
+        modelId: subscription.id,
+        actorId: user.id,
+        userId: user.id,
+        documentId,
+        ip,
+      },
+      { transaction }
+    );
+  }
+
+  if (created) {
+    await Event.create(
+      {
+        name: "subscriptions.create",
+        modelId: subscription.id,
+        actorId: user.id,
+        userId: user.id,
+        documentId,
+        ip,
+      },
+      { transaction }
+    );
+  }
+
+  return subscription;
+}

server/commands/subscriptionDestroyer.test.ts

@@ -0,0 +1,101 @@
+import { sequelize } from "@server/database/sequelize";
+import { Subscription, Event } from "@server/models";
+import {
+  buildDocument,
+  buildSubscription,
+  buildUser,
+} from "@server/test/factories";
+import { getTestDatabase } from "@server/test/support";
+import subscriptionDestroyer from "./subscriptionDestroyer";
+
+const db = getTestDatabase();
+
+beforeEach(db.flush);
+afterAll(db.disconnect);
+
+describe("subscriptionDestroyer", () => {
+  const ip = "127.0.0.1";
+
+  it("should destroy existing subscription", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription = await buildSubscription({
+      userId: user.id,
+      documentId: document.id,
+    });
+
+    await sequelize.transaction(
+      async (transaction) =>
+        await subscriptionDestroyer({
+          user,
+          subscription,
+          ip,
+          transaction,
+        })
+    );
+
+    const count = await Subscription.count();
+
+    expect(count).toEqual(0);
+
+    const event = await Event.findOne();
+
+    expect(event?.name).toEqual("subscriptions.delete");
+    expect(event?.modelId).toEqual(subscription.id);
+    expect(event?.actorId).toEqual(subscription.userId);
+    expect(event?.userId).toEqual(subscription.userId);
+    expect(event?.documentId).toEqual(subscription.documentId);
+  });
+
+  it("should soft delete row", async () => {
+    const user = await buildUser();
+
+    const document = await buildDocument({
+      userId: user.id,
+      teamId: user.teamId,
+    });
+
+    const subscription = await buildSubscription({
+      userId: user.id,
+      documentId: document.id,
+    });
+
+    await sequelize.transaction(
+      async (transaction) =>
+        await subscriptionDestroyer({
+          user,
+          subscription,
+          ip,
+          transaction,
+        })
+    );
+
+    const count = await Subscription.count();
+
+    expect(count).toEqual(0);
+
+    const event = await Event.findOne();
+
+    expect(event?.name).toEqual("subscriptions.delete");
+    expect(event?.modelId).toEqual(subscription.id);
+    expect(event?.actorId).toEqual(subscription.userId);
+    expect(event?.userId).toEqual(subscription.userId);
+    expect(event?.documentId).toEqual(subscription.documentId);
+
+    const deletedSubscription = await Subscription.findOne({
+      where: {
+        userId: user.id,
+        documentId: document.id,
+      },
+      paranoid: false,
+    });
+
+    expect(deletedSubscription).toBeDefined();
+    expect(deletedSubscription?.deletedAt).toBeDefined();
+  });
+});

server/commands/subscriptionDestroyer.ts

@@ -0,0 +1,41 @@
+import { Transaction } from "sequelize";
+import { Event, Subscription, User } from "@server/models";
+
+type Props = {
+  /** The user destroying the subscription */
+  user: User;
+  /** The subscription to destroy */
+  subscription: Subscription;
+  /** The IP address of the incoming request */
+  ip: string;
+  transaction: Transaction;
+};
+
+/**
+ * This command destroys a user subscription to a document so they will no
+ * longer receive notifications.
+ *
+ * @returns The subscription that was destroyed
+ */
+export default async function subscriptionDestroyer({
+  user,
+  subscription,
+  ip,
+  transaction,
+}: Props): Promise<Subscription> {
+  await subscription.destroy({ transaction });
+
+  await Event.create(
+    {
+      name: "subscriptions.delete",
+      modelId: subscription.id,
+      actorId: user.id,
+      userId: user.id,
+      documentId: subscription.documentId,
+      ip,
+    },
+    { transaction }
+  );
+
+  return subscription;
+}

server/commands/userSuspender.ts

@@ -10,6 +10,10 @@ type Props = {
   ip: string;
 };
 
+/**
+ * This command suspends an active user, this will cause them to lose access to
+ * the team.
+ */
 export default async function userSuspender({
   user,
   actorId,

server/commands/userUnsuspender.test.ts

@@ -0,0 +1,46 @@
+import { buildAdmin, buildUser } from "@server/test/factories";
+import { getTestDatabase } from "@server/test/support";
+import userUnsuspender from "./userUnsuspender";
+
+const db = getTestDatabase();
+
+afterAll(db.disconnect);
+
+beforeEach(db.flush);
+
+describe("userUnsuspender", () => {
+  const ip = "127.0.0.1";
+
+  it("should not allow unsuspending self", async () => {
+    const user = await buildUser();
+    let error;
+
+    try {
+      await userUnsuspender({
+        actorId: user.id,
+        user,
+        ip,
+      });
+    } catch (err) {
+      error = err;
+    }
+
+    expect(error.message).toEqual("Unable to unsuspend the current user");
+  });
+
+  it("should unsuspend the user", async () => {
+    const admin = await buildAdmin();
+    const user = await buildUser({
+      teamId: admin.teamId,
+      suspendedAt: new Date(),
+      suspendedById: admin.id,
+    });
+    await userUnsuspender({
+      actorId: admin.id,
+      user,
+      ip,
+    });
+    expect(user.suspendedAt).toEqual(null);
+    expect(user.suspendedById).toEqual(null);
+  });
+});

server/commands/userUnsuspender.ts

@@ -0,0 +1,49 @@
+import { Transaction } from "sequelize";
+import { sequelize } from "@server/database/sequelize";
+import { User, Event } from "@server/models";
+import { ValidationError } from "../errors";
+
+type Props = {
+  user: User;
+  actorId: string;
+  ip: string;
+};
+
+/**
+ * This command unsuspends a previously suspended user, allowing access to the
+ * team again.
+ */
+export default async function userUnsuspender({
+  user,
+  actorId,
+  ip,
+}: Props): Promise<void> {
+  if (user.id === actorId) {
+    throw ValidationError("Unable to unsuspend the current user");
+  }
+
+  await sequelize.transaction(async (transaction: Transaction) => {
+    await user.update(
+      {
+        suspendedById: null,
+        suspendedAt: null,
+      },
+      { transaction }
+    );
+    await Event.create(
+      {
+        name: "users.activate",
+        actorId,
+        userId: user.id,
+        teamId: user.teamId,
+        data: {
+          name: user.name,
+        },
+        ip,
+      },
+      {
+        transaction,
+      }
+    );
+  });
+}

server/editor/__snapshots__/renderToHtml.test.ts.snap

@@ -1,129 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`renders blockquote 1`] = `
-"<blockquote>
-<p>blockquote</p>
-</blockquote>"
-`;
-
-exports[`renders bold marks 1`] = `"<p>this is <strong>bold</strong> text</p>"`;
-
-exports[`renders bullet list 1`] = `
-"<ul>
-<li>item one</li>
-<li>item two
-<ul>
-<li>nested item</li>
-</ul>
-</li>
-</ul>"
-`;
-
-exports[`renders checkbox list 1`] = `
-"<ul>
-<li class=\\"checkbox-list-item\\"><span class=\\"checkbox \\">[ ]</span>unchecked</li>
-<li class=\\"checkbox-list-item\\"><span class=\\"checkbox checked\\">[x]</span>checked</li>
-</ul>"
-`;
-
-exports[`renders code block 1`] = `
-"<pre><code>this is indented code
-</code></pre>"
-`;
-
-exports[`renders code fence 1`] = `
-"<pre><code class=\\"language-javascript\\">this is code
-</code></pre>"
-`;
-
-exports[`renders code marks 1`] = `"<p>this is <code>inline code</code> text</p>"`;
-
-exports[`renders headings 1`] = `
-"<h1>Heading 1</h1>
-<h2>Heading 2</h2>
-<h3>Heading 3</h3>
-<h4>Heading 4</h4>"
-`;
-
-exports[`renders highlight marks 1`] = `"<p>this is <span class=\\"highlight\\">highlighted</span> text</p>"`;
-
-exports[`renders horizontal rule 1`] = `"<hr>"`;
-
-exports[`renders image 1`] = `"<p><img src=\\"https://lorempixel.com/200/200\\" alt=\\"caption\\"></p>"`;
-
-exports[`renders image with alignment 1`] = `"<p><img src=\\"https://lorempixel.com/200/200\\" alt=\\"caption\\" title=\\"left-40\\"></p>"`;
-
-exports[`renders info notice 1`] = `
-"<div class=\\"notice notice-info\\">
-<p>content of notice</p>
-</div>"
-`;
-
-exports[`renders italic marks 1`] = `"<p>this is <em>italic</em> text</p>"`;
-
-exports[`renders italic marks 2`] = `"<p>this is <em>also italic</em> text</p>"`;
-
-exports[`renders link marks 1`] = `"<p>this is <a href=\\"https://www.example.com\\">linked</a> text</p>"`;
-
-exports[`renders ordered list 1`] = `
-"<ol>
-<li>item one</li>
-<li>item two</li>
-</ol>"
-`;
-
-exports[`renders ordered list 2`] = `
-"<ol>
-<li>item one</li>
-<li>item two</li>
-</ol>"
-`;
-
-exports[`renders plain text as paragraph 1`] = `"<p>plain text</p>"`;
-
-exports[`renders table 1`] = `
-"<table>
-<tr>
-<th>
-<p>heading</p></th>
-<th style=\\"text-align:center\\">
-<p>centered</p></th>
-<th style=\\"text-align:right\\">
-<p>right aligned</p></th>
-</tr>
-<tr>
-<td>
-<p></p></td>
-<td style=\\"text-align:center\\">
-<p>center</p></td>
-<td style=\\"text-align:right\\">
-<p></p></td>
-</tr>
-<tr>
-<td>
-<p></p></td>
-<td style=\\"text-align:center\\">
-<p></p></td>
-<td style=\\"text-align:right\\">
-<p>bottom r</p></td>
-</tr>
-</table>"
-`;
-
-exports[`renders template placeholder marks 1`] = `"<p>this is <span class=\\"placeholder\\">a placeholder</span></p>"`;
-
-exports[`renders tip notice 1`] = `
-"<div class=\\"notice notice-tip\\">
-<p>content of notice</p>
-</div>"
-`;
-
-exports[`renders underline marks 1`] = `"<p>this is <underline>underlined</underline> text</p>"`;
-
-exports[`renders underline marks 2`] = `"<p>this is <s>strikethrough</s> text</p>"`;
-
-exports[`renders warning notice 1`] = `
-"<div class=\\"notice notice-warning\\">
-<p>content of notice</p>
-</div>"
-`;

server/editor/index.ts

@@ -1,7 +1,6 @@
 import { Schema } from "prosemirror-model";
 import ExtensionManager from "@shared/editor/lib/ExtensionManager";
 import fullPackage from "@shared/editor/packages/full";
-import render from "./renderToHtml";
 
 const extensions = new ExtensionManager(fullPackage);
 
@@ -16,6 +15,3 @@ export const parser = extensions.parser({
 });
 
 export const serializer = extensions.serializer();
-
-export const renderToHtml = (markdown: string): string =>
-  render(markdown, extensions.rulePlugins);

server/editor/renderToHtml.test.ts

@@ -1,154 +0,0 @@
-import renderToHtml from "./renderToHtml";
-
-test("renders an empty string", () => {
-  expect(renderToHtml("")).toBe("");
-});
-
-test("renders plain text as paragraph", () => {
-  expect(renderToHtml("plain text")).toMatchSnapshot();
-});
-
-test("renders blockquote", () => {
-  expect(renderToHtml("> blockquote")).toMatchSnapshot();
-});
-
-test("renders code block", () => {
-  expect(
-    renderToHtml(`
-    this is indented code
-`)
-  ).toMatchSnapshot();
-});
-
-test("renders code fence", () => {
-  expect(
-    renderToHtml(`\`\`\`javascript
-this is code
-\`\`\``)
-  ).toMatchSnapshot();
-});
-
-test("renders checkbox list", () => {
-  expect(
-    renderToHtml(`- [ ] unchecked
-- [x] checked`)
-  ).toMatchSnapshot();
-});
-
-test("renders bullet list", () => {
-  expect(
-    renderToHtml(`- item one
-- item two
-  - nested item`)
-  ).toMatchSnapshot();
-});
-
-test("renders info notice", () => {
-  expect(
-    renderToHtml(`:::info
-content of notice
-:::`)
-  ).toMatchSnapshot();
-});
-
-test("renders warning notice", () => {
-  expect(
-    renderToHtml(`:::warning
-content of notice
-:::`)
-  ).toMatchSnapshot();
-});
-
-test("renders tip notice", () => {
-  expect(
-    renderToHtml(`:::tip
-content of notice
-:::`)
-  ).toMatchSnapshot();
-});
-
-test("renders headings", () => {
-  expect(
-    renderToHtml(`# Heading 1
-
-## Heading 2
-
-### Heading 3
-
-#### Heading 4`)
-  ).toMatchSnapshot();
-});
-
-test("renders horizontal rule", () => {
-  expect(renderToHtml(`---`)).toMatchSnapshot();
-});
-
-test("renders image", () => {
-  expect(
-    renderToHtml(`![caption](https://lorempixel.com/200/200)`)
-  ).toMatchSnapshot();
-});
-
-test("renders image with alignment", () => {
-  expect(
-    renderToHtml(`![caption](https://lorempixel.com/200/200 "left-40")`)
-  ).toMatchSnapshot();
-});
-
-test("renders table", () => {
-  expect(
-    renderToHtml(`
-| heading | centered | right aligned |
-|---------|:--------:|--------------:|
-|         | center   |               |
-|         |          |      bottom r |
-`)
-  ).toMatchSnapshot();
-});
-
-test("renders bold marks", () => {
-  expect(renderToHtml(`this is **bold** text`)).toMatchSnapshot();
-});
-
-test("renders code marks", () => {
-  expect(renderToHtml(`this is \`inline code\` text`)).toMatchSnapshot();
-});
-
-test("renders highlight marks", () => {
-  expect(renderToHtml(`this is ==highlighted== text`)).toMatchSnapshot();
-});
-
-test("renders italic marks", () => {
-  expect(renderToHtml(`this is *italic* text`)).toMatchSnapshot();
-  expect(renderToHtml(`this is _also italic_ text`)).toMatchSnapshot();
-});
-
-test("renders template placeholder marks", () => {
-  expect(renderToHtml(`this is !!a placeholder!!`)).toMatchSnapshot();
-});
-
-test("renders underline marks", () => {
-  expect(renderToHtml(`this is __underlined__ text`)).toMatchSnapshot();
-});
-
-test("renders link marks", () => {
-  expect(
-    renderToHtml(`this is [linked](https://www.example.com) text`)
-  ).toMatchSnapshot();
-});
-
-test("renders underline marks", () => {
-  expect(renderToHtml(`this is ~~strikethrough~~ text`)).toMatchSnapshot();
-});
-
-test("renders ordered list", () => {
-  expect(
-    renderToHtml(`1. item one
-1. item two`)
-  ).toMatchSnapshot();
-
-  expect(
-    renderToHtml(`1. item one
-2. item two`)
-  ).toMatchSnapshot();
-});

server/editor/renderToHtml.ts

@@ -1,31 +0,0 @@
-import { PluginSimple } from "markdown-it";
-import createMarkdown from "@shared/editor/lib/markdown/rules";
-import attachmentsRule from "@shared/editor/rules/attachments";
-import breakRule from "@shared/editor/rules/breaks";
-import checkboxRule from "@shared/editor/rules/checkboxes";
-import embedsRule from "@shared/editor/rules/embeds";
-import emojiRule from "@shared/editor/rules/emoji";
-import markRule from "@shared/editor/rules/mark";
-import noticesRule from "@shared/editor/rules/notices";
-import tablesRule from "@shared/editor/rules/tables";
-import underlinesRule from "@shared/editor/rules/underlines";
-
-const defaultRules = [
-  embedsRule([]),
-  breakRule,
-  checkboxRule,
-  markRule({ delim: "==", mark: "highlight" }),
-  markRule({ delim: "!!", mark: "placeholder" }),
-  underlinesRule,
-  tablesRule,
-  noticesRule,
-  attachmentsRule,
-  emojiRule,
-];
-
-export default function renderToHtml(
-  markdown: string,
-  rulePlugins: PluginSimple[] = defaultRules
-): string {
-  return createMarkdown({ plugins: rulePlugins }).render(markdown).trim();
-}

server/emails/mailer.tsx

@@ -5,6 +5,7 @@ import Logger from "@server/logging/Logger";
 import { APM } from "@server/logging/tracing";
 import { baseStyles } from "./templates/components/EmailLayout";
 
+const isCloudHosted = env.DEPLOYMENT === "hosted";
 const useTestEmailService =
   env.ENVIRONMENT === "development" && !env.SMTP_USERNAME;
 
@@ -77,6 +78,15 @@ export class Mailer {
         subject: data.subject,
         html,
         text: data.text,
+        attachments: isCloudHosted
+          ? undefined
+          : [
+              {
+                filename: "header-logo.png",
+                path: process.cwd() + "/public/email/header-logo.png",
+                cid: "header-image",
+              },
+            ],
       });
 
       if (useTestEmailService) {

server/emails/templates/components/Header.tsx

@@ -4,6 +4,7 @@ import env from "@server/env";
 import EmptySpace from "./EmptySpace";
 
 const url = env.CDN_URL ?? env.URL;
+const isCloudHosted = env.DEPLOYMENT === "hosted";
 
 export default () => {
   return (
@@ -14,7 +15,11 @@ export default () => {
             <EmptySpace height={40} />
             <img
               alt="Outline"
-              src={`${url}/email/header-logo.png`}
+              src={
+                isCloudHosted
+                  ? `${url}/email/header-logo.png`
+                  : "cid:header-image"
+              }
               height="48"
               width="48"
             />

server/env.ts

@@ -22,6 +22,7 @@ import {
 import { languages } from "@shared/i18n";
 import { CannotUseWithout } from "@server/utils/validators";
 import Deprecated from "./models/decorators/Deprecated";
+import { getArg } from "./utils/args";
 
 export class Environment {
   private validationPromise;
@@ -166,6 +167,15 @@ export class Environment {
   @IsOptional()
   public WEB_CONCURRENCY = this.toOptionalNumber(process.env.WEB_CONCURRENCY);
 
+  /**
+   * How long a request should be processed before giving up and returning an
+   * error response to the client, defaults to 10s
+   */
+  @IsNumber()
+  @IsOptional()
+  public REQUEST_TIMEOUT =
+    this.toOptionalNumber(process.env.REQUEST_TIMEOUT) ?? 10 * 1000;
+
   /**
    * Base64 encoded private key if Outline is to perform SSL termination.
    */
@@ -202,9 +212,14 @@ export class Environment {
   /**
    * A comma separated list of which services should be enabled on this
    * instance – defaults to all.
+   *
+   * If a services flag is passed it takes priority over the environment variable
+   * for example: --services=web,worker
    */
   public SERVICES =
-    process.env.SERVICES ?? "collaboration,websockets,worker,web";
+    getArg("services") ??
+    process.env.SERVICES ??
+    "collaboration,websockets,worker,web";
 
   /**
    * Auto-redirect to https in production. The default is true but you may set
@@ -496,8 +511,7 @@ export class Environment {
   );
 
   /**
-   * A boolean switch to toggle the rate limiter
-   * at application web server.
+   * A boolean switch to toggle the rate limiter at application web server.
    */
   @IsOptional()
   @IsBoolean()
@@ -506,19 +520,18 @@ export class Environment {
   );
 
   /**
-   * Set max allowed requests in a given duration for
-   * default rate limiter to trigger throttling.
+   * Set max allowed requests in a given duration for default rate limiter to
+   * trigger throttling, per IP address.
    */
   @IsOptional()
   @IsNumber()
   @CannotUseWithout("RATE_LIMITER_ENABLED")
   public RATE_LIMITER_REQUESTS =
-    this.toOptionalNumber(process.env.RATE_LIMITER_REQUESTS) ?? 5000;
+    this.toOptionalNumber(process.env.RATE_LIMITER_REQUESTS) ?? 1000;
 
   /**
-   * Set fixed duration window(in secs) for
-   * default rate limiter, elapsing which the request
-   * quota is reset(the bucket is refilled with tokens).
+   * Set fixed duration window(in secs) for default rate limiter, elapsing which
+   * the request quota is reset (the bucket is refilled with tokens).
    */
   @IsOptional()
   @IsNumber()

server/errors.ts

@@ -1,5 +1,11 @@
 import httpErrors from "http-errors";
 
+export function InternalError(message = "Internal error") {
+  return httpErrors(500, message, {
+    id: "internal_error",
+  });
+}
+
 export function AuthenticationError(
   message = "Authentication required",
   redirectUrl = "/"

server/index.ts

@@ -27,16 +27,10 @@ import {
 } from "./utils/startup";
 import { checkUpdates } from "./utils/updates";
 
-// If a services flag is passed it takes priority over the environment variable
-// for example: --services=web,worker
-const normalizedServiceFlag = getArg("services");
-
 // The default is to run all services to make development and OSS installations
 // easier to deal with. Separate services are only needed at scale.
 const serviceNames = uniq(
-  (normalizedServiceFlag || env.SERVICES)
-    .split(",")
-    .map((service) => service.trim())
+  env.SERVICES.split(",").map((service) => service.trim())
 );
 
 // The number of processes to run, defaults to the number of CPU's available
@@ -124,6 +118,8 @@ async function start(id: number, disconnect: () => void) {
   });
 
   server.listen(normalizedPortFlag || env.PORT || "3000");
+  server.setTimeout(env.REQUEST_TIMEOUT);
+
   process.once("SIGTERM", shutdown);
   process.once("SIGINT", shutdown);