chore: Move IndexedDB persistence into codebase

app/scenes/Document/components/MultiplayerEditor.tsx

@@ -3,8 +3,9 @@ import { throttle } from "lodash";
 import * as React from "react";
 import { useTranslation } from "react-i18next";
 import { useHistory } from "react-router-dom";
-import { IndexeddbPersistence } from "y-indexeddb";
 import * as Y from "yjs";
+import { IndexeddbPersistence } from "@shared/editor/lib/IndexeddbPersistence";
+import Multiplayer from "@shared/editor/plugins/Multiplayer";
 import Editor, { Props as EditorProps } from "~/components/Editor";
 import env from "~/env";
 import useCurrentToken from "~/hooks/useCurrentToken";
@@ -14,7 +15,6 @@ import useIsMounted from "~/hooks/useIsMounted";
 import usePageVisibility from "~/hooks/usePageVisibility";
 import useStores from "~/hooks/useStores";
 import useToasts from "~/hooks/useToasts";
-import MultiplayerExtension from "~/multiplayer/MultiplayerExtension";
 import Logger from "~/utils/Logger";
 import { supportsPassiveListener } from "~/utils/browser";
 import { homePath } from "~/utils/routeHelpers";
@@ -200,7 +200,7 @@ function MultiplayerEditor({ onSynced, ...props }: Props, ref: any) {
 
     return [
       ...(props.extensions || []),
-      new MultiplayerExtension({
+      new Multiplayer({
         user,
         provider: remoteProvider,
         document: ydoc,

package.json

@@ -39,7 +39,7 @@
     "url": "git+ssh://git@github.com/outline/outline.git"
   },
   "browserslist": [
-    "> 0.25%, not dead"  
+    "> 0.25%, not dead"
   ],
   "dependencies": {
     "@babel/core": "^7.16.0",
@@ -118,6 +118,7 @@
     "koa-send": "5.0.1",
     "koa-sslify": "2.1.2",
     "koa-static": "^4.0.1",
+    "lib0": "^0.2.35",
     "lodash": "^4.17.21",
     "mammoth": "^1.4.19",
     "markdown-it": "^12.3.2",
@@ -203,7 +204,6 @@
     "validator": "13.7.0",
     "winston": "^3.3.3",
     "ws": "^7.5.3",
-    "y-indexeddb": "^9.0.6",
     "yjs": "^13.5.34"
   },
   "devDependencies": {

shared/editor/lib/IndexeddbPersistence.ts

@@ -0,0 +1,180 @@
+// Based on https://github.com/yjs/y-indexeddb/commit/3a52367c486c9f2b166c2fc0f83fe1a7d196a0fc
+
+import * as idb from "lib0/indexeddb";
+import * as mutex from "lib0/mutex";
+import { Observable } from "lib0/observable";
+import * as Y from "yjs";
+
+const metaStoreName = "metadata";
+const updatesStoreName = "updates";
+
+export const PREFERRED_TRIM_SIZE = 500;
+
+export class IndexeddbPersistence extends Observable<string> {
+  public db: IDBDatabase | null = null;
+
+  public doc: Y.Doc;
+
+  public name: string;
+
+  public synced = false;
+
+  public whenSynced: Promise<void | this>;
+
+  constructor(name: string, doc: Y.Doc) {
+    super();
+    this.doc = doc;
+    this.name = name;
+    this._db = idb.openDB(name, (db) =>
+      idb.createStores(db, [["updates", { autoIncrement: true }], ["meta"]])
+    );
+
+    this.whenSynced = this._db.then(async (db) => {
+      this.db = db;
+      const currState = Y.encodeStateAsUpdate(doc);
+      return this.fetchUpdates()
+        .then((updatesStore) => idb.addAutoKey(updatesStore, currState))
+        .then(() => {
+          this.emit("synced", [this]);
+          this.synced = true;
+          return this;
+        });
+    });
+
+    this._storeUpdate = (update: Uint8Array) =>
+      this._mux(() => {
+        if (this.db) {
+          const [updatesStore] = idb.transact(this.db, [updatesStoreName]);
+          idb.addAutoKey(updatesStore, update);
+          if (++this._dbsize >= PREFERRED_TRIM_SIZE) {
+            // debounce store call
+            if (this._storeTimeoutId !== null) {
+              clearTimeout(this._storeTimeoutId);
+            }
+            this._storeTimeoutId = setTimeout(() => {
+              this.storeState(false);
+              this._storeTimeoutId = null;
+            }, this._storeTimeout);
+          }
+        }
+      });
+
+    doc.on("update", this._storeUpdate);
+    doc.on("destroy", this.destroy);
+  }
+
+  private fetchUpdates = async () => {
+    if (!this.db) {
+      throw new Error("fetchUpdates called before db initialized");
+    }
+
+    const [updatesStore] = idb.transact(this.db, [updatesStoreName]);
+
+    return idb
+      .getAll(updatesStore, idb.createIDBKeyRangeLowerBound(this._dbref, false))
+      .then((updates) =>
+        this._mux(() =>
+          Y.transact(
+            this.doc,
+            () => {
+              updates.forEach((val) => Y.applyUpdate(this.doc, val));
+            },
+            this,
+            false
+          )
+        )
+      )
+      .then(() =>
+        idb.getLastKey(updatesStore).then((lastKey) => {
+          this._dbref = lastKey + 1;
+        })
+      )
+      .then(() =>
+        idb.count(updatesStore).then((cnt) => {
+          this._dbsize = cnt;
+        })
+      )
+      .then(() => updatesStore);
+  };
+
+  private storeState = (forceStore = true) =>
+    this.fetchUpdates().then((updatesStore) => {
+      if (forceStore || this._dbsize >= PREFERRED_TRIM_SIZE) {
+        idb
+          .addAutoKey(updatesStore, Y.encodeStateAsUpdate(this.doc))
+          .then(() =>
+            idb.del(
+              updatesStore,
+              idb.createIDBKeyRangeUpperBound(this._dbref, true)
+            )
+          )
+          .then(() =>
+            idb.count(updatesStore).then((cnt) => {
+              this._dbsize = cnt;
+            })
+          );
+      }
+    });
+
+  destroy = async () => {
+    if (this._storeTimeoutId) {
+      clearTimeout(this._storeTimeoutId);
+    }
+    this.doc.off("update", this._storeUpdate);
+    this.doc.off("destroy", this.destroy);
+    return this._db.then((db) => {
+      db.close();
+    });
+  };
+
+  /**
+   * Destroys this instance and removes all data from indexeddb.
+   */
+  clearData = async (): Promise<void> => {
+    return this.destroy().then(() => {
+      idb.deleteDB(this.name);
+    });
+  };
+
+  get = async (
+    key: string | number | ArrayBuffer | Date
+  ): Promise<string | number | ArrayBuffer | Date | any> => {
+    return this._db.then((db) => {
+      const [meta] = idb.transact(db, [metaStoreName], "readonly");
+      return idb.get(meta, key);
+    });
+  };
+
+  set = async (
+    key: string | number | ArrayBuffer | Date,
+    value: string | number | ArrayBuffer | Date
+  ): Promise<string | number | ArrayBuffer | Date> => {
+    return this._db.then((db) => {
+      const [meta] = idb.transact(db, [metaStoreName]);
+      return idb.put(meta, value, key);
+    });
+  };
+
+  del = async (
+    key: string | number | ArrayBuffer | Date
+  ): Promise<undefined> => {
+    return this._db.then((db) => {
+      const [meta] = idb.transact(db, [metaStoreName]);
+      return idb.del(meta, key);
+    });
+  };
+
+  /**
+   * Timeout in ms until data is merged and persisted in idb.
+   */
+  private _storeTimeout = 1000;
+
+  private _storeTimeoutId: NodeJS.Timeout | null = null;
+
+  private _storeUpdate: (update: Uint8Array) => Promise<void>;
+
+  private _mux = mutex.createMutex();
+  private _dbsize = 0;
+  private _dbref = 0;
+  private _db: Promise<IDBDatabase>;
+}

shared/editor/plugins/Multiplayer.ts

@@ -7,9 +7,9 @@ import {
 } from "@getoutline/y-prosemirror";
 import { keymap } from "prosemirror-keymap";
 import * as Y from "yjs";
-import { Extension } from "~/editor";
+import Extension from "../lib/Extension";
 
-export default class MultiplayerExtension extends Extension {
+export default class Multiplayer extends Extension {
   get name() {
     return "multiplayer";
   }

yarn.lock

@@ -15716,13 +15716,6 @@ xtend@^4.0.0, xtend@~4.0.0, xtend@~4.0.1:
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
 
-y-indexeddb@^9.0.6:
-  version "9.0.6"
-  resolved "https://registry.yarnpkg.com/y-indexeddb/-/y-indexeddb-9.0.6.tgz#49aecac11bc229571fb134e0ec0717c0330b731f"
-  integrity sha512-8mdCYdzZDWS2lGiB9Reaz67ZqvnV6EXH/F7L+TmBC+3mWjIBrPw4UcI79nOhEOh+y9lHXzNpSda4YJ06M13F1A==
-  dependencies:
-    lib0 "^0.2.35"
-
 y-protocols@^1.0.5:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/y-protocols/-/y-protocols-1.0.5.tgz#91d574250060b29fcac8f8eb5e276fbad594245e"