wip: Pull work across from previous branch

.flowconfig

@@ -11,6 +11,10 @@
 .*/node_modules/react-side-effect/.*
 .*/node_modules/fbjs/.*
 .*/node_modules/config-chain/.*
+.*/node_modules/yjs/.*
+.*/node_modules/y-prosemirror/.*
+.*/node_modules/y-protocols/.*
+.*/node_modules/lib0/.*
 .*/server/scripts/.*
 *.test.js
 

app/components/Editor.js

@@ -246,6 +246,50 @@ const StyledEditor = styled(RichMarkdownEditor)`
       }
     }
   }
+
+  .ProseMirror {
+    .ProseMirror-yjs-cursor {
+      position: relative;
+      margin-left: -1px;
+      margin-right: -1px;
+      border-left: 1px solid black;
+      border-right: 1px solid black;
+      height: 1em;
+      word-break: normal;
+      &:after {
+        content: "";
+        display: block;
+        position: absolute;
+        left: -8px;
+        right: -8px;
+        top: 0;
+        bottom: 0;
+      }
+      > div {
+        opacity: 0;
+        position: absolute;
+        top: -1.8em;
+        font-size: 13px;
+        background-color: rgb(250, 129, 0);
+        font-style: normal;
+        line-height: normal;
+        user-select: none;
+        white-space: nowrap;
+        color: white;
+        padding: 2px 6px;
+        font-weight: 500;
+        border-radius: 4px;
+        pointer-events: none;
+        left: -1px;
+      }
+      &:hover {
+        > div {
+          opacity: 1;
+          transition: opacity 100ms ease-in-out;
+        }
+      }
+    }
+  }
 `;
 
 const EditorTooltip = ({ children, ...props }) => (

app/models/Document.js

@@ -261,6 +261,27 @@ export default class Document extends BaseModel {
     this.injectTemplate = true;
   };
 
+  @action
+  update = async (options: SaveOptions & { title: string }) => {
+    if (this.isSaving) return this;
+    this.isSaving = true;
+
+    try {
+      if (options.lastRevision) {
+        return await this.store.update({
+          id: this.id,
+          title: this.title,
+          lastRevision: options.lastRevision,
+          ...options,
+        });
+      }
+
+      throw new Error("Attempting to update without a lastRevision");
+    } finally {
+      this.isSaving = false;
+    }
+  };
+
   @action
   save = async (options: SaveOptions = {}) => {
     if (this.isSaving) return this;

app/models/Team.js

@@ -7,6 +7,7 @@ class Team extends BaseModel {
   name: string;
   avatarUrl: string;
   sharing: boolean;
+  multiplayerEditor: boolean;
   documentEmbeds: boolean;
   guestSignin: boolean;
   subdomain: ?string;

app/multiplayer/MultiplayerExtension.js

@@ -0,0 +1,55 @@
+// @flow
+import { keymap } from "prosemirror-keymap";
+import { Extension } from "rich-markdown-editor";
+import {
+  ySyncPlugin,
+  yCursorPlugin,
+  yUndoPlugin,
+  undo,
+  redo,
+} from "y-prosemirror";
+import * as Y from "yjs";
+
+export default class MultiplayerExtension extends Extension {
+  get name() {
+    return "multiplayer";
+  }
+
+  get plugins() {
+    const { user, provider, document: doc } = this.options;
+    const type = doc.get("prosemirror", Y.XmlFragment);
+
+    const assignUser = (tr) => {
+      const clientIds = Array.from(doc.store.clients.keys());
+
+      if (
+        tr.local &&
+        tr.changed.size > 0 &&
+        !clientIds.includes(doc.clientID)
+      ) {
+        const permanentUserData = new Y.PermanentUserData(doc);
+        permanentUserData.setUserMapping(doc, doc.clientID, user.id);
+        doc.off("afterTransaction", assignUser);
+      }
+    };
+
+    provider.awareness.setLocalStateField("user", {
+      color: user.color,
+      name: user.name,
+      id: user.id,
+    });
+
+    doc.on("afterTransaction", assignUser);
+
+    return [
+      ySyncPlugin(type),
+      yCursorPlugin(provider.awareness),
+      yUndoPlugin(),
+      keymap({
+        "Mod-z": undo,
+        "Mod-y": redo,
+        "Mod-Shift-z": redo,
+      }),
+    ];
+  }
+}

app/routes/authenticated.js

@@ -22,10 +22,8 @@ import { matchDocumentSlug as slug } from "utils/routeHelpers";
 const SettingsRoutes = React.lazy(() =>
   import(/* webpackChunkName: "settings" */ "./settings")
 );
-const KeyedDocument = React.lazy(() =>
-  import(
-    /* webpackChunkName: "keyed-document" */ "scenes/Document/KeyedDocument"
-  )
+const Document = React.lazy(() =>
+  import(/* webpackChunkName: "document" */ "scenes/Document")
 );
 const NotFound = () => <Search notFound />;
 const RedirectDocument = ({ match }: { match: Match }) => (
@@ -66,10 +64,10 @@ export default function AuthenticatedRoutes() {
             <Route
               exact
               path={`/doc/${slug}/history/:revisionId?`}
-              component={KeyedDocument}
+              component={Document}
             />
-            <Route exact path={`/doc/${slug}/edit`} component={KeyedDocument} />
-            <Route path={`/doc/${slug}`} component={KeyedDocument} />
+            <Route exact path={`/doc/${slug}/edit`} component={Document} />
+            <Route path={`/doc/${slug}`} component={Document} />
             <Route exact path="/search" component={Search} />
             <Route exact path="/search/:term" component={Search} />
             <Route path="/404" component={Error404} />

app/routes/index.js

@@ -12,10 +12,8 @@ const Authenticated = React.lazy(() =>
 const AuthenticatedRoutes = React.lazy(() =>
   import(/* webpackChunkName: "authenticated-routes" */ "./authenticated")
 );
-const KeyedDocument = React.lazy(() =>
-  import(
-    /* webpackChunkName: "keyed-document" */ "scenes/Document/KeyedDocument"
-  )
+const SharedDocument = React.lazy(() =>
+  import(/* webpackChunkName: "shared-document" */ "scenes/Document/Shared")
 );
 const Login = React.lazy(() =>
   import(/* webpackChunkName: "login" */ "scenes/Login")
@@ -37,11 +35,11 @@ export default function Routes() {
         <Route exact path="/" component={Login} />
         <Route exact path="/create" component={Login} />
         <Route exact path="/logout" component={Logout} />
-        <Route exact path="/share/:shareId" component={KeyedDocument} />
+        <Route exact path="/share/:shareId" component={SharedDocument} />
         <Route
           exact
           path={`/share/:shareId/doc/${slug}`}
-          component={KeyedDocument}
+          component={SharedDocument}
         />
         <Authenticated>
           <AuthenticatedRoutes />

app/scenes/Document/KeyedDocument.js

@@ -1,25 +0,0 @@
-// @flow
-import { inject } from "mobx-react";
-import * as React from "react";
-import DataLoader from "./components/DataLoader";
-
-class KeyedDocument extends React.Component<*> {
-  componentWillUnmount() {
-    this.props.ui.clearActiveDocument();
-  }
-
-  render() {
-    const { documentSlug, revisionId } = this.props.match.params;
-
-    // the urlId portion of the url does not include the slugified title
-    // we only want to force a re-mount of the document component when the
-    // document changes, not when the title does so only this portion is used
-    // for the key.
-    const urlParts = documentSlug ? documentSlug.split("-") : [];
-    const urlId = urlParts.length ? urlParts[urlParts.length - 1] : undefined;
-
-    return <DataLoader key={[urlId, revisionId].join("/")} {...this.props} />;
-  }
-}
-
-export default inject("ui")(KeyedDocument);

app/scenes/Document/Shared.js

@@ -0,0 +1,61 @@
+// @flow
+import * as React from "react";
+import { type Match } from "react-router-dom";
+import { useTheme } from "styled-components";
+import Error404 from "scenes/Error404";
+import ErrorOffline from "scenes/ErrorOffline";
+import useStores from "../../hooks/useStores";
+import Document from "./components/Document";
+import Loading from "./components/Loading";
+import { type LocationWithState } from "types";
+import { OfflineError } from "utils/errors";
+
+type Props = {|
+  match: Match,
+  location: LocationWithState,
+|};
+
+export default function SharedDocumentScene(props: Props) {
+  const theme = useTheme();
+  const [response, setResponse] = React.useState();
+  const [error, setError] = React.useState<?Error>();
+  const { documents } = useStores();
+  const { shareId, documentSlug } = props.match.params;
+
+  // ensure the wider page color always matches the theme
+  React.useEffect(() => {
+    window.document.body.style.background = theme.background;
+  }, [theme]);
+
+  React.useEffect(() => {
+    async function fetchData() {
+      try {
+        const response = await documents.fetch(documentSlug, {
+          shareId,
+        });
+        setResponse(response);
+      } catch (err) {
+        setError(err);
+      }
+    }
+    fetchData();
+  }, [documents, documentSlug, shareId]);
+
+  if (error) {
+    return error instanceof OfflineError ? <ErrorOffline /> : <Error404 />;
+  }
+
+  if (!response) {
+    return <Loading location={props.location} />;
+  }
+
+  return (
+    <Document
+      document={response.document}
+      sharedTree={response.sharedTree}
+      location={props.location}
+      shareId={shareId}
+      readOnly
+    />
+  );
+}

app/scenes/Document/components/DataLoader.js

@@ -18,16 +18,15 @@ import Document from "models/Document";
 import Revision from "models/Revision";
 import Error404 from "scenes/Error404";
 import ErrorOffline from "scenes/ErrorOffline";
-import DocumentComponent from "./Document";
 import HideSidebar from "./HideSidebar";
 import Loading from "./Loading";
-import SocketPresence from "./SocketPresence";
 import { type LocationWithState, type NavigationNode } from "types";
 import { NotFoundError, OfflineError } from "utils/errors";
 import { matchDocumentEdit, updateDocumentUrl } from "utils/routeHelpers";
 import { isInternalUrl } from "utils/urls";
 type Props = {|
   match: Match,
+  auth: AuthStore,
   location: LocationWithState,
   shares: SharesStore,
   documents: DocumentsStore,
@@ -36,6 +35,7 @@ type Props = {|
   auth: AuthStore,
   ui: UiStore,
   history: RouterHistory,
+  children: (any) => React.Node,
 |};
 
 const sharedTreeCache = {};
@@ -223,7 +223,7 @@ class DataLoader extends React.Component<Props> {
   };
 
   render() {
-    const { location, policies, ui } = this.props;
+    const { location, policies, auth, ui } = this.props;
 
     if (this.error) {
       return this.error instanceof OfflineError ? (
@@ -233,10 +233,11 @@ class DataLoader extends React.Component<Props> {
       );
     }
 
+    const team = auth.team;
     const document = this.document;
     const revision = this.revision;
 
-    if (!document) {
+    if (!document || !team) {
       return (
         <>
           <Loading location={location} />
@@ -247,20 +248,26 @@ class DataLoader extends React.Component<Props> {
 
     const abilities = policies.abilities(document.id);
 
+    const key = team.multiplayerEditor
+      ? ""
+      : this.isEditing
+      ? "editing"
+      : "read-only";
+
     return (
-      <SocketPresence documentId={document.id} isEditing={this.isEditing}>
+      <React.Fragment key={key}>
         {this.isEditing && <HideSidebar ui={ui} />}
-        <DocumentComponent
-          document={document}
-          revision={revision}
-          abilities={abilities}
-          location={location}
-          readOnly={!this.isEditing || !abilities.update || document.isArchived}
-          onSearchLink={this.onSearchLink}
-          onCreateLink={this.onCreateLink}
-          sharedTree={this.sharedTree}
-        />
-      </SocketPresence>
+        {this.props.children({
+          document,
+          revision,
+          abilities,
+          isEditing: this.isEditing,
+          readOnly: !this.isEditing || !abilities.update || document.isArchived,
+          onSearchLink: this.onSearchLink,
+          onCreateLink: this.onCreateLink,
+          sharedTree: this.sharedTree,
+        })}
+      </React.Fragment>
     );
   }
 }

app/scenes/Document/components/Document.js

@@ -197,7 +197,7 @@ class DocumentScene extends React.Component<Props> {
       autosave?: boolean,
     } = {}
   ) => {
-    const { document } = this.props;
+    const { document, auth } = this.props;
 
     // prevent saves when we are already saving
     if (document.isSaving) return;
@@ -227,10 +227,22 @@ class DocumentScene extends React.Component<Props> {
     document.tasks = getTasks(document.text);
 
     try {
-      const savedDocument = await document.save({
-        ...options,
-        lastRevision: this.lastRevision,
-      });
+      let savedDocument = document;
+      if (auth.team && auth.team.multiplayerEditor) {
+        // update does not send "text" field to the API, this is a workaround
+        // while the multiplayer editor is toggleable. Once it's finalized
+        // this can be cleaned up to single code path
+        savedDocument = await document.update({
+          ...options,
+          lastRevision: this.lastRevision,
+        });
+      } else {
+        savedDocument = await document.save({
+          ...options,
+          lastRevision: this.lastRevision,
+        });
+      }
+
       this.isDirty = false;
       this.lastRevision = savedDocument.revision;
 
@@ -275,6 +287,11 @@ class DocumentScene extends React.Component<Props> {
   };
 
   onChange = (getEditorText) => {
+    const { auth } = this.props;
+    if (auth.team?.multiplayerEditor) {
+      return;
+    }
+
     this.getEditorText = getEditorText;
 
     // document change while read only is presumed to be a checkbox edit,
@@ -332,7 +349,7 @@ class DocumentScene extends React.Component<Props> {
           auto
         >
           <Route
-            path={`${match.url}/move`}
+            path={`${document.url}/move`}
             component={() => (
               <Modal
                 title={`Move ${document.noun}`}
@@ -356,7 +373,11 @@ class DocumentScene extends React.Component<Props> {
             {!readOnly && (
               <>
                 <Prompt
-                  when={this.isDirty && !this.isUploading}
+                  when={
+                    this.isDirty &&
+                    !this.isUploading &&
+                    !team?.multiplayerEditor
+                  }
                   message={t(
                     `You have unsaved changes.\nAre you sure you want to discard them?`
                   )}
@@ -444,6 +465,7 @@ class DocumentScene extends React.Component<Props> {
                   <Editor
                     id={document.id}
                     innerRef={this.editor}
+                    multiplayer={team?.multiplayerEditor}
                     shareId={shareId}
                     isDraft={document.isDraft}
                     template={document.isTemplate}

app/scenes/Document/components/Editor.js

@@ -17,6 +17,7 @@ import Editor, { type Props as EditorProps } from "components/Editor";
 import Flex from "components/Flex";
 import HoverPreview from "components/HoverPreview";
 import Star, { AnimatedStar } from "components/Star";
+import MultiplayerEditor from "./MultiplayerEditor";
 import { isModKey } from "utils/keyboard";
 import { documentHistoryUrl } from "utils/routeHelpers";
 
@@ -27,6 +28,7 @@ type Props = {|
   document: Document,
   isDraft: boolean,
   shareId: ?string,
+  multiplayer?: boolean,
   onSave: ({ done?: boolean, autosave?: boolean, publish?: boolean }) => any,
   innerRef: { current: any },
   children: React.Node,
@@ -107,10 +109,12 @@ class DocumentEditor extends React.Component<Props> {
       innerRef,
       children,
       policies,
+      multiplayer,
       t,
       ...rest
     } = this.props;
 
+    const EditorComponent = multiplayer ? MultiplayerEditor : Editor;
     const can = policies.abilities(document.id);
     const { emoji } = parseTitle(title);
     const startsWithEmojiAndSpace = !!(emoji && title.startsWith(`${emoji} `));
@@ -162,7 +166,7 @@ class DocumentEditor extends React.Component<Props> {
             }
           />
         )}
-        <Editor
+        <EditorComponent
           ref={innerRef}
           autoFocus={!!title && !this.props.defaultValue}
           placeholder={t("…the rest is up to you")}

app/scenes/Document/components/MultiplayerEditor.js

@@ -0,0 +1,65 @@
+// @flow
+import { HocuspocusProvider } from "@hocuspocus/provider";
+import * as React from "react";
+import * as Y from "yjs";
+import Editor from "components/Editor";
+import env from "env";
+import useCurrentUser from "hooks/useCurrentUser";
+import MultiplayerExtension from "multiplayer/MultiplayerExtension";
+
+// TODO: typing
+export default function MultiplayerEditor(props: any) {
+  const user = useCurrentUser();
+
+  // TODO
+  //const [showCachedDocument, setShowCachedDocument] = React.useState(true);
+
+  // React.useEffect(() => {
+  //   if (isRemoteSynced) {
+  //     setTimeout(() => setShowCachedDocument(false), 100);
+  //   }
+  // }, [showCachedDocument, isRemoteSynced]);
+
+  const extensions = React.useMemo(() => {
+    console.log("extensions");
+
+    const ydoc = new Y.Doc();
+    const provider = new HocuspocusProvider({
+      url: env.MULTIPLAYER_URL,
+
+      // TODO: pipe documentId
+      name: "example-document",
+      document: ydoc,
+    });
+
+    return [
+      new MultiplayerExtension({
+        user,
+        provider,
+        document: ydoc,
+      }),
+    ];
+  }, [user]);
+
+  return (
+    <span style={{ position: "relative" }}>
+      {true && (
+        <Editor
+          {...props}
+          key="multiplayer"
+          defaultValue={undefined}
+          value={undefined}
+          extensions={extensions}
+          style={{ position: "absolute", width: "100%" }}
+        />
+      )}
+      {/* {showCachedDocument && (
+        <Editor
+          {...props}
+          style={{ position: "absolute", width: "100%" }}
+          readOnly
+        />
+      )} */}
+    </span>
+  );
+}

app/scenes/Document/index.js

@@ -1,3 +1,61 @@
 // @flow
+import * as React from "react";
+import { type Match } from "react-router-dom";
 import DataLoader from "./components/DataLoader";
-export default DataLoader;
+import Document from "./components/Document";
+import SocketPresence from "./components/SocketPresence";
+import useCurrentTeam from "hooks/useCurrentTeam";
+import useCurrentUser from "hooks/useCurrentUser";
+import useStores from "hooks/useStores";
+import { type LocationWithState } from "types";
+
+type Props = {|
+  location: LocationWithState,
+  match: Match,
+|};
+
+export default function DocumentScene(props: Props) {
+  const { ui } = useStores();
+  const team = useCurrentTeam();
+  const user = useCurrentUser();
+
+  React.useEffect(() => {
+    return () => ui.clearActiveDocument();
+  }, [ui]);
+
+  const { documentSlug, revisionId } = props.match.params;
+
+  // the urlId portion of the url does not include the slugified title
+  // we only want to force a re-mount of the document component when the
+  // document changes, not when the title does so only this portion is used
+  // for the key.
+  const urlParts = documentSlug ? documentSlug.split("-") : [];
+  const urlId = urlParts.length ? urlParts[urlParts.length - 1] : undefined;
+  const key = [urlId, revisionId].join("/");
+  const isMultiplayer = team.multiplayerEditor;
+
+  return (
+    <DataLoader key={key} match={props.match}>
+      {({ document, isEditing, ...rest }) => {
+        const isActive =
+          !document.isArchived && !document.isDeleted && !revisionId;
+
+        // TODO: Remove once multiplayer is 100% rollout, SocketPresence will
+        // no longer be required
+        if (isActive && !isMultiplayer) {
+          return (
+            <SocketPresence
+              documentId={document.id}
+              userId={user.id}
+              isEditing={isEditing}
+            >
+              <Document document={document} match={props.match} {...rest} />
+            </SocketPresence>
+          );
+        }
+
+        return <Document document={document} match={props.match} {...rest} />;
+      }}
+    </DataLoader>
+  );
+}

flow-typed/globals.js

@@ -2,6 +2,7 @@
 declare var process: {
   exit: (code?: number) => void,
   cwd: () => string,
+  argv: Array<string>,
   env: {
     [string]: string,
   },

package.json

@@ -7,10 +7,13 @@
     "clean": "rimraf build",
     "build:i18n": "i18next 'app/**/*.js' 'server/**/*.js' && mkdir -p ./build/shared/i18n && cp -R ./shared/i18n/locales ./build/shared/i18n",
     "build:server": "babel -d ./build/server ./server && babel -d ./build/shared ./shared && cp package.json ./build && ln -sf \"$(pwd)/webpack.config.dev.js\" ./build",
+    "build:multiplayer": "babel -d ./build/multiplayer ./multiplayer",
     "build:webpack": "webpack --config webpack.config.prod.js",
     "build": "yarn clean && yarn build:webpack && yarn build:i18n && yarn build:server",
     "start": "node ./build/server/index.js",
+    "start:multiplayer": "node ./build/multiplayer/index.js",
     "dev": "nodemon --exec \"yarn build:server && yarn build:i18n && node --inspect=0.0.0.0 build/server/index.js\" -e js --ignore build/ --ignore app/ --ignore flow-typed/",
+    "dev:multiplayer": "nodemon --exec \"yarn build:server && node --inspect=0.0.0.0 build/server/index.js --multiplayer\" -e js --ignore build/ --ignore app/ --ignore flow-typed/",
     "lint": "eslint app server shared",
     "deploy": "git push heroku master",
     "prepare": "yarn yarn-deduplicate yarn.lock",
@@ -45,6 +48,9 @@
     "@babel/preset-env": "^7.11.0",
     "@babel/preset-flow": "^7.10.4",
     "@babel/preset-react": "^7.10.4",
+    "@hocuspocus/extension-logger": "^1.0.0-alpha.35",
+    "@hocuspocus/provider": "^1.0.0-alpha.8",
+    "@hocuspocus/server": "^1.0.0-alpha.60",
     "@outlinewiki/koa-passport": "^4.1.4",
     "@outlinewiki/passport-azure-ad-oauth2": "^0.1.0",
     "@sentry/node": "^6.3.1",
@@ -165,7 +171,9 @@
     "turndown": "^7.1.1",
     "utf8": "^3.0.0",
     "uuid": "^8.3.2",
-    "validator": "5.2.0"
+    "validator": "5.2.0",
+    "y-prosemirror": "^1.0.9",
+    "yjs": "^13.5.12"
   },
   "devDependencies": {
     "@babel/cli": "^7.10.5",

server/api/team.js

@@ -17,6 +17,7 @@ router.post("team.update", auth(), async (ctx) => {
     sharing,
     guestSignin,
     documentEmbeds,
+    multiplayerEditor,
   } = ctx.body;
   const user = ctx.state.user;
   const team = await Team.findByPk(user.teamId);
@@ -31,6 +32,9 @@ router.post("team.update", auth(), async (ctx) => {
   if (documentEmbeds !== undefined) team.documentEmbeds = documentEmbeds;
   if (guestSignin !== undefined) team.guestSignin = guestSignin;
   if (avatarUrl !== undefined) team.avatarUrl = avatarUrl;
+  if (multiplayerEditor !== undefined) {
+    team.multiplayerEditor = multiplayerEditor;
+  }
 
   const changes = team.changed();
   const data = {};

server/commands/documentUpdater.js

@@ -0,0 +1,67 @@
+// @flow
+import { uniq } from "lodash";
+import { schema, serializer } from "rich-markdown-editor";
+import { yDocToProsemirror } from "y-prosemirror";
+import * as Y from "yjs";
+import { Document, Event } from "../models";
+
+export default async function documentUpdater({
+  documentId,
+  ydoc,
+  userId,
+  done,
+}: {
+  documentId: string,
+  ydoc: Y.Doc,
+  userId: string,
+  done?: boolean,
+}) {
+  const document = await Document.findByPk(documentId);
+  const state = Y.encodeStateAsUpdate(ydoc);
+  const node = yDocToProsemirror(schema, ydoc);
+  const text = serializer.serialize(node);
+
+  // extract collaborators from doc user data
+  const pud = new Y.PermanentUserData(ydoc);
+  const pudIds = Array.from(pud.clients.values());
+  const existingIds = document.collaboratorIds;
+  const collaboratorIds = uniq([...pudIds, ...existingIds]);
+
+  if (document.text === text) {
+    return;
+  }
+
+  await Document.update(
+    {
+      text,
+      state: Buffer.from(state),
+      updatedAt: new Date(),
+      lastModifiedById: userId,
+      collaboratorIds,
+    },
+    {
+      hooks: false,
+      where: {
+        id: document.id,
+      },
+    }
+  );
+
+  const event = {
+    name: "documents.update",
+    documentId: document.id,
+    collectionId: document.collectionId,
+    teamId: document.teamId,
+    actorId: userId,
+    data: {
+      multiplayer: true,
+      title: document.title,
+    },
+  };
+
+  if (done) {
+    await Event.create(event);
+  } else {
+    await Event.add(event);
+  }
+}

server/env.js

@@ -1,10 +1,11 @@
 // @flow
 
 // Note: This entire object is stringified in the HTML exposed to the client
-// do not add anything here that should be a secret or password
+// do not add anything here that would be considered a secret or password
 export default {
   URL: process.env.URL,
   CDN_URL: process.env.CDN_URL || "",
+  MULTIPLAYER_URL: process.env.MULTIPLAYER_URL || "",
   DEPLOYMENT: process.env.DEPLOYMENT,
   ENVIRONMENT: process.env.NODE_ENV,
   SENTRY_DSN: process.env.SENTRY_DSN,

server/events.js

@@ -1,7 +1,6 @@
 // @flow
 import * as Sentry from "@sentry/node";
 import debug from "debug";
-import services from "./services";
 import { createQueue } from "./utils/queue";
 
 const log = debug("services");
@@ -195,39 +194,44 @@ export type Event =
 const globalEventsQueue = createQueue("global events");
 const serviceEventsQueue = createQueue("service events");
 
-// this queue processes global events and hands them off to service hooks
-globalEventsQueue.process(async (job) => {
-  const names = Object.keys(services);
-  names.forEach((name) => {
-    const service = services[name];
-    if (service.on) {
-      serviceEventsQueue.add(
-        { ...job.data, service: name },
-        { removeOnComplete: true }
-      );
-    }
-  });
-});
+// TODO: This is a hack to prevent a require loop from models -> Event -> services -> main
+if (!process.argv.includes("--multiplayer")) {
+  const services = require("./services");
 
-// this queue processes an individual event for a specific service
-serviceEventsQueue.process(async (job) => {
-  const event = job.data;
-  const service = services[event.service];
-
-  if (service.on) {
-    log(`${event.service} processing ${event.name}`);
-
-    service.on(event).catch((error) => {
-      if (process.env.SENTRY_DSN) {
-        Sentry.withScope(function (scope) {
-          scope.setExtra("event", event);
-          Sentry.captureException(error);
-        });
-      } else {
-        throw error;
+  // this queue processes global events and hands them off to service hooks
+  globalEventsQueue.process(async (job) => {
+    const names = Object.keys(services);
+    names.forEach((name) => {
+      const service = services[name];
+      if (service.on) {
+        serviceEventsQueue.add(
+          { ...job.data, service: name },
+          { removeOnComplete: true }
+        );
       }
     });
-  }
-});
+  });
+
+  // this queue processes an individual event for a specific service
+  serviceEventsQueue.process(async (job) => {
+    const event = job.data;
+    const service = services[event.service];
+
+    if (service.on) {
+      log(`${event.service} processing ${event.name}`);
+
+      service.on(event).catch((error) => {
+        if (process.env.SENTRY_DSN) {
+          Sentry.withScope(function (scope) {
+            scope.setExtra("event", event);
+            Sentry.captureException(error);
+          });
+        } else {
+          throw error;
+        }
+      });
+    }
+  });
+}
 
 export default globalEventsQueue;

server/index.js

@@ -111,11 +111,22 @@ Is your team enjoying Outline? Consider supporting future development by sponsor
   );
 }
 
-const { start } = require("./main");
+const isMultiplayer = process.argv.includes("--multiplayer");
 
-throng({
-  worker: start,
+if (isMultiplayer) {
+  Error.stackTraceLimit = Infinity;
 
-  // The number of workers to run, defaults to the number of CPUs available
-  count: process.env.WEB_CONCURRENCY || undefined,
-});
+  const { start } = require("./multiplayer");
+
+  // TODO: Not using throng until multiplayer server has multi-process support
+  start();
+} else {
+  const { start } = require("./main");
+
+  throng({
+    worker: start,
+
+    // The number of workers to run, defaults to the number of CPUs available
+    count: process.env.WEB_CONCURRENCY || undefined,
+  });
+}

server/migrations/20201013030525-create-realtime.js

@@ -0,0 +1,18 @@
+'use strict';
+
+module.exports = {
+  up: async (queryInterface, Sequelize) => {
+    await queryInterface.addColumn('documents', 'state', {
+      type: Sequelize.BLOB
+    });
+    await queryInterface.addColumn('teams', 'multiplayerEditor', {
+      type: Sequelize.BOOLEAN,
+      defaultValue: false,
+    });
+  },
+
+  down: async (queryInterface, Sequelize) => {
+    await queryInterface.removeColumn('documents', 'state');
+    await queryInterface.removeColumn('teams', 'multiplayerEditor');
+  }
+};

server/models/Document.js

@@ -81,6 +81,7 @@ const Document = sequelize.define(
     template: DataTypes.BOOLEAN,
     editorVersion: DataTypes.STRING,
     text: DataTypes.TEXT,
+    state: DataTypes.BLOB,
     isWelcome: { type: DataTypes.BOOLEAN, defaultValue: false },
     revisionCount: { type: DataTypes.INTEGER, defaultValue: 0 },
     archivedAt: DataTypes.DATE,

server/models/Team.js

@@ -69,6 +69,11 @@ const Team = sequelize.define(
       allowNull: false,
       defaultValue: true,
     },
+    multiplayerEditor: {
+      type: DataTypes.BOOLEAN,
+      allowNull: false,
+      defaultValue: false,
+    },
   },
   {
     paranoid: true,

server/models/User.js

@@ -7,6 +7,7 @@ import { languages } from "../../shared/i18n";
 import { ValidationError } from "../errors";
 import { DataTypes, sequelize, encryptedFields, Op } from "../sequelize";
 import { DEFAULT_AVATAR_HOST } from "../utils/avatars";
+import { palette } from "../utils/color";
 import { publicS3Endpoint, uploadToS3FromUrl } from "../utils/s3";
 import {
   UserAuthentication,
@@ -74,6 +75,11 @@ const User = sequelize.define(
           .digest("hex");
         return `${DEFAULT_AVATAR_HOST}/avatar/${hash}/${initial}.png`;
       },
+      color() {
+        const idAsHex = crypto.createHash("md5").update(this.id).digest("hex");
+        const idAsNumber = parseInt(idAsHex, 16);
+        return palette[idAsNumber % palette.length];
+      },
     },
   }
 );

server/multiplayer/index.js

@@ -0,0 +1,68 @@
+// @flow
+import { Logger } from "@hocuspocus/extension-logger";
+import { Server } from "@hocuspocus/server";
+import debug from "debug";
+//import { RocksDB } from "@hocuspocus/extension-rocksdb";
+import { AuthenticationError } from "../errors";
+//import policy from "../policies";
+import { Document } from "../models";
+import { getUserForJWT } from "../utils/jwt";
+
+const isProduction = process.env.NODE_ENV === "production";
+const log = debug("multiplayer");
+//const { can } = policy;
+const can = () => true;
+
+const server = Server.configure({
+  port: process.env.MULTIPLAYER_PORT || process.env.PORT || 80,
+
+  async onConnect(data) {
+    const { requestParameters, documentName } = data;
+
+    // allows for different entity types to use this multiplayer provider later
+    const [, documentId] = documentName.split(".");
+    const { token } = requestParameters;
+
+    if (!token) {
+      throw new AuthenticationError("Authentication required");
+    }
+
+    const user = await getUserForJWT(token);
+    if (user.isSuspended) {
+      throw new AuthenticationError("Account suspended");
+    }
+
+    const document = await Document.findByPk(documentId, { userid: user.id });
+    if (!can(user, "read", document)) {
+      throw new AuthenticationError("Authorization required");
+    }
+
+    // set document to read only for the current user, thus changes will not be
+    // accepted and synced to other clients
+    if (!can(user, "update", document)) {
+      data.connection.readOnly = true;
+    }
+
+    return {
+      user,
+    };
+  },
+
+  extensions: [
+    new Logger(),
+    // new RocksDB({
+    //   path: "./database",
+
+    //   options: {
+    //     // see available options:
+    //     // https://www.npmjs.com/package/leveldown#options
+    //     createIfMissing: true,
+    //   },
+    // }),
+  ],
+});
+
+export async function start() {
+  console.log(`Started multiplayer server`);
+  server.listen();
+}

server/presenters/team.js

@@ -1,4 +1,5 @@
 // @flow
+import env from "../env";
 import { Team } from "../models";
 
 export default function present(team: Team) {
@@ -7,6 +8,7 @@ export default function present(team: Team) {
     name: team.name,
     avatarUrl: team.logoUrl,
     sharing: team.sharing,
+    multiplayerEditor: team.multiplayerEditor && env.MULTIPLAYER_URL,
     documentEmbeds: team.documentEmbeds,
     guestSignin: team.guestSignin,
     subdomain: team.subdomain,

server/presenters/user.js

@@ -10,6 +10,7 @@ type UserPresentation = {
   name: string,
   avatarUrl: ?string,
   email?: string,
+  color: string,
   isAdmin: boolean,
   isSuspended: boolean,
   isViewer: boolean,
@@ -21,6 +22,7 @@ export default (user: User, options: Options = {}): ?UserPresentation => {
   userData.id = user.id;
   userData.createdAt = user.createdAt;
   userData.name = user.name;
+  userData.color = user.color;
   userData.isAdmin = user.isAdmin;
   userData.isViewer = user.isViewer;
   userData.isSuspended = user.isSuspended;

server/services/backlinks.js

@@ -1,6 +1,6 @@
 // @flow
 import type { DocumentEvent, RevisionEvent } from "../events";
-import { Document, Backlink } from "../models";
+import { Document, Team, Backlink } from "../models";
 import { Op } from "../sequelize";
 import parseDocumentIds from "../utils/parseDocumentIds";
 import slugify from "../utils/slugify";
@@ -78,13 +78,19 @@ export default class Backlinks {
         break;
       }
       case "documents.title_change": {
-        const document = await Document.findByPk(event.documentId);
-        if (!document) return;
-
         // might as well check
         const { title, previousTitle } = event.data;
         if (!previousTitle || title === previousTitle) break;
 
+        const document = await Document.findByPk(event.documentId);
+        if (!document) return;
+
+        // TODO: Handle re-writing of titles into CRDT
+        const team = await Team.findByPk(document.teamId);
+        if (team.multiplayerEditor) {
+          break;
+        }
+
         // update any link titles in documents that lead to this one
         const backlinks = await Backlink.findAll({
           where: {

server/utils/color.js

@@ -0,0 +1,20 @@
+// @flow
+import { darken } from "polished";
+import theme from "../../shared/theme";
+
+export const palette = [
+  theme.brand.red,
+  theme.brand.blue,
+  theme.brand.purple,
+  theme.brand.pink,
+  theme.brand.marine,
+  theme.brand.green,
+  theme.brand.yellow,
+  darken(0.2, theme.brand.red),
+  darken(0.2, theme.brand.blue),
+  darken(0.2, theme.brand.purple),
+  darken(0.2, theme.brand.pink),
+  darken(0.2, theme.brand.marine),
+  darken(0.2, theme.brand.green),
+  darken(0.2, theme.brand.yellow),
+];

shared/theme.js

@@ -40,6 +40,7 @@ const colors = {
     blue: "#3633FF",
     marine: "#2BC2FF",
     green: "#42DED1",
+    yellow: "#F5BE31",
   },
 };
 

yarn.lock

@@ -1116,6 +1116,36 @@
   dependencies:
     "@hapi/hoek" "^8.3.0"
 
+"@hocuspocus/extension-logger@^1.0.0-alpha.35":
+  version "1.0.0-alpha.35"
+  resolved "https://registry.yarnpkg.com/@hocuspocus/extension-logger/-/extension-logger-1.0.0-alpha.35.tgz#1d13c82b5da9eb91442e121f374f0a7609cd11fd"
+  integrity sha512-yZ0pEmUvsfor9QhztAVnLOoY1LZpxA8UXJ0ZHr94oYgnmKo+aEK2P7xeiPT7GMH2t+ZYZn5UKoG3z4aaAEt6jg==
+  dependencies:
+    "@hocuspocus/server" "^1.0.0-alpha.60"
+
+"@hocuspocus/provider@^1.0.0-alpha.8":
+  version "1.0.0-alpha.8"
+  resolved "https://registry.yarnpkg.com/@hocuspocus/provider/-/provider-1.0.0-alpha.8.tgz#6e6ccb7e29622df84680de542815695e651ffad9"
+  integrity sha512-7BuHVISzv5RRrX4yU5JJ63/6epP6kkoJc97Th17Jm1rPybFA3/0LpmrLw8eEU35pRWtR0BWIzR9G7hPwjaOo1g==
+  dependencies:
+    lib0 "^0.2.42"
+    y-protocols "^1.0.5"
+    yjs "^13.5.8"
+
+"@hocuspocus/server@^1.0.0-alpha.60":
+  version "1.0.0-alpha.60"
+  resolved "https://registry.yarnpkg.com/@hocuspocus/server/-/server-1.0.0-alpha.60.tgz#28221dfc8e5fa27d4b4e17e377ab766edde2a26e"
+  integrity sha512-PKLIYHwk5NmssutlZr8DAIvqRWkDONpCMspDBKWnGbA+RiapVA7oIOZLTH37EpkkVOkRFpvhM+GAsOqUU2/K6w==
+  dependencies:
+    "@types/async-lock" "^1.1.2"
+    "@types/uuid" "^8.3.0"
+    "@types/ws" "^7.4.0"
+    async-lock "^1.2.8"
+    lib0 "^0.2.41"
+    uuid "^8.3.2"
+    ws "^7.4.3"
+    yjs "^13.5.0"
+
 "@icons/material@^0.2.4":
   version "0.2.4"
   resolved "https://registry.yarnpkg.com/@icons/material/-/material-0.2.4.tgz#e90c9f71768b3736e76d7dd6783fc6c2afa88bc8"
@@ -2012,6 +2042,11 @@
   resolved "https://registry.yarnpkg.com/@tommoor/remove-markdown/-/remove-markdown-0.3.2.tgz#5288ddd0e26b6b173e76ebb31c94653b0dcff45d"
   integrity sha512-awcc9hfLZqyyZHOGzAHbnjgZJpQGS1W1oZZ5GXOTTnbKVdKQ4OWYbrRWPUvXI2YAKJazrcS8rxPh67PX3rpGkQ==
 
+"@types/async-lock@^1.1.2":
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/@types/async-lock/-/async-lock-1.1.3.tgz#0d86017cf87abbcb941c55360e533d37a3f23b3d"
+  integrity sha512-UpeDcjGKsYEQMeqEbfESm8OWJI305I7b9KE4ji3aBjoKWyN5CTdn8izcA1FM1DVDne30R5fNEnIy89vZw5LXJQ==
+
 "@types/babel__core@^7.0.0", "@types/babel__core@^7.1.7":
   version "7.1.12"
   resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.1.12.tgz#4d8e9e51eb265552a7e4f1ff2219ab6133bdfb2d"
@@ -2187,6 +2222,18 @@
   resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.3.tgz#9c088679876f374eb5983f150d4787aa6fb32d7e"
   integrity sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==
 
+"@types/uuid@^8.3.0":
+  version "8.3.1"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.1.tgz#1a32969cf8f0364b3d8c8af9cc3555b7805df14f"
+  integrity sha512-Y2mHTRAbqfFkpjldbkHGY8JIzRN6XqYRliG8/24FcHm2D2PwW24fl5xMRTVGdrb7iMrwCaIEbLWerGIkXuFWVg==
+
+"@types/ws@^7.4.0":
+  version "7.4.7"
+  resolved "https://registry.yarnpkg.com/@types/ws/-/ws-7.4.7.tgz#f7c390a36f7a0679aa69de2d501319f4f8d9b702"
+  integrity sha512-JQbbmxZTZehdc2iszGKs5oC3NFnjeay7mtAWrdt7qNtAVK0g19muApzAy4bm9byz79xa2ZnO/BOBC2R8RC5Lww==
+  dependencies:
+    "@types/node" "*"
+
 "@types/yargs-parser@*":
   version "15.0.0"
   resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-15.0.0.tgz#cb3f9f741869e20cce330ffbeb9271590483882d"
@@ -2734,6 +2781,11 @@ async-each@^1.0.1:
   resolved "https://registry.yarnpkg.com/async-each/-/async-each-1.0.3.tgz#b727dbf87d7651602f06f4d4ac387f47d91b0cbf"
   integrity sha512-z/WhQ5FPySLdvREByI2vZiTWwCnF0moMJ1hK9YQwDTHKh6I7/uSckMetoRGb5UBZPC1z0jlw+n/XCgjeH7y1AQ==
 
+async-lock@^1.2.8:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/async-lock/-/async-lock-1.3.0.tgz#0fba111bea8b9693020857eba4f9adca173df3e5"
+  integrity sha512-8A7SkiisnEgME2zEedtDYPxUPzdv3x//E7n5IFktPAtMYSEAV7eNJF0rMwrVyUFj6d/8rgajLantbjcNRQYXIg==
+
 asynckit@^0.4.0:
   version "0.4.0"
   resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79"
@@ -7499,6 +7551,11 @@ isomorphic-fetch@^3.0.0:
     node-fetch "^2.6.1"
     whatwg-fetch "^3.4.1"
 
+isomorphic.js@^0.2.4:
+  version "0.2.4"
+  resolved "https://registry.yarnpkg.com/isomorphic.js/-/isomorphic.js-0.2.4.tgz#24ca374163ae54a7ce3b86ce63b701b91aa84969"
+  integrity sha512-Y4NjZceAwaPXctwsHgNsmfuPxR8lJ3f8X7QTAkhltrX4oGIv+eTlgHLXn4tWysC9zGTi929gapnPp+8F8cg7nA==
+
 isstream@~0.1.2:
   version "0.1.2"
   resolved "https://registry.yarnpkg.com/isstream/-/isstream-0.1.2.tgz#47e63f7af55afa6f92e1500e690eb8b8529c099a"
@@ -8470,6 +8527,13 @@ levn@~0.3.0:
     prelude-ls "~1.1.2"
     type-check "~0.3.2"
 
+lib0@^0.2.34, lib0@^0.2.41, lib0@^0.2.42:
+  version "0.2.42"
+  resolved "https://registry.yarnpkg.com/lib0/-/lib0-0.2.42.tgz#6d8bf1fb8205dec37a953c521c5ee403fd8769b0"
+  integrity sha512-8BNM4MiokEKzMvSxTOC3gnCBisJH+jL67CnSnqzHv3jli3pUvGC8wz+0DQ2YvGr4wVQdb2R2uNNPw9LEpVvJ4Q==
+  dependencies:
+    isomorphic.js "^0.2.4"
+
 lie@~3.3.0:
   version "3.3.0"
   resolved "https://registry.yarnpkg.com/lie/-/lie-3.3.0.tgz#dcf82dee545f46074daf200c7c1c5a08e0f40f6a"
@@ -14228,7 +14292,12 @@ write@1.0.3:
   dependencies:
     mkdirp "^0.5.1"
 
-ws@^7.2.3, ws@~7.4.2:
+ws@^7.2.3, ws@^7.4.3:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.3.tgz#160835b63c7d97bfab418fc1b8a9fced2ac01a74"
+  integrity sha512-kQ/dHIzuLrS6Je9+uv81ueZomEwH0qVYstcAQ4/Z93K8zeko9gtAbttJWzoC5ukqXY1PpoouV3+VSOqEAFt5wg==
+
+ws@~7.4.2:
   version "7.4.6"
   resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
   integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==
@@ -14309,6 +14378,20 @@ xtend@^4.0.0, xtend@~4.0.0, xtend@~4.0.1:
   resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"
   integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==
 
+y-prosemirror@^1.0.9:
+  version "1.0.9"
+  resolved "https://registry.yarnpkg.com/y-prosemirror/-/y-prosemirror-1.0.9.tgz#c0b5bf4e2c6620093ba0658c2aca52055346a683"
+  integrity sha512-OM12aPx04lwiIy1IOBidb6ONAof2KFxQE/Gww26SEsMQuA2dibrJkjaMwXwY1KnYY7yOpwbIFRdwecdNXLU9yQ==
+  dependencies:
+    lib0 "^0.2.34"
+
+y-protocols@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/y-protocols/-/y-protocols-1.0.5.tgz#91d574250060b29fcac8f8eb5e276fbad594245e"
+  integrity sha512-Wil92b7cGk712lRHDqS4T90IczF6RkcvCwAD0A2OPg+adKmOe+nOiT/N2hvpQIWS3zfjmtL4CPaH5sIW1Hkm/A==
+  dependencies:
+    lib0 "^0.2.42"
+
 y18n@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
@@ -14405,6 +14488,13 @@ yeast@0.1.2:
   resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419"
   integrity sha1-AI4G2AlDIMNy28L47XagymyKxBk=
 
+yjs@^13.5.0, yjs@^13.5.12, yjs@^13.5.8:
+  version "13.5.12"
+  resolved "https://registry.yarnpkg.com/yjs/-/yjs-13.5.12.tgz#7a0cf3119fb368c07243825e989a55de164b3f9c"
+  integrity sha512-/buy1kh8Ls+t733Lgov9hiNxCsjHSCymTuZNahj2hsPNoGbvnSdDmCz9Z4F19Yr1eUAAXQLJF3q7fiBcvPC6Qg==
+  dependencies:
+    lib0 "^0.2.41"
+
 ylru@^1.2.0:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/ylru/-/ylru-1.2.1.tgz#f576b63341547989c1de7ba288760923b27fe84f"