Add webpackStatsFile

.github/workflows/ci.yml

@@ -63,6 +63,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       server: ${{ steps.filter.outputs.server }}
+      app: ${{ steps.filter.outputs.app }}
     steps:
       - uses: actions/checkout@v4
       - uses: dorny/paths-filter@v2
@@ -81,7 +82,7 @@ jobs:
               - 'yarn.lock'
 
   test:
-    needs: build
+    needs: [build, changes]
     if: ${{ needs.changes.outputs.app == 'true' }}
     runs-on: ubuntu-latest
     strategy:
@@ -143,7 +144,7 @@ jobs:
         yarn test --maxWorkers=2 $TESTFILES
 
   bundle-size:
-    needs: [build, types]
+    needs: [build, types, changes]
     if: ${{ needs.changes.outputs.app == 'true' }}
     runs-on: ubuntu-latest
     steps:
@@ -161,3 +162,4 @@ jobs:
       with:
         key: ${{ secrets.RELATIVE_CI_KEY }}
         token: ${{ secrets.GITHUB_TOKEN }}
+        webpackStatsFile: ./build/app/webpack-stats.json

shared/editor/extensions/CodeHighlighting.ts

@@ -4,7 +4,7 @@ import { Node } from "prosemirror-model";
 import { Plugin, PluginKey, Transaction } from "prosemirror-state";
 import { Decoration, DecorationSet } from "prosemirror-view";
 import refractor from "refractor/core";
-import { getPrismLangForLanguage } from "../lib/code";
+import { getRefractorLangForLanguage } from "../lib/code";
 import { isRemoteTransaction } from "../lib/multiplayer";
 import { findBlockNodes } from "../queries/findChildren";
 
@@ -14,6 +14,34 @@ type ParsedNode = {
 };
 
 const cache: Record<number, { node: Node; decorations: Decoration[] }> = {};
+const languagesToImport = new Set<string>();
+
+async function loadLanguage(language: string) {
+  if (!language || refractor.registered(language)) {
+    return;
+  }
+  try {
+    // @ts-expect-error we are adding a module to the window object to work
+    // around the fact that refractor doesn't export ESM but import expects it.
+    // See the rules of dynamic imports:
+    // https://github.com/rollup/plugins/blob/e1a5ef99f1578eb38a8c87563cb9651db228f3bd/packages/dynamic-import-vars/README.md#limitations
+    window.module ??= {};
+    return import(`../../../node_modules/refractor/lang/${language}.js`).then(
+      () => {
+        refractor.register(window.module.exports);
+        return language;
+      }
+    );
+  } catch (err) {
+    // It will retry loading the language on the next render
+    // eslint-disable-next-line no-console
+    console.error(
+      `Failed to load language ${language} for code highlighting`,
+      err
+    );
+  }
+  return;
+}
 
 function getDecorations({
   doc,
@@ -57,12 +85,20 @@ function getDecorations({
 
   blocks.forEach((block) => {
     let startPos = block.pos + 1;
-    const language = getPrismLangForLanguage(block.node.attrs.language);
+    const language = getRefractorLangForLanguage(block.node.attrs.language);
 
-    if (!language || !refractor.registered(language)) {
+    if (!language) {
       return;
     }
 
+    // If the language isn't registered yet, trigger loading it
+    if (!refractor.registered(language)) {
+      languagesToImport.add(language);
+      return;
+    } else {
+      languagesToImport.delete(language);
+    }
+
     const lineDecorations = [];
 
     if (!cache[block.pos] || !cache[block.pos].node.eq(block.node)) {
@@ -133,7 +169,7 @@ function getDecorations({
   return DecorationSet.create(doc, decorations);
 }
 
-export default function Prism({
+export function CodeHighlighting({
   name,
   lineNumbers,
 }: {
@@ -145,7 +181,7 @@ export default function Prism({
   let highlighted = false;
 
   return new Plugin({
-    key: new PluginKey("prism"),
+    key: new PluginKey("codeHighlighting"),
     state: {
       init: (_, { doc }) => DecorationSet.create(doc, []),
       apply: (transaction: Transaction, decorationSet, oldState, state) => {
@@ -156,11 +192,13 @@ export default function Prism({
 
         // @ts-expect-error accessing private field.
         const isPaste = transaction.meta?.paste;
+        const langLoaded = transaction.getMeta("codeHighlighting")?.langLoaded;
 
         if (
           !highlighted ||
           codeBlockChanged ||
           isPaste ||
+          langLoaded ||
           isRemoteTransaction(transaction)
         ) {
           highlighted = true;
@@ -174,15 +212,34 @@ export default function Prism({
       if (!highlighted) {
         // we don't highlight code blocks on the first render as part of mounting
         // as it's expensive (relative to the rest of the document). Instead let
-        // it render un-highlighted and then trigger a defered render of Prism
+        // it render un-highlighted and then trigger a defered render of highlighting
         // by updating the plugins metadata
-        setTimeout(() => {
+        requestAnimationFrame(() => {
           if (!view.isDestroyed) {
-            view.dispatch(view.state.tr.setMeta("prism", { loaded: true }));
+            view.dispatch(
+              view.state.tr.setMeta("codeHighlighting", { loaded: true })
+            );
           }
-        }, 10);
+        });
       }
-      return {};
+      return {
+        update: () => {
+          if (!languagesToImport.size) {
+            return;
+          }
+
+          void Promise.all([...languagesToImport].map(loadLanguage)).then(
+            (language) =>
+              languagesToImport.size
+                ? view.dispatch(
+                    view.state.tr.setMeta("codeHighlighting", {
+                      langLoaded: language,
+                    })
+                  )
+                : null
+          );
+        },
+      };
     },
     props: {
       decorations(state) {

shared/editor/lib/code.test.ts

@@ -1,12 +1,12 @@
-import { getPrismLangForLanguage, getLabelForLanguage } from "./code";
+import { getRefractorLangForLanguage, getLabelForLanguage } from "./code";
 
-describe("getPrismLangForLanguage", () => {
-  it("should return the correct Prism language identifier for a given language", () => {
-    expect(getPrismLangForLanguage("javascript")).toBe("javascript");
-    expect(getPrismLangForLanguage("mermaidjs")).toBe("mermaid");
-    expect(getPrismLangForLanguage("xml")).toBe("markup");
-    expect(getPrismLangForLanguage("unknown")).toBeUndefined();
-    expect(getPrismLangForLanguage("")).toBeUndefined();
+describe("getRefractorLangForLanguage", () => {
+  it("should return the correct lang identifier for a given language", () => {
+    expect(getRefractorLangForLanguage("javascript")).toBe("javascript");
+    expect(getRefractorLangForLanguage("mermaidjs")).toBe("mermaid");
+    expect(getRefractorLangForLanguage("xml")).toBe("markup");
+    expect(getRefractorLangForLanguage("unknown")).toBeUndefined();
+    expect(getRefractorLangForLanguage("")).toBeUndefined();
   });
 });
 

shared/editor/lib/code.ts

@@ -1,6 +1,6 @@
 import Storage from "../../utils/Storage";
 
-const RecentStorageKey = "rme-code-language";
+const RecentlyUsedStorageKey = "rme-code-language";
 const StorageKey = "frequent-code-languages";
 const frequentLanguagesToGet = 5;
 const frequentLanguagesToTrack = 10;
@@ -9,7 +9,7 @@ const frequentLanguagesToTrack = 10;
  * List of supported code languages.
  *
  * Object key is the language identifier used in the editor, lang is the
- * language identifier used by Prism. Note mismatches such as `markup` and
+ * language identifier used by Refractor. Note mismatches such as `markup` and
  * `mermaid`.
  */
 export const codeLanguages = {
@@ -19,8 +19,10 @@ export const codeLanguages = {
   cpp: { lang: "cpp", label: "C++" },
   csharp: { lang: "csharp", label: "C#" },
   css: { lang: "css", label: "CSS" },
+  csv: { lang: "csv", label: "CSV" },
   docker: { lang: "docker", label: "Docker" },
   elixir: { lang: "elixir", label: "Elixir" },
+  erb: { lang: "erb", label: "ERB" },
   erlang: { lang: "erlang", label: "Erlang" },
   go: { lang: "go", label: "Go" },
   graphql: { lang: "graphql", label: "GraphQL" },
@@ -34,8 +36,11 @@ export const codeLanguages = {
   json: { lang: "json", label: "JSON" },
   jsx: { lang: "jsx", label: "JSX" },
   kotlin: { lang: "kotlin", label: "Kotlin" },
+  kusto: { lang: "kusto", label: "Kusto" },
   lisp: { lang: "lisp", label: "Lisp" },
   lua: { lang: "lua", label: "Lua" },
+  makefile: { lang: "makefile", label: "Makefile" },
+  markdown: { lang: "markdown", label: "Markdown" },
   mermaidjs: { lang: "mermaid", label: "Mermaid Diagram" },
   nginx: { lang: "nginx", label: "Nginx" },
   nix: { lang: "nix", label: "Nix" },
@@ -47,11 +52,13 @@ export const codeLanguages = {
   protobuf: { lang: "protobuf", label: "Protobuf" },
   python: { lang: "python", label: "Python" },
   r: { lang: "r", label: "R" },
+  regex: { lang: "regex", label: "Regex" },
   ruby: { lang: "ruby", label: "Ruby" },
   rust: { lang: "rust", label: "Rust" },
   scala: { lang: "scala", label: "Scala" },
   sass: { lang: "sass", label: "Sass" },
   scss: { lang: "scss", label: "SCSS" },
+  "splunk-spl": { lang: "splunk-spl", label: "Splunk SPL" },
   sql: { lang: "sql", label: "SQL" },
   solidity: { lang: "solidity", label: "Solidity" },
   swift: { lang: "swift", label: "Swift" },
@@ -79,12 +86,14 @@ export const getLabelForLanguage = (language: string) => {
 };
 
 /**
- * Get the Prism language identifier for a given language.
+ * Get the Refractor language identifier for a given language.
  *
  * @param language The language identifier.
- * @returns The Prism language identifier for the language.
+ * @returns The Refractor language identifier for the language.
  */
-export const getPrismLangForLanguage = (language: string): string | undefined =>
+export const getRefractorLangForLanguage = (
+  language: string
+): string | undefined =>
   codeLanguages[language as keyof typeof codeLanguages]?.lang;
 
 /**
@@ -92,14 +101,14 @@ export const getPrismLangForLanguage = (language: string): string | undefined =>
  *
  * @param language The language identifier.
  */
-export const setRecentCodeLanguage = (language: string) => {
+export const setRecentlyUsedCodeLanguage = (language: string) => {
   const frequentLangs = (Storage.get(StorageKey) ?? {}) as Record<
     string,
     number
   >;
 
   if (Object.keys(frequentLangs).length === 0) {
-    const lastUsedLang = Storage.get(RecentStorageKey);
+    const lastUsedLang = Storage.get(RecentlyUsedStorageKey);
     if (lastUsedLang) {
       frequentLangs[lastUsedLang] = 1;
     }
@@ -121,7 +130,7 @@ export const setRecentCodeLanguage = (language: string) => {
   }
 
   Storage.set(StorageKey, Object.fromEntries(frequentLangEntries));
-  Storage.set(RecentStorageKey, language);
+  Storage.set(RecentlyUsedStorageKey, language);
 };
 
 /**
@@ -129,8 +138,8 @@ export const setRecentCodeLanguage = (language: string) => {
  *
  * @returns The most recent code language used, or undefined if none is set.
  */
-export const getRecentCodeLanguage = () =>
-  Storage.get(RecentStorageKey) as keyof typeof codeLanguages | undefined;
+export const getRecentlyUsedCodeLanguage = () =>
+  Storage.get(RecentlyUsedStorageKey) as keyof typeof codeLanguages | undefined;
 
 /**
  * Get the most frequent code languages used.
@@ -138,7 +147,7 @@ export const getRecentCodeLanguage = () =>
  * @returns An array of the most frequent code languages used.
  */
 export const getFrequentCodeLanguages = () => {
-  const recentLang = Storage.get(RecentStorageKey);
+  const recentLang = Storage.get(RecentlyUsedStorageKey);
   const frequentLangEntries = Object.entries(Storage.get(StorageKey) ?? {}) as [
     keyof typeof codeLanguages,
     number

shared/editor/nodes/CodeFence.ts

@@ -9,58 +9,6 @@ import {
 } from "prosemirror-model";
 import { Command, Plugin, PluginKey, TextSelection } from "prosemirror-state";
 import { Decoration, DecorationSet } from "prosemirror-view";
-import refractor from "refractor/core";
-import bash from "refractor/lang/bash";
-import clike from "refractor/lang/clike";
-import cpp from "refractor/lang/cpp";
-import csharp from "refractor/lang/csharp";
-import css from "refractor/lang/css";
-import docker from "refractor/lang/docker";
-import elixir from "refractor/lang/elixir";
-import erlang from "refractor/lang/erlang";
-import go from "refractor/lang/go";
-import graphql from "refractor/lang/graphql";
-import groovy from "refractor/lang/groovy";
-import haskell from "refractor/lang/haskell";
-import hcl from "refractor/lang/hcl";
-import ini from "refractor/lang/ini";
-import java from "refractor/lang/java";
-import javascript from "refractor/lang/javascript";
-import json from "refractor/lang/json";
-import jsx from "refractor/lang/jsx";
-import kotlin from "refractor/lang/kotlin";
-import lisp from "refractor/lang/lisp";
-import lua from "refractor/lang/lua";
-import markup from "refractor/lang/markup";
-// @ts-expect-error type definition is missing, but package exists
-import mermaid from "refractor/lang/mermaid";
-import nginx from "refractor/lang/nginx";
-import nix from "refractor/lang/nix";
-import objectivec from "refractor/lang/objectivec";
-import ocaml from "refractor/lang/ocaml";
-import perl from "refractor/lang/perl";
-import php from "refractor/lang/php";
-import powershell from "refractor/lang/powershell";
-import protobuf from "refractor/lang/protobuf";
-import python from "refractor/lang/python";
-import r from "refractor/lang/r";
-import ruby from "refractor/lang/ruby";
-import rust from "refractor/lang/rust";
-import sass from "refractor/lang/sass";
-import scala from "refractor/lang/scala";
-import scss from "refractor/lang/scss";
-import solidity from "refractor/lang/solidity";
-import sql from "refractor/lang/sql";
-import swift from "refractor/lang/swift";
-import toml from "refractor/lang/toml";
-import tsx from "refractor/lang/tsx";
-import typescript from "refractor/lang/typescript";
-import verilog from "refractor/lang/verilog";
-import vhdl from "refractor/lang/vhdl";
-import visualbasic from "refractor/lang/visual-basic";
-import yaml from "refractor/lang/yaml";
-import zig from "refractor/lang/zig";
-
 import { toast } from "sonner";
 import { Primitive } from "utility-types";
 import type { Dictionary } from "~/hooks/useDictionary";
@@ -77,9 +25,12 @@ import {
 } from "../commands/codeFence";
 import { selectAll } from "../commands/selectAll";
 import toggleBlockType from "../commands/toggleBlockType";
+import { CodeHighlighting } from "../extensions/CodeHighlighting";
 import Mermaid from "../extensions/Mermaid";
-import Prism from "../extensions/Prism";
-import { getRecentCodeLanguage, setRecentCodeLanguage } from "../lib/code";
+import {
+  getRecentlyUsedCodeLanguage,
+  setRecentlyUsedCodeLanguage,
+} from "../lib/code";
 import { isCode } from "../lib/isCode";
 import { MarkdownSerializerState } from "../lib/markdown/serializer";
 import { findNextNewline, findPreviousNewline } from "../queries/findNewlines";
@@ -90,58 +41,6 @@ import Node from "./Node";
 
 const DEFAULT_LANGUAGE = "javascript";
 
-[
-  bash,
-  cpp,
-  css,
-  clike,
-  csharp,
-  docker,
-  elixir,
-  erlang,
-  go,
-  graphql,
-  groovy,
-  haskell,
-  hcl,
-  ini,
-  java,
-  javascript,
-  jsx,
-  json,
-  kotlin,
-  lisp,
-  lua,
-  markup,
-  mermaid,
-  nginx,
-  nix,
-  objectivec,
-  ocaml,
-  perl,
-  php,
-  python,
-  powershell,
-  protobuf,
-  r,
-  ruby,
-  rust,
-  scala,
-  sql,
-  solidity,
-  sass,
-  scss,
-  swift,
-  toml,
-  typescript,
-  tsx,
-  verilog,
-  vhdl,
-  visualbasic,
-  yaml,
-  zig,
-].forEach(refractor.register);
-
 export default class CodeFence extends Node {
   constructor(options: {
     dictionary: Dictionary;
@@ -212,10 +111,10 @@ export default class CodeFence extends Node {
     return {
       code_block: (attrs: Record<string, Primitive>) => {
         if (attrs?.language) {
-          setRecentCodeLanguage(attrs.language as string);
+          setRecentlyUsedCodeLanguage(attrs.language as string);
         }
         return toggleBlockType(type, schema.nodes.paragraph, {
-          language: getRecentCodeLanguage() ?? DEFAULT_LANGUAGE,
+          language: getRecentlyUsedCodeLanguage() ?? DEFAULT_LANGUAGE,
           ...attrs,
         });
       },
@@ -286,7 +185,7 @@ export default class CodeFence extends Node {
 
   get plugins() {
     return [
-      Prism({
+      CodeHighlighting({
         name: this.name,
         lineNumbers: this.showLineNumbers,
       }),
@@ -353,7 +252,7 @@ export default class CodeFence extends Node {
   inputRules({ type }: { type: NodeType }) {
     return [
       textblockTypeInputRule(/^```$/, type, () => ({
-        language: getRecentCodeLanguage() ?? DEFAULT_LANGUAGE,
+        language: getRecentlyUsedCodeLanguage() ?? DEFAULT_LANGUAGE,
       })),
     ];
   }