fix: Toggling a nested list no longer converts parent lists (#12670)

* fix: Toggling a nested list no longer converts parent lists

When the selection was inside a nested list, toggling the list type from
the toolbar or keyboard shortcut converted every list in the tree,
including ancestors of the selected list. This was caused by
doc.nodesBetween visiting ancestor nodes whose range overlaps the
selected list - these are now skipped so only the closest list and its
children are converted. Also guards against converting nested lists with
incompatible content such as checkbox lists.

Closes #12653

https://claude.ai/code/session_01Q5hkRNp1Fo3jAc9fW5t68h

* test: Throw when selection text is not found in toggleList test helper

https://claude.ai/code/session_01Q5hkRNp1Fo3jAc9fW5t68h

---------

Co-authored-by: Claude <noreply@anthropic.com>

shared/editor/commands/toggleList.test.ts

@@ -0,0 +1,112 @@
+import type { Node } from "prosemirror-model";
+import type { Command } from "prosemirror-state";
+import {
+  createEditorStateWithSelection,
+  doc,
+  p,
+  schema,
+} from "@shared/test/editor";
+import toggleList from "./toggleList";
+
+const { bullet_list, ordered_list, list_item } = schema.nodes;
+
+/**
+ * Creates a list item node with the given block content.
+ */
+function li(content: Node[]) {
+  return list_item.create(null, content);
+}
+
+/**
+ * Returns a position inside the first text node matching the given text.
+ *
+ * @throws if no matching text node exists in the document.
+ */
+function posOfText(node: Node, text: string) {
+  let found = -1;
+  node.descendants((child, pos) => {
+    if (found === -1 && child.isText && child.text === text) {
+      found = pos;
+    }
+    return found === -1;
+  });
+  if (found === -1) {
+    throw new Error(`Text "${text}" not found in document`);
+  }
+  return found + 1;
+}
+
+/**
+ * Runs a command with the selection placed inside the given text and returns
+ * the resulting document.
+ */
+function run(testDoc: Node, selectionText: string, command: Command) {
+  let state = createEditorStateWithSelection(
+    testDoc,
+    posOfText(testDoc, selectionText)
+  );
+  command(state, (tr) => {
+    state = state.apply(tr);
+  });
+  return state.doc;
+}
+
+describe("toggleList", () => {
+  it("converts a nested ordered list to bullet without changing the parent list", () => {
+    const testDoc = doc([
+      ordered_list.create(null, [
+        li([p("one")]),
+        li([p("two"), ordered_list.create(null, [li([p("nested")])])]),
+      ]),
+    ]);
+
+    const result = run(testDoc, "nested", toggleList(bullet_list, list_item));
+
+    const outer = result.firstChild;
+    expect(outer?.type.name).toBe("ordered_list");
+    expect(outer?.child(1).child(1).type.name).toBe("bullet_list");
+  });
+
+  it("converts a nested bullet list to ordered without changing the parent list", () => {
+    const testDoc = doc([
+      bullet_list.create(null, [
+        li([p("one")]),
+        li([p("two"), bullet_list.create(null, [li([p("nested")])])]),
+      ]),
+    ]);
+
+    const result = run(testDoc, "nested", toggleList(ordered_list, list_item));
+
+    const outer = result.firstChild;
+    expect(outer?.type.name).toBe("bullet_list");
+    expect(outer?.child(1).child(1).type.name).toBe("ordered_list");
+  });
+
+  it("converts the list and its children when the selection is in the parent list", () => {
+    const testDoc = doc([
+      ordered_list.create(null, [
+        li([p("one")]),
+        li([p("two"), ordered_list.create(null, [li([p("nested")])])]),
+      ]),
+    ]);
+
+    const result = run(testDoc, "two", toggleList(bullet_list, list_item));
+
+    const outer = result.firstChild;
+    expect(outer?.type.name).toBe("bullet_list");
+    expect(outer?.child(1).child(1).type.name).toBe("bullet_list");
+  });
+
+  it("lifts the item out of the list when toggling the same list type", () => {
+    const testDoc = doc([
+      bullet_list.create(null, [li([p("one")]), li([p("two")])]),
+    ]);
+
+    const result = run(testDoc, "two", toggleList(bullet_list, list_item));
+
+    expect(result.childCount).toBe(2);
+    expect(result.child(0).type.name).toBe("bullet_list");
+    expect(result.child(1).type.name).toBe("paragraph");
+    expect(result.child(1).textContent).toBe("two");
+  });
+});

shared/editor/commands/toggleList.ts

@@ -54,7 +54,14 @@ export default function toggleList(
           parentList.pos,
           parentList.pos + parentList.node.nodeSize,
           (node, pos) => {
-            if (isList(node, schema)) {
+            // nodesBetween also visits the ancestors of the given range, these
+            // must be skipped so that toggling a nested list does not convert
+            // the lists it is nested within.
+            if (
+              pos >= parentList.pos &&
+              isList(node, schema) &&
+              listType.validContent(node.content)
+            ) {
               tr.setNodeMarkup(pos, listType, listStyle ? { listStyle } : {});
             }
           }