[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels

.changeset/rare-lamps-cough.md

@@ -0,0 +1,6 @@
+---
+"@khanacademy/perseus": patch
+"@khanacademy/perseus-editor": patch
+---
+
+[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels

packages/perseus-editor/package.json

@@ -40,6 +40,7 @@
         "@khanacademy/math-input": "^21.1.4",
         "@khanacademy/perseus": "^41.2.0",
         "@khanacademy/perseus-core": "1.5.3",
+        "@khanacademy/pure-markdown": "^0.3.11",
         "mafs": "^0.19.0"
     },
     "devDependencies": {

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx

@@ -26,6 +26,14 @@ const defaultProps = {
 
 const defaultLabel = getDefaultFigureForType("label");
 
+// Mock the async function generateSpokenMathDetails
+jest.mock("./util", () => ({
+    ...jest.requireActual("./util"),
+    generateSpokenMathDetails: (input) => {
+        return Promise.resolve(`Spoken math details for ${input}`);
+    },
+}));
+
 describe("LockedPointSettings", () => {
     let userEvent: UserEvent;
     beforeEach(() => {
@@ -409,8 +417,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -439,8 +450,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A at (0, 0). Appearance solid gray.",
         });
     });
 
@@ -473,8 +487,11 @@ describe("LockedPointSettings", () => {
         await userEvent.click(autoGenButton);
 
         // Assert
+        // generateSpokenMathDetails is mocked to return the input string
+        // with "Spoken math details for " prepended.
         expect(onChangeProps).toHaveBeenCalledWith({
-            ariaLabel: "Point A, B at (0, 0). Appearance solid gray.",
+            ariaLabel:
+                "Spoken math details for Point A, B at (0, 0). Appearance solid gray.",
         });
     });
 });

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx

@@ -24,6 +24,7 @@ import LockedFigureSettingsActions from "./locked-figure-settings-actions";
 import LockedLabelSettings from "./locked-label-settings";
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -104,28 +105,50 @@ const LockedPointSettings = (props: Props) => {
 
     const isDefiningPoint = !onMove && !onRemove;
 
-    /**
-     * Get a prepopulated aria label for the point.
-     *
-     * If the point has no labels, the aria label will just be
-     * "Point at (x, y)".
-     *
-     * If the point has labels, the aria label will be
-     * "Point at (x, y) with label1, label2, label3".
-     */
-    function getPrepopulatedAriaLabel() {
-        let visiblelabel = "";
-        if (labels && labels.length > 0) {
-            visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`;
+    const [prepopulatedAriaLabel, setPrepopulatedAriaLabel] =
+        React.useState("");
+
+    React.useEffect(() => {
+        /**
+         * Get a prepopulated aria label for the point, with the math
+         * details converted into spoken words.
+         *
+         * If the point has no labels, the aria label will just be
+         * "Point at (x, y)".
+         *
+         * If the point has labels, the aria label will be
+         * "Point label1, label2, label3 at (x, y)".
+         */
+        async function getPrepopulatedAriaLabel() {
+            let visiblelabel = "";
+            if (labels && labels.length > 0) {
+                visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`;
+            }
+
+            let str = await generateSpokenMathDetails(
+                `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`,
+            );
+
+            const pointAppearance =
+                generateLockedFigureAppearanceDescription(pointColor);
+            str += pointAppearance;
+
+            return str;
         }
-        let str = `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`;
 
-        const pointAppearance =
-            generateLockedFigureAppearanceDescription(pointColor);
-        str += pointAppearance;
+        // Guard against the race condition where we're getting the
+        // aria label from a previous render with different coords,
+        // labels, or pointColor.
+        let canceled = false;
 
-        return str;
-    }
+        getPrepopulatedAriaLabel().then(
+            (label) => !canceled && setPrepopulatedAriaLabel(label),
+        );
+
+        return () => {
+            canceled = true;
+        };
+    }, [coord, labels, pointColor]);
 
     function handleColorChange(newValue) {
         const newProps: Partial<LockedPointType> = {
@@ -247,7 +270,7 @@ const LockedPointSettings = (props: Props) => {
 
                     <LockedFigureAria
                         ariaLabel={ariaLabel}
-                        prePopulatedAriaLabel={getPrepopulatedAriaLabel()}
+                        prePopulatedAriaLabel={prepopulatedAriaLabel}
                         onChangeProps={(newProps) => {
                             onChangeProps(newProps);
                         }}

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts

@@ -1,5 +1,6 @@
 import {
     generateLockedFigureAppearanceDescription,
+    generateSpokenMathDetails,
     getDefaultFigureForType,
 } from "./util";
 
@@ -258,3 +259,84 @@ describe("generateLockedFigureAppearanceDescription", () => {
         },
     );
 });
+
+// TODO(LEMS-2616): Update these tests to mock SpeechRuleEngine.setup()
+// so that the tests don't have to make HTTP requests.
+describe("generateMathDetails", () => {
+    test("should convert TeX to spoken language (root, fraction)", async () => {
+        const mathString = "$\\sqrt{\\frac{1}{2}}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("StartRoot one half EndRoot");
+    });
+
+    test("should convert TeX to spoken language (exponent)", async () => {
+        const mathString = "$x^{2}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("x Superscript 2");
+    });
+
+    test("should convert TeX to spoken language (negative)", async () => {
+        const mathString = "$-2$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("negative 2");
+    });
+
+    test("should converte TeX to spoken language (subtraction)", async () => {
+        const mathString = "$2-1$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("2 minus 1");
+    });
+
+    test("should convert TeX to spoken language (normal words)", async () => {
+        const mathString = "$\\text{square b}$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("square b");
+    });
+
+    test("should convert TeX to spoken language (random letters)", async () => {
+        const mathString = "$cat$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("c a t");
+    });
+
+    test("should keep non-math text as is", async () => {
+        const mathString = "Circle with radius $\\frac{1}{2}$ units";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Circle with radius one half units");
+    });
+
+    test("should read dollar signs as dollars inside tex", async () => {
+        const mathString = "This sandwich costs ${$}12.34$";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs dollar sign 12.34");
+    });
+
+    test("should read dollar signs as dollars outside tex", async () => {
+        const mathString = "This sandwich costs \\$12.34";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("This sandwich costs $12.34");
+    });
+
+    test("should read curly braces", async () => {
+        const mathString = "Hello}{";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("Hello}{");
+    });
+
+    test("should read backslashes", async () => {
+        const mathString = "\\";
+        const convertedString = await generateSpokenMathDetails(mathString);
+
+        expect(convertedString).toBe("\\");
+    });
+});

packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts

@@ -1,3 +1,5 @@
+import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer";
+import * as SimpleMarkdown from "@khanacademy/pure-markdown";
 import {UnreachableCaseError} from "@khanacademy/wonder-stuff-core";
 
 import type {
@@ -124,3 +126,29 @@ export function generateLockedFigureAppearanceDescription(
             throw new UnreachableCaseError(fill);
     }
 }
+
+export async function generateSpokenMathDetails(mathString: string) {
+    const engine = await SpeechRuleEngine.setup("en");
+    let convertedSpeech = "";
+
+    // All the information we need is in the first section,
+    // whether it's typed as "blockmath" or "paragraph"
+    const firstSection = SimpleMarkdown.parse(mathString)[0];
+
+    // If it's blockMath, the outer level has the full math content.
+    if (firstSection.type === "blockMath") {
+        convertedSpeech += engine.texToSpeech(firstSection.content);
+    }
+
+    // If it's a paragraph, we need to iterate through the sections
+    // to look for individual math blocks.
+    if (firstSection.type === "paragraph") {
+        for (const piece of firstSection.content) {
+            piece.type === "math"
+                ? (convertedSpeech += engine.texToSpeech(piece.content))
+                : (convertedSpeech += piece.content);
+        }
+    }
+
+    return convertedSpeech;
+}

packages/perseus/src/widgets/interactive-graphs/utils.ts

@@ -109,9 +109,13 @@ type ParsedNode = {
 
 // Helper function for replaceOutsideTeX()
 // Condense adjacent text nodes into a single text node
-function condenseTextNodes(nodes: Array<ParsedNode>): Array<ParsedNode> {
+function condenseTextNodes(nodes: ParsedNode[] | undefined): Array<ParsedNode> {
     const result: ParsedNode[] = [];
 
+    if (!nodes) {
+        return result;
+    }
+
     let currentText = "";
     for (const node of nodes) {
         if (node.type === "math") {