diff --git a/.changeset/rare-lamps-cough.md b/.changeset/rare-lamps-cough.md new file mode 100644 index 0000000000..7d1ce572a1 --- /dev/null +++ b/.changeset/rare-lamps-cough.md @@ -0,0 +1,6 @@ +--- +"@khanacademy/perseus": patch +"@khanacademy/perseus-editor": patch +--- + +[Locked Figure Labels] Util function to generate spoken math + use it within Locked Point aria labels diff --git a/packages/perseus-editor/package.json b/packages/perseus-editor/package.json index dbe7a6e947..6febce6869 100644 --- a/packages/perseus-editor/package.json +++ b/packages/perseus-editor/package.json @@ -40,6 +40,7 @@ "@khanacademy/math-input": "^21.1.4", "@khanacademy/perseus": "^41.2.0", "@khanacademy/perseus-core": "1.5.3", + "@khanacademy/pure-markdown": "^0.3.11", "mafs": "^0.19.0" }, "devDependencies": { diff --git a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx index 6fcba57fe0..a7e02bb6fc 100644 --- a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx +++ b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.test.tsx @@ -26,6 +26,14 @@ const defaultProps = { const defaultLabel = getDefaultFigureForType("label"); +// Mock the async function generateSpokenMathDetails +jest.mock("./util", () => ({ + ...jest.requireActual("./util"), + generateSpokenMathDetails: (input) => { + return Promise.resolve(`Spoken math details for ${input}`); + }, +})); + describe("LockedPointSettings", () => { let userEvent: UserEvent; beforeEach(() => { @@ -409,8 +417,11 @@ describe("LockedPointSettings", () => { await userEvent.click(autoGenButton); // Assert + // generateSpokenMathDetails is mocked to return the input string + // with "Spoken math details for " prepended. expect(onChangeProps).toHaveBeenCalledWith({ - ariaLabel: "Point at (0, 0). Appearance solid gray.", + ariaLabel: + "Spoken math details for Point at (0, 0). Appearance solid gray.", }); }); @@ -439,8 +450,11 @@ describe("LockedPointSettings", () => { await userEvent.click(autoGenButton); // Assert + // generateSpokenMathDetails is mocked to return the input string + // with "Spoken math details for " prepended. expect(onChangeProps).toHaveBeenCalledWith({ - ariaLabel: "Point A at (0, 0). Appearance solid gray.", + ariaLabel: + "Spoken math details for Point A at (0, 0). Appearance solid gray.", }); }); @@ -473,8 +487,11 @@ describe("LockedPointSettings", () => { await userEvent.click(autoGenButton); // Assert + // generateSpokenMathDetails is mocked to return the input string + // with "Spoken math details for " prepended. expect(onChangeProps).toHaveBeenCalledWith({ - ariaLabel: "Point A, B at (0, 0). Appearance solid gray.", + ariaLabel: + "Spoken math details for Point A, B at (0, 0). Appearance solid gray.", }); }); }); diff --git a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx index 28478fcab0..3bddf92d8b 100644 --- a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx +++ b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/locked-point-settings.tsx @@ -24,6 +24,7 @@ import LockedFigureSettingsActions from "./locked-figure-settings-actions"; import LockedLabelSettings from "./locked-label-settings"; import { generateLockedFigureAppearanceDescription, + generateSpokenMathDetails, getDefaultFigureForType, } from "./util"; @@ -104,28 +105,50 @@ const LockedPointSettings = (props: Props) => { const isDefiningPoint = !onMove && !onRemove; - /** - * Get a prepopulated aria label for the point. - * - * If the point has no labels, the aria label will just be - * "Point at (x, y)". - * - * If the point has labels, the aria label will be - * "Point at (x, y) with label1, label2, label3". - */ - function getPrepopulatedAriaLabel() { - let visiblelabel = ""; - if (labels && labels.length > 0) { - visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`; + const [prepopulatedAriaLabel, setPrepopulatedAriaLabel] = + React.useState(""); + + React.useEffect(() => { + /** + * Get a prepopulated aria label for the point, with the math + * details converted into spoken words. + * + * If the point has no labels, the aria label will just be + * "Point at (x, y)". + * + * If the point has labels, the aria label will be + * "Point label1, label2, label3 at (x, y)". + */ + async function getPrepopulatedAriaLabel() { + let visiblelabel = ""; + if (labels && labels.length > 0) { + visiblelabel += ` ${labels.map((l) => l.text).join(", ")}`; + } + + let str = await generateSpokenMathDetails( + `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`, + ); + + const pointAppearance = + generateLockedFigureAppearanceDescription(pointColor); + str += pointAppearance; + + return str; } - let str = `Point${visiblelabel} at (${coord[0]}, ${coord[1]})`; - const pointAppearance = - generateLockedFigureAppearanceDescription(pointColor); - str += pointAppearance; + // Guard against the race condition where we're getting the + // aria label from a previous render with different coords, + // labels, or pointColor. + let canceled = false; - return str; - } + getPrepopulatedAriaLabel().then( + (label) => !canceled && setPrepopulatedAriaLabel(label), + ); + + return () => { + canceled = true; + }; + }, [coord, labels, pointColor]); function handleColorChange(newValue) { const newProps: Partial = { @@ -247,7 +270,7 @@ const LockedPointSettings = (props: Props) => { { onChangeProps(newProps); }} diff --git a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts index 7d8aaedfc6..a09225a1e4 100644 --- a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts +++ b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.test.ts @@ -1,5 +1,6 @@ import { generateLockedFigureAppearanceDescription, + generateSpokenMathDetails, getDefaultFigureForType, } from "./util"; @@ -258,3 +259,84 @@ describe("generateLockedFigureAppearanceDescription", () => { }, ); }); + +// TODO(LEMS-2616): Update these tests to mock SpeechRuleEngine.setup() +// so that the tests don't have to make HTTP requests. +describe("generateMathDetails", () => { + test("should convert TeX to spoken language (root, fraction)", async () => { + const mathString = "$\\sqrt{\\frac{1}{2}}$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("StartRoot one half EndRoot"); + }); + + test("should convert TeX to spoken language (exponent)", async () => { + const mathString = "$x^{2}$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("x Superscript 2"); + }); + + test("should convert TeX to spoken language (negative)", async () => { + const mathString = "$-2$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("negative 2"); + }); + + test("should converte TeX to spoken language (subtraction)", async () => { + const mathString = "$2-1$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("2 minus 1"); + }); + + test("should convert TeX to spoken language (normal words)", async () => { + const mathString = "$\\text{square b}$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("square b"); + }); + + test("should convert TeX to spoken language (random letters)", async () => { + const mathString = "$cat$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("c a t"); + }); + + test("should keep non-math text as is", async () => { + const mathString = "Circle with radius $\\frac{1}{2}$ units"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("Circle with radius one half units"); + }); + + test("should read dollar signs as dollars inside tex", async () => { + const mathString = "This sandwich costs ${$}12.34$"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("This sandwich costs dollar sign 12.34"); + }); + + test("should read dollar signs as dollars outside tex", async () => { + const mathString = "This sandwich costs \\$12.34"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("This sandwich costs $12.34"); + }); + + test("should read curly braces", async () => { + const mathString = "Hello}{"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("Hello}{"); + }); + + test("should read backslashes", async () => { + const mathString = "\\"; + const convertedString = await generateSpokenMathDetails(mathString); + + expect(convertedString).toBe("\\"); + }); +}); diff --git a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts index cf4f49ecd6..b81723cecc 100644 --- a/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts +++ b/packages/perseus-editor/src/widgets/interactive-graph-editor/locked-figures/util.ts @@ -1,3 +1,5 @@ +import {SpeechRuleEngine} from "@khanacademy/mathjax-renderer"; +import * as SimpleMarkdown from "@khanacademy/pure-markdown"; import {UnreachableCaseError} from "@khanacademy/wonder-stuff-core"; import type { @@ -124,3 +126,29 @@ export function generateLockedFigureAppearanceDescription( throw new UnreachableCaseError(fill); } } + +export async function generateSpokenMathDetails(mathString: string) { + const engine = await SpeechRuleEngine.setup("en"); + let convertedSpeech = ""; + + // All the information we need is in the first section, + // whether it's typed as "blockmath" or "paragraph" + const firstSection = SimpleMarkdown.parse(mathString)[0]; + + // If it's blockMath, the outer level has the full math content. + if (firstSection.type === "blockMath") { + convertedSpeech += engine.texToSpeech(firstSection.content); + } + + // If it's a paragraph, we need to iterate through the sections + // to look for individual math blocks. + if (firstSection.type === "paragraph") { + for (const piece of firstSection.content) { + piece.type === "math" + ? (convertedSpeech += engine.texToSpeech(piece.content)) + : (convertedSpeech += piece.content); + } + } + + return convertedSpeech; +} diff --git a/packages/perseus/src/widgets/interactive-graphs/utils.ts b/packages/perseus/src/widgets/interactive-graphs/utils.ts index 08ac235102..bc25cc0f60 100644 --- a/packages/perseus/src/widgets/interactive-graphs/utils.ts +++ b/packages/perseus/src/widgets/interactive-graphs/utils.ts @@ -109,9 +109,13 @@ type ParsedNode = { // Helper function for replaceOutsideTeX() // Condense adjacent text nodes into a single text node -function condenseTextNodes(nodes: Array): Array { +function condenseTextNodes(nodes: ParsedNode[] | undefined): Array { const result: ParsedNode[] = []; + if (!nodes) { + return result; + } + let currentText = ""; for (const node of nodes) { if (node.type === "math") {