feat: add accessibility annotations to hotspots (#19)

trmquang93 · web-flow · commit 8ebcf1780234 · 2026-04-02T14:35:04.000+07:00
* feat: add accessibility annotations to hotspots

Allow users to annotate hotspots with accessibility metadata (label,
role, hint, traits) so generated instructions include VoiceOver/TalkBack
implementation guidance. Bumps file format to v12.

- Add collapsible Accessibility section to HotspotModal with label,
  role select, hint, and trait toggle pills
- Add platform-specific accessibility API mappings for SwiftUI, React
  Native, Flutter, and Jetpack Compose
- Render Accessibility table in screens.md and guidance in build-guide.md
- Add accessibility field to MCP server create/update hotspot tools
- Backfill accessibility: null for older .drawd files
- Save/restore accessibility data with hotspot presets
- Add 18 new tests covering rendering, migration, and integration

* refactor: simplify accessibility code after review

- Extract shared accessibilitySchema constant in hotspot-tools.js
  (was duplicated verbatim in create and update tool definitions)
- Fix stale closure bug in trait toggle onClick (use functional
  updater so rapid clicks don't lose state)
- Add per-platform docsUrl to ACCESSIBILITY_PLATFORM_MAP (was
  hardcoded to Apple docs URL for all platforms)
- Remove unused _platform parameter from renderAccessibilityBlock

---------

Co-authored-by: Quang Tran &lt;16215255+trmquang93@users.noreply.github.com&gt;
diff --git a/mcp-server/src/state.js b/mcp-server/src/state.js
@@ -243,6 +243,7 @@ export class FlowState {
       validation: hotspot.validation || null,
       transitionType: hotspot.transitionType || null,
       transitionLabel: hotspot.transitionLabel || "",
+      accessibility: hotspot.accessibility || null,
     };
 
     if (!screen.hotspots) screen.hotspots = [];
@@ -304,7 +305,7 @@ export class FlowState {
       "requestSchema", "responseSchema", "customDescription", "documentId",
       "conditions", "onSuccessAction", "onSuccessTargetId", "onSuccessCustomDesc",
       "onErrorAction", "onErrorTargetId", "onErrorCustomDesc",
-      "tbd", "tbdNote", "validation", "transitionType", "transitionLabel",
+      "tbd", "tbdNote", "validation", "transitionType", "transitionLabel", "accessibility",
     ];
     for (const key of allowed) {
       if (updates[key] !== undefined) {
diff --git a/mcp-server/src/tools/hotspot-tools.js b/mcp-server/src/tools/hotspot-tools.js
@@ -1,3 +1,25 @@
+const accessibilitySchema = {
+  type: "object",
+  properties: {
+    label: { type: "string", description: "Accessibility label (VoiceOver/TalkBack)" },
+    role: {
+      type: "string",
+      enum: ["button", "link", "image", "heading", "text", "search-field",
+             "toggle", "slider", "tab", "alert", "menu", "other"],
+    },
+    hint: { type: "string", description: "Usage hint (e.g. 'Double tap to sign in')" },
+    traits: {
+      type: "array",
+      items: {
+        type: "string",
+        enum: ["selected", "disabled", "adjustable", "header", "summary",
+               "plays-sound", "starts-media", "allows-direct-interaction"],
+      },
+    },
+  },
+  description: "Accessibility annotations for screen readers",
+};
+
 export const hotspotTools = [
   {
     name: "create_hotspot",
@@ -43,6 +65,7 @@ export const hotspotTools = [
         onErrorAction: { type: "string", enum: ["navigate", "back", "modal", "custom", ""] },
         onErrorTargetId: { type: "string" },
         onErrorCustomDesc: { type: "string" },
+        accessibility: accessibilitySchema,
       },
       required: ["screenId", "label", "x", "y", "w", "h", "action"],
     },
@@ -66,6 +89,7 @@ export const hotspotTools = [
         apiEndpoint: { type: "string" },
         apiMethod: { type: "string" },
         customDescription: { type: "string" },
+        accessibility: accessibilitySchema,
       },
       required: ["screenId", "hotspotId"],
     },
diff --git a/src/components/HotspotModal.jsx b/src/components/HotspotModal.jsx
@@ -2,7 +2,7 @@ import { useState, useEffect } from "react";
 import { COLORS, FONTS, styles } from "../styles/theme";
 import { generateId } from "../utils/generateId";
 import { DataFlowEditor } from "./DataFlowEditor";
-import { TRANSITION_TYPES } from "../constants";
+import { TRANSITION_TYPES, ACCESSIBILITY_ROLES, ACCESSIBILITY_TRAITS } from "../constants";
 
 function FollowUpSection({ title, titleColor, action, setAction, targetId, setTargetId,
                            customDesc, setCustomDesc, otherScreens, dataFlow, onDataFlowChange }) {
@@ -143,6 +143,13 @@ export function HotspotModal({ screen, hotspot, connection, screens, documents =
   const [validationPattern, setValidationPattern] = useState(hotspot?.validation?.pattern || "");
   const [validationErrorMessage, setValidationErrorMessage] = useState(hotspot?.validation?.errorMessage || "");
 
+  // Accessibility annotations
+  const [a11yExpanded, setA11yExpanded] = useState(!!(hotspot?.accessibility));
+  const [a11yLabel, setA11yLabel] = useState(hotspot?.accessibility?.label || "");
+  const [a11yRole, setA11yRole] = useState(hotspot?.accessibility?.role || "");
+  const [a11yHint, setA11yHint] = useState(hotspot?.accessibility?.hint || "");
+  const [a11yTraits, setA11yTraits] = useState(hotspot?.accessibility?.traits || []);
+
   useEffect(() => {
     const handleKeyDown = (e) => {
       if (e.key === "Escape") onClose();
@@ -200,6 +207,13 @@ export function HotspotModal({ screen, hotspot, connection, screens, documents =
               setW(preset.w ?? w);
               setH(preset.h ?? h);
               if (preset.customDescription) setCustomDescription(preset.customDescription);
+              if (preset.accessibility) {
+                setA11yExpanded(true);
+                setA11yLabel(preset.accessibility.label || "");
+                setA11yRole(preset.accessibility.role || "");
+                setA11yHint(preset.accessibility.hint || "");
+                setA11yTraits(preset.accessibility.traits || []);
+              }
             }}
             style={{ ...styles.select, flex: 1, marginTop: 0 }}
           >
@@ -222,6 +236,7 @@ export function HotspotModal({ screen, hotspot, connection, screens, documents =
                 w,
                 h,
                 customDescription,
+                accessibility: a11yExpanded ? { label: a11yLabel, role: a11yRole, hint: a11yHint, traits: a11yTraits } : null,
               };
               const updated = [...presets, preset];
               setPresets(updated);
@@ -307,6 +322,12 @@ export function HotspotModal({ screen, hotspot, connection, screens, documents =
               pattern: validationPattern,
               errorMessage: validationErrorMessage,
             } : null,
+            accessibility: a11yExpanded ? {
+              label: a11yLabel,
+              role: a11yRole,
+              hint: a11yHint,
+              traits: a11yTraits,
+            } : null,
           });
         }}>
           <div style={{ display: "flex", flexDirection: "column", gap: 14 }}>
@@ -738,6 +759,116 @@ export function HotspotModal({ screen, hotspot, connection, screens, documents =
               </div>
             )}
 
+            {/* Accessibility annotations */}
+            <div style={{
+              border: `1px solid ${a11yExpanded ? "rgba(198,120,221,0.25)" : COLORS.border}`,
+              borderRadius: 8,
+              overflow: "hidden",
+            }}>
+              <button
+                type="button"
+                onClick={() => setA11yExpanded(!a11yExpanded)}
+                style={{
+                  width: "100%",
+                  display: "flex",
+                  alignItems: "center",
+                  justifyContent: "space-between",
+                  padding: "10px 12px",
+                  background: a11yExpanded ? "rgba(198,120,221,0.06)" : "rgba(255,255,255,0.02)",
+                  border: "none",
+                  cursor: "pointer",
+                  fontFamily: FONTS.mono,
+                }}
+              >
+                <span style={{
+                  fontSize: 10,
+                  fontWeight: 700,
+                  letterSpacing: "0.08em",
+                  color: a11yExpanded ? "#c678dd" : COLORS.textMuted,
+                  textTransform: "uppercase",
+                }}>
+                  Accessibility
+                </span>
+                <span style={{ color: COLORS.textDim, fontSize: 12 }}>{a11yExpanded ? "\u25B2" : "\u25BC"}</span>
+              </button>
+              {a11yExpanded && (
+                <div style={{ padding: "0 12px 12px", display: "flex", flexDirection: "column", gap: 10 }}>
+                  <label style={{ ...styles.monoLabel, marginTop: 8 }}>
+                    ACCESSIBILITY LABEL
+                    <input
+                      value={a11yLabel}
+                      onChange={(e) => setA11yLabel(e.target.value)}
+                      placeholder="e.g. Sign in button"
+                      style={styles.input}
+                    />
+                  </label>
+                  <label style={styles.monoLabel}>
+                    ROLE
+                    <select value={a11yRole} onChange={(e) => setA11yRole(e.target.value)} style={styles.select}>
+                      <option value="">-- None --</option>
+                      {ACCESSIBILITY_ROLES.map((r) => (
+                        <option key={r} value={r}>{r}</option>
+                      ))}
+                    </select>
+                  </label>
+                  <label style={styles.monoLabel}>
+                    HINT
+                    <input
+                      value={a11yHint}
+                      onChange={(e) => setA11yHint(e.target.value)}
+                      placeholder="e.g. Double tap to sign in"
+                      style={styles.input}
+                    />
+                  </label>
+                  <div>
+                    <span style={{
+                      fontSize: 10,
+                      fontWeight: 700,
+                      letterSpacing: "0.08em",
+                      color: COLORS.textMuted,
+                      textTransform: "uppercase",
+                      fontFamily: FONTS.mono,
+                      display: "block",
+                      marginBottom: 6,
+                    }}>
+                      TRAITS
+                    </span>
+                    <div style={{ display: "flex", flexWrap: "wrap", gap: 6 }}>
+                      {ACCESSIBILITY_TRAITS.map((trait) => {
+                        const isActive = a11yTraits.includes(trait);
+                        return (
+                          <button
+                            key={trait}
+                            type="button"
+                            onClick={() => {
+                              setA11yTraits((prev) =>
+                                prev.includes(trait)
+                                  ? prev.filter((t) => t !== trait)
+                                  : [...prev, trait]
+                              );
+                            }}
+                            style={{
+                              padding: "3px 9px",
+                              borderRadius: 4,
+                              border: `1px solid ${isActive ? "rgba(198,120,221,0.35)" : COLORS.border}`,
+                              background: isActive ? "rgba(198,120,221,0.12)" : "rgba(255,255,255,0.05)",
+                              color: isActive ? "#c678dd" : COLORS.textDim,
+                              fontFamily: FONTS.mono,
+                              fontSize: 11,
+                              fontWeight: 600,
+                              cursor: "pointer",
+                            }}
+                          >
+                            {trait}
+                          </button>
+                        );
+                      })}
+                    </div>
+                  </div>
+                </div>
+              )}
+            </div>
+
             {/* Transition type — only visible when editing a connection-backed hotspot */}
             {connection && (action === "navigate" || action === "back" || action === "modal" || action === "api") && (
               <>
diff --git a/src/constants.js b/src/constants.js
@@ -5,7 +5,7 @@ export const GITHUB_URL = "https://github.com/codeflow-studio/drawd";
 export const DOMAIN = "drawd.app";
 
 // ── File Format ──────────────────────────────
-export const FILE_VERSION = 11;
+export const FILE_VERSION = 12;
 export const FILE_EXTENSION = ".drawd";
 export const LEGACY_FILE_EXTENSION = ".flowforge";
 export const DEFAULT_EXPORT_FILENAME = "flow-export";
@@ -89,3 +89,14 @@ export const TRANSITION_TYPES = [
 export const TRANSITION_LABELS = Object.fromEntries(
   TRANSITION_TYPES.map(t => [t.value, t.label])
 );
+
+// ── Accessibility ───────────────────────────
+export const ACCESSIBILITY_ROLES = [
+  "button", "link", "image", "heading", "text", "search-field",
+  "toggle", "slider", "tab", "alert", "menu", "other",
+];
+
+export const ACCESSIBILITY_TRAITS = [
+  "selected", "disabled", "adjustable", "header", "summary",
+  "plays-sound", "starts-media", "allows-direct-interaction",
+];
diff --git a/src/pages/docs/userGuide.md b/src/pages/docs/userGuide.md
@@ -145,6 +145,20 @@ When a screen has text-input hotspots, a yellow `Form` button appears in the scr
 > [!TIP]
 > Use the Form Summary to audit your form screens before generating instructions. Missing validations are easy to overlook in individual hotspot modals.
 
+### Accessibility annotations
+
+Every hotspot has an optional Accessibility section. Expand it to define how screen readers should announce the element:
+
+- **Label** -- What VoiceOver or TalkBack reads aloud (e.g. "Sign in button")
+- **Role** -- The semantic role: button, link, image, heading, text, search-field, toggle, slider, tab, alert, menu, or other
+- **Hint** -- A usage hint describing the result of activating the element (e.g. "Double tap to sign in")
+- **Traits** -- Additional traits: selected, disabled, adjustable, header, summary, plays-sound, starts-media, allows-direct-interaction
+
+When you generate instructions, screens with accessibility annotations include an **Accessibility** table listing each annotated element. If a specific platform is selected (SwiftUI, React Native, Flutter, Jetpack Compose), the build guide includes platform-specific implementation patterns for accessibility modifiers.
+
+> [!TIP]
+> Accessibility annotations are saved with presets, so you can reuse common patterns across hotspots.
+
 ## Connecting Screens
 
 Connections (navigation links) show how a user moves from one screen to another. They appear as curved arrows on the canvas between screen cards.
diff --git a/src/utils/buildPayload.test.js b/src/utils/buildPayload.test.js
@@ -6,9 +6,9 @@ describe("buildPayload", () => {
   const connections = [{ id: "c1" }];
   const documents = [{ id: "d1" }, { id: "d2" }, { id: "d3" }];
 
-  it("sets version to 11", () => {
+  it("sets version to 12", () => {
     const payload = buildPayload([], [], { x: 0, y: 0 }, 1);
-    expect(payload.version).toBe(11);
+    expect(payload.version).toBe(12);
   });
 
   it("sets metadata.screenCount to screens.length", () => {
diff --git a/src/utils/generateInstructionFiles.js b/src/utils/generateInstructionFiles.js
@@ -1,5 +1,5 @@
 import { analyzeNavGraph } from "./analyzeNavGraph.js";
-import { PLATFORM_TERMINOLOGY, renderHotspotDetailBlock, renderBuildGuideActionTable, renderBuildGuideTransitionTable } from "./instructionRenderers.js";
+import { PLATFORM_TERMINOLOGY, renderHotspotDetailBlock, renderBuildGuideActionTable, renderBuildGuideTransitionTable, renderAccessibilityBlock, renderAccessibilityGuidance } from "./instructionRenderers.js";
 import { screenReqId, connectionReqId } from "./generateReqIds.js";
 import { TRANSITION_LABELS } from "../constants.js";
 
@@ -501,6 +501,10 @@ function generateScreenDetailMd(s, screens, connections, images, documents = [])
         md += `> **Note:** ${missing.length} field(s) marked with \u26A0\uFE0F have no validation rules configured.\n\n`;
       }
     }
+
+    // Accessibility subsection
+    const a11yBlock = renderAccessibilityBlock(s.hotspots);
+    if (a11yBlock) md += a11yBlock;
   } else {
     md += `*No interactive elements defined*\n\n`;
   }
@@ -771,6 +775,9 @@ function generateBuildGuideMd(screens, connections, options, screenGroups = [])
     const transitionTable = renderBuildGuideTransitionTable(platform);
     if (transitionTable) md += transitionTable;
 
+    const a11yGuide = renderAccessibilityGuidance(platform);
+    if (a11yGuide) md += a11yGuide;
+
     md += `### Steps\n\n`;
     md += `1. Implement each screen from screens.md as a separate ${pt.name} view/component\n`;
     md += `2. For EACH screen, open its reference image from the \`images/\` folder and replicate the visual design exactly — colors, typography, spacing, layout, and component hierarchy\n`;
diff --git a/src/utils/generateInstructionFiles.test.js b/src/utils/generateInstructionFiles.test.js
@@ -218,4 +218,53 @@ describe("generateInstructionFiles", () => {
     expect(buildGuide.content).toContain("fullScreenCover");
     expect(buildGuide.content).toContain("slideUp");
   });
+
+  it("includes Accessibility section in screens.md when hotspots have accessibility data", () => {
+    const screenWithA11y = {
+      ...minimalScreen,
+      hotspots: [{
+        id: "h1",
+        label: "Login",
+        elementType: "button",
+        interactionType: "tap",
+        action: "navigate",
+        x: 10, y: 10, w: 80, h: 15,
+        accessibility: { label: "Sign in", role: "button", hint: "Double tap to sign in", traits: [] },
+      }],
+    };
+    const result = generateInstructionFiles([screenWithA11y], [], defaultOptions);
+    const screensFile = result.files.find((f) => f.name === "screens.md");
+    expect(screensFile.content).toContain("#### Accessibility");
+    expect(screensFile.content).toContain("Sign in");
+  });
+
+  it("does not include Accessibility section when no hotspots have accessibility data", () => {
+    const screenNoA11y = {
+      ...minimalScreen,
+      hotspots: [{
+        id: "h1",
+        label: "Button",
+        elementType: "button",
+        interactionType: "tap",
+        action: "navigate",
+        x: 10, y: 10, w: 80, h: 15,
+      }],
+    };
+    const result = generateInstructionFiles([screenNoA11y], [], defaultOptions);
+    const screensFile = result.files.find((f) => f.name === "screens.md");
+    expect(screensFile.content).not.toContain("#### Accessibility");
+  });
+
+  it("includes Accessibility guidance in build-guide.md for platform-specific output", () => {
+    const result = generateInstructionFiles([minimalScreen], [], { platform: "swiftui" });
+    const buildGuide = result.files.find((f) => f.name === "build-guide.md");
+    expect(buildGuide.content).toContain("### Accessibility");
+    expect(buildGuide.content).toContain(".accessibilityLabel");
+  });
+
+  it("does not include platform Accessibility guidance in build-guide.md for auto platform", () => {
+    const result = generateInstructionFiles([minimalScreen], [], { platform: "auto" });
+    const buildGuide = result.files.find((f) => f.name === "build-guide.md");
+    expect(buildGuide.content).not.toContain(".accessibilityLabel");
+  });
 });
diff --git a/src/utils/importFlow.js b/src/utils/importFlow.js
@@ -66,6 +66,7 @@ export function importFlow(fileText) {
         if (!Array.isArray(hs.dataFlow)) hs.dataFlow = [];
         if (!Array.isArray(hs.onSuccessDataFlow)) hs.onSuccessDataFlow = [];
         if (!Array.isArray(hs.onErrorDataFlow)) hs.onErrorDataFlow = [];
+        if (hs.accessibility === undefined) hs.accessibility = null;
 
         // v4 -> v5 migration: promote inline apiDocs to a document
         if (data.version < 5) {
diff --git a/src/utils/importFlow.test.js b/src/utils/importFlow.test.js
diff --git a/src/utils/instructionRenderers.js b/src/utils/instructionRenderers.js
diff --git a/src/utils/instructionRenderers.test.js b/src/utils/instructionRenderers.test.js
