diff --git a/README.md b/README.md index 2b5eff4..f5a3170 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Open Generative UI -An open-source showcase for building rich, interactive AI-generated UI with [CopilotKit](https://copilotkit.ai) and [LangGraph](https://langchain-ai.github.io/langgraph/). Ask the agent to visualize algorithms, create 3D animations, render charts, or generate interactive diagrams — all rendered as live HTML/SVG inside a sandboxed iframe. +An open-source showcase for building rich, interactive AI-generated UI with [CopilotKit](https://copilotkit.ai) and [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview). Ask the agent to visualize algorithms, create 3D animations, render charts, or generate interactive diagrams — all rendered as live HTML/SVG inside a sandboxed iframe. https://github.com/user-attachments/assets/ed28c734-e54e-4412-873f-4801da544a7f @@ -104,14 +104,27 @@ Turborepo monorepo with three packages: ``` apps/ ├── app/ Next.js 16 frontend (CopilotKit v2, React 19, Tailwind 4) -├── agent/ LangGraph Python agent (GPT-5.4, CopilotKit middleware) +├── agent/ Deep Agent (deepagents + CopilotKit middleware, skills-based) └── mcp/ Standalone MCP server (design system + skills + document assembler) ``` +### Deep Agent + Skills + +The agent backend uses [LangChain Deep Agents](https://docs.langchain.com/oss/python/deepagents/overview) (`create_deep_agent`) with a skills-based architecture. Instead of injecting all visualization instructions into the system prompt, skills are defined as `SKILL.md` files in `apps/agent/skills/` and loaded on-demand via progressive disclosure: + +``` +apps/agent/skills/ +├── advanced-visualization/SKILL.md # UI mockups, dashboards, Chart.js, generative art +├── master-playbook/SKILL.md # Response philosophy, decision trees, narration patterns +└── svg-diagrams/SKILL.md # SVG generation rules, component patterns, diagram types +``` + +Deep agents also provide built-in planning (`write_todos`), filesystem tools, and sub-agent support. + ### How It Works 1. **User sends a prompt** via the CopilotKit chat UI -2. **Agent decides** whether to respond with text, call a tool, or render a visual component +2. **Deep agent decides** whether to respond with text, call a tool, or render a visual component — consulting relevant skills as needed 3. **`widgetRenderer`** — a frontend `useComponent` hook — receives the agent's HTML and renders it in a sandboxed iframe 4. **Skeleton loading** shows while the iframe loads, then content fades in smoothly 5. **ResizeObserver** inside the iframe reports content height back to the parent for seamless auto-sizing @@ -154,7 +167,7 @@ apps/ ## Tech Stack -Next.js 16, React 19, Tailwind CSS 4, LangGraph, CopilotKit v2, Turborepo, Recharts +Next.js 16, React 19, Tailwind CSS 4, LangChain Deep Agents, LangGraph, CopilotKit v2, Turborepo, Recharts ## License diff --git a/apps/agent/main.py b/apps/agent/main.py index 9d558e6..4c0c17c 100644 --- a/apps/agent/main.py +++ b/apps/agent/main.py @@ -4,26 +4,24 @@ """ import os +from pathlib import Path from copilotkit import CopilotKitMiddleware -from langchain.agents import create_agent +from deepagents import create_deep_agent from langchain_openai import ChatOpenAI from src.query import query_data from src.todos import AgentState, todo_tools from src.form import generate_form from src.templates import template_tools -from skills import load_all_skills -# Load all visualization skills -_skills_text = load_all_skills() - -agent = create_agent( +agent = create_deep_agent( model=ChatOpenAI(model=os.environ.get("LLM_MODEL", "gpt-5.4-2026-03-05")), tools=[query_data, *todo_tools, generate_form, *template_tools], middleware=[CopilotKitMiddleware()], - state_schema=AgentState, - system_prompt=f""" + context_schema=AgentState, + skills=[str(Path(__file__).parent / "skills")], + system_prompt=""" You are a helpful assistant that helps users understand CopilotKit and LangGraph used together. Be brief in your explanations of CopilotKit and LangGraph, 1 to 2 sentences. @@ -47,10 +45,6 @@ - Pre-styled form elements (buttons, inputs, sliders look native automatically) - Pre-built SVG CSS classes for color ramps (.c-purple, .c-teal, .c-blue, etc.) - Follow the skills below for how to produce high-quality visuals: - - {_skills_text} - ## UI Templates Users can save generated UIs as reusable templates and apply them later. diff --git a/apps/agent/pyproject.toml b/apps/agent/pyproject.toml index 5bb21d3..26ca9e5 100644 --- a/apps/agent/pyproject.toml +++ b/apps/agent/pyproject.toml @@ -16,4 +16,5 @@ dependencies = [ "copilotkit>=0.1.77", "langgraph-api>=0.7.16", "langchain-mcp-adapters>=0.2.1", + "deepagents>=0.1.0", ] diff --git a/apps/agent/skills/__init__.py b/apps/agent/skills/__init__.py deleted file mode 100644 index a871edf..0000000 --- a/apps/agent/skills/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Skill loader — reads .txt skill files from this directory -and exposes them for injection into the agent system prompt. -""" - -from pathlib import Path - - -_SKILLS_DIR = Path(__file__).parent - - -def load_skill(name: str) -> str: - """Load a single skill file by name (without extension).""" - path = _SKILLS_DIR / f"{name}.txt" - if not path.exists(): - raise FileNotFoundError(f"Skill file not found: {path}") - return path.read_text() - - -def load_all_skills() -> str: - """Load and concatenate all .txt skill files in this directory.""" - parts: list[str] = [] - for path in sorted(_SKILLS_DIR.glob("*.txt")): - parts.append(f"\n\n{'='*60}\n# SKILL: {path.stem}\n{'='*60}\n\n") - parts.append(path.read_text()) - return "".join(parts) diff --git a/apps/agent/skills/agent-skills-vol2.txt b/apps/agent/skills/advanced-visualization/SKILL.md similarity index 98% rename from apps/agent/skills/agent-skills-vol2.txt rename to apps/agent/skills/advanced-visualization/SKILL.md index a92d5e4..f40735e 100644 --- a/apps/agent/skills/agent-skills-vol2.txt +++ b/apps/agent/skills/advanced-visualization/SKILL.md @@ -1,3 +1,9 @@ +--- +name: "Advanced Visualization Techniques" +description: "UI mockups, dashboards, advanced interactivity, generative art, simulations, math visualizations, and design system rules for producing rich HTML widget output." +allowed-tools: [] +--- + # Agent Visualization Skills — Volume 2: Advanced Techniques Prerequisite: Volume 1 (SVG diagrams, basic interactive widgets, Chart.js, Mermaid). @@ -427,7 +433,7 @@ hardcoded hex values. ``` - Height goes on the wrapper div ONLY, never on canvas. - Always set `responsive: true, maintainAspectRatio: false`. -- For horizontal bar charts: height = (bars × 40) + 80 pixels. +- For horizontal bar charts: height = (bars x 40) + 80 pixels. ### Custom Legend (Always Use This) Disable Chart.js default legend and build HTML: @@ -450,7 +456,7 @@ plugins: { legend: { display: false } } ``` ### Dashboard Layout -Metric cards on top → chart below → sendPrompt for drill-down: +Metric cards on top -> chart below -> sendPrompt for drill-down: ```html
Answer in 1-2 sentences. + | + +- Is it conceptual / "how does X work"? + | +- Is it spatial or visual? -> SVG illustrative diagram + | +- Is it a process/flow? -> SVG flowchart or HTML stepper + | +- Is it data-driven? -> Interactive chart (Chart.js / Recharts) + | +- Is it abstract but explorable? -> Interactive HTML widget with controls + | + +- Is it "build me X"? -> Working code artifact, fully functional + | + +- Is it a comparison? -> Side-by-side table or comparative visual + | + +- Is it emotional/personal? -> Warm text response. No visuals needed. ``` ### The 3-Layer Response Pattern diff --git a/apps/agent/skills/svg-diagram-skill.txt b/apps/agent/skills/svg-diagrams/SKILL.md similarity index 94% rename from apps/agent/skills/svg-diagram-skill.txt rename to apps/agent/skills/svg-diagrams/SKILL.md index 2a28385..fedb655 100644 --- a/apps/agent/skills/svg-diagram-skill.txt +++ b/apps/agent/skills/svg-diagrams/SKILL.md @@ -1,3 +1,9 @@ +--- +name: "SVG Diagram Generation" +description: "Generating rich inline SVG diagrams to visually explain systems, processes, architectures, and abstract concepts." +allowed-tools: [] +--- + # SVG Diagram Generation Skill You can generate rich, inline SVG diagrams to visually explain concepts. Use this skill whenever a visual would help the user understand a system, process, architecture, or mechanism better than text alone. @@ -46,9 +52,9 @@ Always use this template: - **Sentence case always**. Never Title Case or ALL CAPS. ### Text Width Estimation -At 14px, each character ≈ 8px wide. At 12px, each character ≈ 7px wide. -- "Load Balancer" (13 chars) at 14px ≈ 104px → needs rect ≈ 140px wide (with padding). -- Always compute: `rect_width = max(title_chars × 8, subtitle_chars × 7) + 48px padding`. +At 14px, each character ~ 8px wide. At 12px, each character ~ 7px wide. +- "Load Balancer" (13 chars) at 14px ~ 104px -> needs rect ~ 140px wide (with padding). +- Always compute: `rect_width = max(title_chars x 8, subtitle_chars x 7) + 48px padding`. ### Colors (Light/Dark Mode Safe) Use these semantic color sets that work in both modes: @@ -169,7 +175,7 @@ If you're rendering inside a system that supports CSS variables, prefer: 1. **ViewBox height**: Find your lowest element (max y + height). Set H = that + 40px. 2. **No content past x=640 or below y=(H-40)**. -3. **Text fits in boxes**: `(char_count × 8) + 48 < rect_width` for 14px text. +3. **Text fits in boxes**: `(char_count x 8) + 48 < rect_width` for 14px text. 4. **No arrows through boxes**: Trace every line's path — if it crosses a rect, reroute. 5. **All `` connectors have `fill="none"`**. 6. **All text has appropriate fill color** — never rely on inheritance (SVG defaults to black). @@ -207,7 +213,7 @@ For complex topics, use multiple smaller SVGs instead of one dense one: HTTP POST /api/data - + @@ -219,7 +225,7 @@ For complex topics, use multiple smaller SVGs instead of one dense one: Validate and transform - + diff --git a/apps/app/src/components/generative-ui/widget-renderer.tsx b/apps/app/src/components/generative-ui/widget-renderer.tsx index 44ed82e..6a64a1a 100644 --- a/apps/app/src/components/generative-ui/widget-renderer.tsx +++ b/apps/app/src/components/generative-ui/widget-renderer.tsx @@ -369,7 +369,7 @@ window.addEventListener('message', function(e) { var scriptCloses = (rawHtml.match(/<\\/script>/gi) || []).length; var allScriptsClosed = scriptOpens <= scriptCloses; tmp.querySelectorAll('script').forEach(function(s) { - incomingScripts.push({ src: s.src, text: s.textContent }); + incomingScripts.push({ src: s.src, text: s.textContent, type: s.type || '' }); s.remove(); }); @@ -414,24 +414,36 @@ window.addEventListener('message', function(e) { content.innerHTML = tmp.innerHTML; } - // Execute only new scripts — skip entirely while a