feat: add auto subtitles with editable timeline

package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "@ffmpeg/ffmpeg": "^0.12.10",
     "@ffmpeg/util": "^0.12.2",
+    "@xenova/transformers": "^2.17.2",
     "clsx": "^2.1.1",
     "focus-trap-react": "^12.0.1",
     "jszip": "^3.10.1",

src/components/SubtitlesControl.tsx

@@ -0,0 +1,151 @@
+import { EditRecipe, Subtitle } from "@/lib/types";
+import { useSubtitles } from "@/hooks/useSubtitles";
+import { Wand2, X, Plus, Clock } from "lucide-react";
+import { cn } from "@/lib/utils";
+
+interface SubtitlesControlProps {
+  recipe: EditRecipe;
+  onChange: (patch: Partial<EditRecipe>) => void;
+  file: File | null;
+  duration: number;
+}
+
+export default function SubtitlesControl({ recipe, onChange, file, duration }: SubtitlesControlProps) {
+  const { isGenerating, progressText, progressPercent, error, generateSubtitles, cancelGeneration } = useSubtitles();
+
+  const handleGenerate = () => {
+    if (!file) return;
+    generateSubtitles(file, (newSubtitles) => {
+      // Append or replace? Let's replace for now, or append if user wants to.
+      // Auto generator should probably replace to avoid duplicates if they run it twice.
+      onChange({ subtitles: newSubtitles });
+    });
+  };
+
+  const updateSubtitle = (id: string, patch: Partial<Subtitle>) => {
+    onChange({
+      subtitles: recipe.subtitles.map(s => s.id === id ? { ...s, ...patch } : s)
+    });
+  };
+
+  const removeSubtitle = (id: string) => {
+    onChange({
+      subtitles: recipe.subtitles.filter(s => s.id !== id)
+    });
+  };
+
+  const addSubtitle = () => {
+    const newSub: Subtitle = {
+      id: `sub-${Date.now()}`,
+      text: "New subtitle",
+      startTime: 0,
+      endTime: 2,
+      x: -1,
+      y: 90,
+      fontSize: 48,
+      color: "#ffffff",
+      fontWeight: "bold",
+    };
+    onChange({ subtitles: [...(recipe.subtitles || []), newSub] });
+  };
+
+  return (
+    <div className="space-y-4">
+      <div className="flex justify-between items-center">
+        <button
+          type="button"
+          onClick={isGenerating ? cancelGeneration : handleGenerate}
+          disabled={!file}
+          className={cn(
+            "flex-1 flex items-center justify-center gap-2 py-2 px-4 rounded-lg text-sm font-semibold transition-colors",
+            isGenerating
+              ? "bg-[var(--error-bg)] text-[var(--error)] border border-[var(--error-border)] hover:bg-[var(--error-hover)]"
+              : "bg-film-100 text-film-700 border border-film-200 hover:bg-film-200"
+          )}
+        >
+          {isGenerating ? <X size={16} /> : <Wand2 size={16} />}
+          {isGenerating ? "Cancel Generation" : "Auto Generate AI Subtitles"}
+        </button>
+      </div>
+
+      {isGenerating && (
+        <div className="space-y-2 bg-[var(--bg)] p-3 rounded-lg border border-[var(--border)]">
+          <div className="flex justify-between text-xs text-[var(--muted)]">
+            <span>{progressText}</span>
+            {progressPercent > 0 && <span>{Math.round(progressPercent)}%</span>}
+          </div>
+          {progressPercent > 0 && (
+            <div className="h-1.5 w-full bg-[var(--border)] rounded-full overflow-hidden">
+              <div 
+                className="h-full bg-film-500 transition-all duration-300" 
+                style={{ width: `${progressPercent}%` }} 
+              />
+            </div>
+          )}
+        </div>
+      )}
+
+      {error && (
+        <div className="text-xs text-[var(--error)] bg-[var(--error-bg)] p-2 rounded border border-[var(--error-border)]">
+          {error}
+        </div>
+      )}
+
+      <div className="space-y-3 max-h-[300px] overflow-y-auto pr-1">
+        {recipe.subtitles?.map((sub, index) => (
+          <div key={sub.id} className="p-3 bg-[var(--bg)] border border-[var(--border)] rounded-lg space-y-2 relative group">
+            <button
+              onClick={() => removeSubtitle(sub.id)}
+              className="absolute top-2 right-2 p-1 text-[var(--muted)] hover:text-[var(--error)] opacity-0 group-hover:opacity-100 transition-opacity"
+              title="Remove subtitle"
+            >
+              <X size={14} />
+            </button>
+            <div className="flex items-center gap-2 text-xs text-[var(--muted)]">
+              <span className="font-mono bg-[var(--border)] px-1 rounded">{index + 1}</span>
+              <Clock size={12} />
+              <input
+                type="number"
+                step="0.1"
+                min="0"
+                max={duration}
+                value={sub.startTime.toFixed(1)}
+                onChange={(e) => updateSubtitle(sub.id, { startTime: parseFloat(e.target.value) || 0 })}
+                className="w-14 bg-transparent border-b border-transparent hover:border-[var(--border)] focus:border-film-500 outline-none px-1"
+              />
+              <span>to</span>
+              <input
+                type="number"
+                step="0.1"
+                min="0"
+                max={duration}
+                value={sub.endTime.toFixed(1)}
+                onChange={(e) => updateSubtitle(sub.id, { endTime: parseFloat(e.target.value) || 0 })}
+                className="w-14 bg-transparent border-b border-transparent hover:border-[var(--border)] focus:border-film-500 outline-none px-1"
+              />
+            </div>
+            <textarea
+              value={sub.text}
+              onChange={(e) => updateSubtitle(sub.id, { text: e.target.value })}
+              className="w-full bg-[var(--surface)] border border-[var(--border)] rounded p-2 text-sm text-[var(--text)] focus:border-film-500 outline-none resize-none"
+              rows={2}
+            />
+          </div>
+        ))}
+        {recipe.subtitles?.length === 0 && !isGenerating && (
+          <div className="text-center text-xs text-[var(--muted)] py-4">
+            No subtitles yet. Click above to generate or add manually.
+          </div>
+        )}
+      </div>
+
+      <button
+        onClick={addSubtitle}
+        className="w-full flex items-center justify-center gap-2 py-2 border border-dashed border-[var(--border)] rounded-lg text-xs text-[var(--muted)] hover:text-[var(--text)] hover:border-[var(--text)] transition-colors"
+      >
+        <Plus size={14} />
+        Add Manual Subtitle
+      </button>
+    </div>
+  );
+}

src/components/SubtitlesOverlay.tsx

@@ -0,0 +1,76 @@
+import { useEffect, useState, RefObject } from "react";
+import { EditRecipe, Subtitle } from "@/lib/types";
+import { cn } from "@/lib/utils";
+
+interface SubtitlesOverlayProps {
+  recipe: EditRecipe;
+  videoRef: RefObject<HTMLVideoElement | null>;
+  containerWidth: number;
+  containerHeight: number;
+}
+
+export default function SubtitlesOverlay({
+  recipe,
+  videoRef,
+  containerWidth,
+  containerHeight,
+}: SubtitlesOverlayProps) {
+  const [currentTime, setCurrentTime] = useState(0);
+
+  useEffect(() => {
+    const video = videoRef.current;
+    if (!video) return;
+
+    const handleTimeUpdate = () => {
+      setCurrentTime(video.currentTime);
+    };
+
+    video.addEventListener("timeupdate", handleTimeUpdate);
+    return () => {
+      video.removeEventListener("timeupdate", handleTimeUpdate);
+    };
+  }, [videoRef]);
+
+  if (!recipe.subtitles || recipe.subtitles.length === 0) {
+    return null;
+  }
+
+  // Find all active subtitles for current time
+  const activeSubtitles = recipe.subtitles.filter(
+    (sub) => currentTime >= sub.startTime && currentTime <= sub.endTime
+  );
+
+  if (activeSubtitles.length === 0) return null;
+
+  return (
+    <div className="absolute inset-0 pointer-events-none overflow-hidden">
+      {activeSubtitles.map((sub) => {
+        // Calculate position
+        const style: React.CSSProperties = {
+          position: "absolute",
+          fontSize: `${sub.fontSize}px`,
+          color: sub.color,
+          fontWeight: sub.fontWeight === "900" ? "900" : sub.fontWeight === "bold" ? "bold" : "normal",
+          textShadow: "2px 2px 4px rgba(0,0,0,0.8), -1px -1px 0 #000, 1px -1px 0 #000, -1px 1px 0 #000, 1px 1px 0 #000",
+          textAlign: "center",
+          whiteSpace: "pre-wrap",
+        };
+
+        if (sub.x === -1) {
+          style.left = "50%";
+          style.transform = "translateX(-50%)";
+        } else {
+          style.left = `${(sub.x / 100) * containerWidth}px`;
+        }
+
+        style.top = `${(sub.y / 100) * containerHeight}px`;
+
+        return (
+          <div key={sub.id} style={style} className="max-w-[90%]">
+            {sub.text}
+          </div>
+        );
+      })}
+    </div>
+  );
+}

src/components/VideoEditor.tsx

@@ -17,6 +17,7 @@ import ExportSettings from "./ExportSettings";
 import ExportOverlay from "./ExportOverlay";
 import DownloadResult from "./DownloadResult";
 import ImageOverlay from "./ImageOverlay"
+import SubtitlesControl from "./SubtitlesControl"
 import { getPresetById } from "@/lib/presets";
 
 import { cn } from "@/lib/utils";
@@ -228,6 +229,7 @@ export default function VideoEditor() {
   const [selectedTextId, setSelectedTextId] = useState<string | null>(null);
   const [openSections, setOpenSections] = useState({
     resize: true,
+    subtitles: false,
     trim: false,
     rotation: false,
     text: false,
@@ -722,6 +724,17 @@ export default function VideoEditor() {
                 </div>
               </AccordionSection>
 
+              <AccordionSection
+                id="subtitles"
+                icon={<Type size={12} />}
+                title="AI Subtitles"
+                isOpen={openSections.subtitles}
+                onToggle={() => toggleSection("subtitles")}
+                delay={60}
+              >
+                <SubtitlesControl recipe={recipe} onChange={updateRecipe} file={file} duration={duration} />
+              </AccordionSection>
+
               <div className="pt-2 flex justify-center items-center gap-6">
                 <button
                   type="button"

src/components/VideoPreview.tsx

@@ -8,6 +8,7 @@ import { cn } from "@/lib/utils";
 import { Camera } from "lucide-react";
 import ComparisonPreview from "./ComparisonPreview";
 import DraggableTextOverlays from "./DraggableTextOverlays";
+import SubtitlesOverlay from "./SubtitlesOverlay";
 
 interface Props {
   file: File | null;
@@ -283,6 +284,16 @@ export default function VideoPreview({
             onUpdateText={onUpdateText || (() => {})}
           />
         )}
+
+        {/* Subtitles Overlay */}
+        {recipe && !isLoading && containerDimensions.width > 0 && (
+          <SubtitlesOverlay
+            recipe={recipe}
+            videoRef={videoRef}
+            containerWidth={containerDimensions.width}
+            containerHeight={containerDimensions.height}
+          />
+        )}
 
         {/* Toggle button */}
         {recipe && !isLoading && (

src/hooks/useSubtitles.ts

@@ -0,0 +1,100 @@
+import { useState, useRef, useCallback } from "react";
+import { Subtitle } from "@/lib/types";
+
+export function useSubtitles() {
+  const [isGenerating, setIsGenerating] = useState(false);
+  const [progressText, setProgressText] = useState("");
+  const [progressPercent, setProgressPercent] = useState(0);
+  const [error, setError] = useState<string | null>(null);
+  const workerRef = useRef<Worker | null>(null);
+
+  const generateSubtitles = useCallback(async (file: File, onComplete: (subtitles: Subtitle[]) => void) => {
+    setIsGenerating(true);
+    setError(null);
+    setProgressText("Extracting audio...");
+    setProgressPercent(0);
+
+    try {
+      // 1. Extract audio from video file using Web Audio API
+      const audioContext = new (window.AudioContext || (window as any).webkitAudioContext)({ sampleRate: 16000 });
+      const arrayBuffer = await file.arrayBuffer();
+      const audioBuffer = await audioContext.decodeAudioData(arrayBuffer);
+      const audioData = audioBuffer.getChannelData(0); // Whisper expects mono audio
+
+      setProgressText("Loading AI model (this may take a minute)...");
+
+      // 2. Initialize worker
+      if (!workerRef.current) {
+        workerRef.current = new Worker(new URL('@/lib/whisper.worker.ts', import.meta.url), { type: 'module' });
+      }
+
+      const worker = workerRef.current;
+
+      worker.onmessage = (e) => {
+        const { type, status, info, output, message } = e.data;
+
+        if (type === 'progress') {
+          if (status) setProgressText(status);
+          if (info && info.progress) {
+            setProgressPercent(info.progress);
+          }
+        } else if (type === 'ready') {
+          setProgressText("Transcribing audio...");
+          setProgressPercent(0);
+          worker.postMessage({ type: 'generate', audio: audioData });
+        } else if (type === 'result') {
+          // Process output into subtitles
+          const chunks = output.chunks || [];
+          const newSubtitles: Subtitle[] = chunks.map((chunk: any, i: number) => {
+            const [start, end] = chunk.timestamp;
+            return {
+              id: `sub-${Date.now()}-${i}`,
+              text: chunk.text.trim(),
+              startTime: start,
+              endTime: end || start + 2, // fallback if end is null
+              x: -1, // center horizontally
+              y: 90, // bottom
+              fontSize: 48,
+              color: "#ffffff",
+              fontWeight: "bold",
+            };
+          });
+
+          onComplete(newSubtitles);
+          setIsGenerating(false);
+          setProgressText("");
+        } else if (type === 'error') {
+          setError(message);
+          setIsGenerating(false);
+        }
+      };
+
+      // Load model
+      worker.postMessage({ type: 'load' });
+
+    } catch (err: any) {
+      console.error(err);
+      setError(err.message || "Failed to generate subtitles.");
+      setIsGenerating(false);
+    }
+  }, []);
+
+  const cancelGeneration = useCallback(() => {
+    if (workerRef.current) {
+      workerRef.current.terminate();
+      workerRef.current = null;
+    }
+    setIsGenerating(false);
+    setProgressText("");
+    setProgressPercent(0);
+  }, []);
+
+  return {
+    isGenerating,
+    progressText,
+    progressPercent,
+    error,
+    generateSubtitles,
+    cancelGeneration,
+  };
+}