diff --git a/electron/handlers/ai.js b/electron/handlers/ai.js index 0449c8d..2280596 100644 --- a/electron/handlers/ai.js +++ b/electron/handlers/ai.js @@ -457,7 +457,7 @@ async function transcribeWithProvider(asrProvider, asrModel, audioBuffer) { // ============================================================ // IPC: ai:score — with per-provider retry + provider fallback chain // ============================================================ -const SYSTEM_PROMPT = 'You are an expert AI video editor for TikTok/Shorts/Reels. Return ONLY valid JSON.'; +const SYSTEM_PROMPT = 'You are an expert AI video editor for TikTok/Shorts/Reels. You understand pacing, engagement, and viral mechanics. Your task is to identify highly engaging clips that contain a strong Hook, a Build-up, and a Climax or Call to Action. For each clip, provide a detailed explanation of WHY it is viral. Return ONLY valid JSON matching the requested schema.'; /** * Extract an array of jpeg base64 frames from a video clip @@ -728,6 +728,47 @@ Return exactly this JSON format: } }); +async function suggestBRollInternal(transcript, durationMs) { + try { + const preferredProvider = await getSetting('ai_scoring_provider', config.DEFAULT_LLM_PROVIDER); + const model = await getSetting('ai_scoring_model', config.DEFAULT_LLM_MODEL); + + const promptText = `Analyze the following video transcript which is ${durationMs}ms long. +Identify at most 3 key moments (between 2 to 4 seconds long) that would benefit from visual B-roll overlay to keep the viewer engaged. +For each moment, provide a short specific keyword to search on Pexels (e.g., 'trading chart', 'happy people', 'coffee cup'). +Make sure the moments don't overlap, and avoid the very beginning (0-2s) where the hook happens. + +Transcript: +${JSON.stringify(transcript)} + +Return exactly this JSON format: +{ + "broll": [ + { "keyword": "...", "startMs": 10000, "endMs": 13000 } + ] +}`; + + const chain = buildLLMChain(preferredProvider); + let lastError; + + for (const provider of chain) { + try { + const result = await scoreWithProvider(provider, model, promptText); + return { success: true, result, usedProvider: provider }; + } catch (err) { + if (err.message.startsWith('no-key:')) continue; + lastError = err; + } + } + throw lastError || new Error('All LLM providers failed. Check API keys.'); + } catch (error) { + console.error('[AI] B-Roll suggestion failed:', error.message); + return { success: false, error: error.message }; + } +} + +module.exports = { suggestBRollInternal }; + // ============================================================ // IPC: ai:generateImage — Generate an image (DALL-E 3 default) // ============================================================ diff --git a/electron/handlers/broll.js b/electron/handlers/broll.js index 82f44f4..4207ab9 100644 --- a/electron/handlers/broll.js +++ b/electron/handlers/broll.js @@ -60,6 +60,45 @@ ipcMain.handle('broll:search', async (_, { keywords, orientation = 'portrait', p } }); +async function searchPexelsInternal(keywords, orientation = 'portrait', perPage = 5) { + try { + const pexelsKey = await keytar.getPassword(SERVICE_NAME, 'pexels_api_key'); + if (!pexelsKey) return { success: false, error: 'no key' }; + + const query = Array.isArray(keywords) ? keywords.slice(0, 3).join(' ') : keywords; + const url = `${PEXELS_API_URL}?query=${encodeURIComponent(query)}&orientation=${orientation}&per_page=${perPage}&size=small`; + + const res = await fetch(url, { headers: { 'Authorization': pexelsKey } }); + if (!res.ok) throw new Error(`API error`); + const data = await res.json(); + const videos = (data.videos || []).map(v => { + const file = v.video_files?.find(f => f.quality === 'sd') || v.video_files?.find(f => f.quality === 'hd') || v.video_files?.[0]; + return { id: v.id, url: v.url, downloadUrl: file?.link }; + }).filter(v => v.downloadUrl); + + return { success: true, videos }; + } catch (e) { + return { success: false, error: e.message }; + } +} + +async function downloadPexelsInternal(videoId, downloadUrl) { + try { + const cacheDir = await getDir('brollCache'); + const cacheFile = path.join(cacheDir, `broll_${videoId}.mp4`); + if (fs.existsSync(cacheFile)) return { success: true, localPath: cacheFile }; + const res = await fetch(downloadUrl); + if (!res.ok) throw new Error(`Download failed`); + const buffer = Buffer.from(await res.arrayBuffer()); + fs.writeFileSync(cacheFile, buffer); + return { success: true, localPath: cacheFile }; + } catch (e) { + return { success: false, error: e.message }; + } +} + +module.exports = { searchPexelsInternal, downloadPexelsInternal }; + // ── Download a Pexels video to local cache ─────────────────────── ipcMain.handle('broll:download', async (_, { videoId, downloadUrl }) => { try { diff --git a/electron/handlers/jobs.js b/electron/handlers/jobs.js index ea60b69..66f7bff 100644 --- a/electron/handlers/jobs.js +++ b/electron/handlers/jobs.js @@ -140,6 +140,40 @@ async function processNextJob() { if (job.type === 'RENDER') { broadcast('job:progress', { jobId: job.id, percent: 0, label: 'Starting render...' }); + if (payload.isAutopilot && (!payload.brollLayers || payload.brollLayers.length === 0)) { + try { + broadcast('job:progress', { jobId: job.id, percent: 10, label: 'Fetching Auto B-Roll...' }); + const transcriptText = (payload.segments || []).map(s => s.text).join(' '); + const durationMs = payload.endMs - payload.startMs; + + const aiModule = require('./ai'); + const brollModule = require('./broll'); + if (aiModule.suggestBRollInternal && brollModule.searchPexelsInternal) { + const brollSuggestionRes = await aiModule.suggestBRollInternal(transcriptText, durationMs); + if (brollSuggestionRes?.success && brollSuggestionRes.result?.broll) { + const brollList = []; + for (const b of brollSuggestionRes.result.broll) { + const searchRes = await brollModule.searchPexelsInternal(b.keyword, 'portrait', 1); + if (searchRes?.success && searchRes.videos?.length > 0) { + const dlRes = await brollModule.downloadPexelsInternal(searchRes.videos[0].id, searchRes.videos[0].downloadUrl); + if (dlRes?.success) { + brollList.push({ + id: Date.now() + Math.random().toString(), + path: dlRes.localPath, + startMs: b.startMs, + endMs: b.endMs, + }); + } + } + } + payload.brollLayers = brollList; + } + } + } catch (e) { + console.error("[Job Queue] Failed to apply auto b-roll: ", e); + } + } + // Derive outputPath if not in payload if (!payload.outputPath) { const clipsDir = await getDir('clips'); diff --git a/electron/handlers/render.js b/electron/handlers/render.js index 2a46343..0e38d44 100644 --- a/electron/handlers/render.js +++ b/electron/handlers/render.js @@ -75,16 +75,30 @@ ${hookText ? `Dialogue: 1,0:00:00.00,0:00:03.00,Hook,,0,0,0,,${hookText}\n` : '' const relStart = Math.max(0, seg.start - startSec); const relEnd = Math.max(0, seg.end - startSec); if (relEnd <= 0) continue; - let textLine = ''; + if (seg.words && seg.words.length > 0) { - for (const w of seg.words) { - const durCs = Math.round((w.end - w.start) * 100); - textLine += `{\\k${durCs}}${w.text} `; + for (let i = 0; i < seg.words.length; i++) { + const w = seg.words[i]; + const wStart = Math.max(0, w.start - startSec); + const wEnd = Math.max(0, w.end - startSec); + const durMs = Math.round((w.end - w.start) * 1000); + + let textLine = ''; + for (let j = 0; j < seg.words.length; j++) { + if (j === i) { + textLine += `{\\c&H00FFFF&\\fscx120\\fscy120\\t(0,${durMs/2},\\fscx100\\fscy100)}${seg.words[j].text} `; + } else if (j < i) { + textLine += `{\\c${primaryColor}\\fscx100\\fscy100}${seg.words[j].text} `; + } else { + textLine += `{\\c&HFFFFFF&\\fscx100\\fscy100}${seg.words[j].text} `; + } + } + assContent += `Dialogue: 0,${formatAssTime(wStart)},${formatAssTime(wEnd)},Default,,0,0,0,,${textLine.trim()}\n`; } } else { - textLine = seg.text; + const popAnim = `{\\fscx80\\fscy80\\t(0,150,\\fscx110\\fscy110)\\t(150,300,\\fscx100\\fscy100)}`; + assContent += `Dialogue: 0,${formatAssTime(relStart)},${formatAssTime(relEnd)},Default,,0,0,0,,${popAnim}${seg.text.trim()}\n`; } - assContent += `Dialogue: 0,${formatAssTime(relStart)},${formatAssTime(relEnd)},Default,,0,0,0,,${textLine.trim()}\n`; } fs.writeFileSync(assPath, assContent, 'utf8'); @@ -203,6 +217,17 @@ ${hookText ? `Dialogue: 1,0:00:00.00,0:00:03.00,Hook,,0,0,0,,${hookText}\n` : '' const px = '(iw-iw/zoom)/2'; const py = '(ih-ih/zoom)/2'; + filterChain.push(`${vOut}zoompan=z='${finalZ}':x='${px}':y='${py}':d=${durationSec*fps}:s=1080x1920:fps=${fps}[vzoom]`); + vOut = '[vzoom]'; + } else if ((options.format === '9:16' || !options.format) && options.autoZoom !== false) { + // Dynamic Auto-Zoom (CapCut style subtle zoom) + // If no specific keyframes, apply a very subtle slow zoom to keep the viewer engaged + const fps = 30; + // Zoom from 1.0 to 1.05 over the entire clip duration + const finalZ = `min(zoom+0.0015,1.1)`; + const px = '(iw-iw/zoom)/2'; + const py = '(ih-ih/zoom)/2'; + filterChain.push(`${vOut}zoompan=z='${finalZ}':x='${px}':y='${py}':d=${durationSec*fps}:s=1080x1920:fps=${fps}[vzoom]`); vOut = '[vzoom]'; } diff --git a/package-lock.json b/package-lock.json index cef9628..a3c527e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,7 +36,7 @@ "fluent-ffmpeg": "^2.1.3", "keytar": "^7.9.0", "lucide-react": "^0.575.0", - "next": "14.2.35", + "next": "^14.2.35", "next-intl": "^4.8.3", "next-themes": "^0.4.6", "react": "^18", @@ -12611,17 +12611,6 @@ } } }, - "node_modules/next-intl/node_modules/@swc/helpers": { - "version": "0.5.19", - "resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.19.tgz", - "integrity": "sha512-QamiFeIK3txNjgUTNppE6MiG3p7TdninpZu0E0PbqVh1a9FNLT2FRhisaa4NcaX52XVhA5l7Pk58Ft7Sqi/2sA==", - "license": "Apache-2.0", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.8.0" - } - }, "node_modules/next-themes": { "version": "0.4.6", "resolved": "https://registry.npmjs.org/next-themes/-/next-themes-0.4.6.tgz", diff --git a/package.json b/package.json index cea5dc1..56d3a2b 100644 --- a/package.json +++ b/package.json @@ -140,7 +140,7 @@ "fluent-ffmpeg": "^2.1.3", "keytar": "^7.9.0", "lucide-react": "^0.575.0", - "next": "14.2.35", + "next": "^14.2.35", "next-intl": "^4.8.3", "next-themes": "^0.4.6", "react": "^18", diff --git a/src/app/editor/page.tsx b/src/app/editor/page.tsx index 7f8a3db..b34024d 100644 --- a/src/app/editor/page.tsx +++ b/src/app/editor/page.tsx @@ -58,6 +58,7 @@ function EditorInner() { const [activePanel, setActivePanel] = useState<'text' | 'audio' | 'color' | 'effects' | 'transitions' | 'keyframes' | 'image' | null>('color'); const [selectedLayerId, setSelectedLayerId] = useState(null); const [draggingHandle, setDraggingHandle] = useState<'in' | 'out' | null>(null); + const [draggingTrack, setDraggingTrack] = useState<{ id: string; type: 'text' | 'broll'; handle: 'in' | 'out' } | null>(null); // Undo/Redo history const historyRef = useRef([]); @@ -212,26 +213,47 @@ function EditorInner() { // ── Timeline drag ───────────────────────────────────────────────────────── const handleTimelineDrag = useCallback((e: React.MouseEvent | React.PointerEvent) => { - const tl = timelineRef.current; if (!tl || !draggingHandle) return; + const tl = timelineRef.current; if (!tl) return; + if (!draggingHandle && !draggingTrack) return; e.preventDefault(); const rect = tl.getBoundingClientRect(); const ratio = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width)); const rawMs = initStart + ratio * (initEnd - initStart); setEditRaw(prev => { - if (draggingHandle === 'in') { - return { ...prev, startMs: Math.min(rawMs, prev.endMs - 500) }; - } else { - return { ...prev, endMs: Math.max(rawMs, prev.startMs + 500) }; + if (draggingHandle) { + if (draggingHandle === 'in') { + return { ...prev, startMs: Math.min(rawMs, prev.endMs - 500) }; + } else { + return { ...prev, endMs: Math.max(rawMs, prev.startMs + 500) }; + } + } else if (draggingTrack) { + const type = draggingTrack.type; + const id = draggingTrack.id; + const list = type === 'text' ? prev.textLayers : prev.brollLayers; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const newList = list.map((l: any) => { + if (l.id === id) { + if (draggingTrack.handle === 'in') { + return { ...l, startMs: Math.min(rawMs - prev.startMs, l.endMs - 500) }; + } else { + return { ...l, endMs: Math.max(rawMs - prev.startMs, l.startMs + 500) }; + } + } + return l; + }); + return { ...prev, [type === 'text' ? 'textLayers' : 'brollLayers']: newList }; } + return prev; }); - }, [draggingHandle, initStart, initEnd]); + }, [draggingHandle, draggingTrack, initStart, initEnd]); const stopDrag = useCallback(() => { - if (draggingHandle) { + if (draggingHandle || draggingTrack) { setEdit(prev => ({ ...prev })); // commit to history setDraggingHandle(null); + setDraggingTrack(null); } - }, [draggingHandle, setEdit]); + }, [draggingHandle, draggingTrack, setEdit]); const previewFilter = (() => { @@ -696,100 +718,133 @@ function EditorInner() { {/* TIMELINE */} -
-
- Timeline — drag handles to trim -
- - {/* Main video clip bar */} -
{ - if (draggingHandle) return; - const rect = timelineRef.current!.getBoundingClientRect(); - const ratio = (e.clientX - rect.left) / rect.width; - seekTo(ratio * (edit.endMs - edit.startMs)); - }} - > - {/* Clip region */} -
- - {/* IN handle */} -
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingHandle('in'); }} - > -
+
+
+
+ Timeline — Multi-track Editor
- - {/* OUT handle */} -
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingHandle('out'); }} - > -
+
+ {msToTime(initStart)} + {msToTime(initEnd)}
- - {/* Waveform Background Overlay */} - {waveformUrl && ( -
- )} - - {/* Playhead */} -
- - {/* Text layer markers */} - {edit.textLayers.map(l => ( -
- ))}
- {/* Duration labels */} -
- {msToTime(initStart)} - {msToTime(initEnd)} +
+ {/* Global Playhead */} +
+
+
+ + {/* Video Track */} +
+
Video
+
{ + if (draggingHandle) return; + const rect = timelineRef.current!.getBoundingClientRect(); + const ratio = (e.clientX - rect.left) / rect.width; + seekTo(ratio * (edit.endMs - edit.startMs)); + }} + > +
+
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingHandle('in'); }} + > +
+
+
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingHandle('out'); }} + > +
+
+ {waveformUrl && ( +
+ )} +
+
+ + {/* B-Roll Track */} + {edit.brollLayers.length > 0 && ( +
+
B-Roll
+
+ {edit.brollLayers.map(b => ( +
+
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingTrack({ id: b.id, type: 'broll', handle: 'in' }); }} /> + {b.path.split(/[\\/]/).pop()} +
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingTrack({ id: b.id, type: 'broll', handle: 'out' }); }} /> +
+ ))} +
+
+ )} + + {/* Text Track */} + {edit.textLayers.length > 0 && ( +
+
Text
+
+ {edit.textLayers.map(l => ( +
+
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingTrack({ id: l.id, type: 'text', handle: 'in' }); }} /> + {l.text} +
{ e.stopPropagation(); e.currentTarget.setPointerCapture(e.pointerId); setDraggingTrack({ id: l.id, type: 'text', handle: 'out' }); }} /> +
+ ))} +
+
+ )} + + {/* Audio Track Indication */} + {edit.audioTrack && ( +
+
Audio
+
+
+
+ {edit.audioTrack.path.split(/[\\/]/).pop()} +
+
+
+ )}
- - {/* Audio Track Indication */} - {edit.audioTrack && ( -
-
-
- - {edit.audioTrack.path.split(/[\\/]/).pop()} -
-
- )}