feat: Human-in-the-Loop (HITL) system implementation

.playground/test-manual-actions-e2e.ts

@@ -0,0 +1,242 @@
+/**
+ * E2E Test: Manual Action Components with Dynamic Templates
+ * 
+ * This script tests the complete manual action flow:
+ * 1. Creates a workflow with Manual Approval, Selection, and Form
+ * 2. Uses dynamic variables and templates in all of them
+ * 3. Runs the workflow with runtime inputs
+ * 4. Verifies the interpolated content in pending requests
+ * 5. Resolves each request via API
+ * 6. Verifies workflow completion
+ */
+
+const API_BASE = 'http://localhost:3211/api/v1';
+
+const HEADERS = {
+  'Content-Type': 'application/json',
+  'x-internal-token': 'local-internal-token',
+};
+
+async function sleep(ms: number): Promise<void> {
+  return new Promise(resolve => setTimeout(resolve, ms));
+}
+
+async function main() {
+  console.log('🚀 Starting E2E Manual Actions Test...\n');
+
+  const TEST_USER = 'betterclever';
+  const TEST_PROJECT = 'ShipSec-Studio-Refactor';
+
+  // 1. Create Workflow
+  console.log('📝 Creating multi-action workflow...');
+
+  const workflowGraph = {
+    name: 'E2E Manual Actions Test ' + Date.now(),
+    nodes: [
+      {
+        id: 'start',
+        type: 'core.workflow.entrypoint',
+        position: { x: 0, y: 0 },
+        data: {
+          label: 'Start',
+          config: {
+            runtimeInputs: [
+              { id: 'userName', label: 'User Name', type: 'string', required: true }
+            ]
+          },
+        },
+      },
+      {
+        id: 'logic',
+        type: 'core.logic.script',
+        position: { x: 200, y: 0 },
+        data: {
+          label: 'Prepare Data',
+          config: {
+            code: `export async function script(input: Input): Promise<Output> { return { projectName: "${TEST_PROJECT}" }; }`,
+            returns: [{ name: 'projectName', type: 'string' }]
+          },
+        },
+      },
+      {
+        id: 'approval',
+        type: 'core.manual_action.approval',
+        position: { x: 400, y: 0 },
+        data: {
+          label: 'Manual Approval',
+          config: {
+            title: 'Approve {{projectName}}',
+            description: 'Hello **{{userName}}**, please approve the release of **{{projectName}}**.',
+            variables: [
+              { name: 'userName', type: 'string' },
+              { name: 'projectName', type: 'string' }
+            ]
+          },
+        },
+      },
+      {
+        id: 'selection',
+        type: 'core.manual_action.selection',
+        position: { x: 600, y: 0 },
+        data: {
+          label: 'Manual Selection',
+          config: {
+            title: 'Select Role for {{userName}}',
+            description: 'Project context: {{projectName}}',
+            options: ['Admin', 'Editor', 'Viewer'],
+            variables: [
+              { name: 'userName', type: 'string' },
+              { name: 'projectName', type: 'string' }
+            ]
+          },
+        },
+      },
+      {
+        id: 'form',
+        type: 'core.manual_action.form',
+        position: { x: 800, y: 0 },
+        data: {
+          label: 'Manual Form',
+          config: {
+            title: 'Metadata for {{projectName}}',
+            description: 'Please provide details for **{{projectName}}** deployment.',
+            schema: {
+              type: 'object',
+              properties: {
+                environment: { type: 'string', enum: ['prod', 'staging'] },
+                nodes: { type: 'number', default: 3 }
+              },
+              required: ['environment']
+            },
+            variables: [
+              { name: 'projectName', type: 'string' }
+            ]
+          },
+        },
+      },
+    ],
+    edges: [
+      { id: 'e1', source: 'start', target: 'logic' },
+      { id: 'e2', source: 'logic', target: 'approval' },
+      { id: 'e3', source: 'approval', target: 'selection' },
+      { id: 'e4', source: 'selection', target: 'form' },
+
+      // Data connections
+      { id: 'd1', source: 'start', sourceHandle: 'userName', target: 'approval', targetHandle: 'userName' },
+      { id: 'd2', source: 'logic', sourceHandle: 'projectName', target: 'approval', targetHandle: 'projectName' },
+      { id: 'd3', source: 'start', sourceHandle: 'userName', target: 'selection', targetHandle: 'userName' },
+      { id: 'd4', source: 'logic', sourceHandle: 'projectName', target: 'selection', targetHandle: 'projectName' },
+      { id: 'd5', source: 'logic', sourceHandle: 'projectName', target: 'form', targetHandle: 'projectName' },
+    ],
+  };
+
+  let workflowId = '';
+  const createRes = await fetch(`${API_BASE}/workflows`, {
+    method: 'POST',
+    headers: HEADERS,
+    body: JSON.stringify(workflowGraph),
+  });
+  const wfData = await createRes.json();
+  workflowId = wfData.id;
+  console.log('   ✅ Workflow created:', workflowId);
+
+  // 2. Run Workflow
+  console.log('\n▶️ Running workflow with userName:', TEST_USER);
+  const runRes = await fetch(`${API_BASE}/workflows/${workflowId}/run`, {
+    method: 'POST',
+    headers: HEADERS,
+    body: JSON.stringify({ inputs: { userName: TEST_USER } })
+  });
+  const runData = await runRes.json();
+  const runId = runData.runId;
+  console.log('   ✅ Run started:', runId);
+
+  const resolveAction = async (expectedType: string, expectedTitle: string, responseData: any) => {
+    console.log(`\n🔍 Waiting for ${expectedType} request (runId=${runId})...`);
+    let action = null;
+    let lastFound = null;
+    for (let i = 0; i < 20; i++) {
+        await sleep(1500);
+        const res = await fetch(`${API_BASE}/human-inputs?runId=${runId}&status=pending`, { headers: HEADERS });
+        const list = await res.json();
+        lastFound = list;
+        action = list.find((a: any) => a.inputType === expectedType);
+        if (action) break;
+    }
+
+    if (!action) {
+        console.error(`❌ Timeout waiting for ${expectedType}. Pending actions in list:`, JSON.stringify(lastFound));
+        const statusRes = await fetch(`${API_BASE}/workflows/runs/${runId}/status`, { headers: HEADERS });
+        console.log('Run status:', await statusRes.json());
+        process.exit(1);
+    }
+
+    console.log(`   Found: "${action.title}"`);
+    if (action.title !== expectedTitle) {
+        console.error(`❌ Title mismatch! Expected: "${expectedTitle}", Got: "${action.title}"`);
+        process.exit(1);
+    }
+    console.log(`   Description check: ${action.description.substring(0, 50)}...`);
+    if (!action.description.includes(TEST_PROJECT) || !action.description.includes(TEST_USER)) {
+        if (expectedType !== 'form' || action.description.includes(TEST_PROJECT)) {
+             // Form only has projectName
+        } else {
+            console.error(`❌ Interpolation failed in description: ${action.description}`);
+            process.exit(1);
+        }
+    }
+
+    console.log(`✅ Resolving ${expectedType}...`);
+    const resolveRes = await fetch(`${API_BASE}/human-inputs/${action.id}/resolve`, {
+        method: 'POST',
+        headers: HEADERS,
+        body: JSON.stringify({ responseData })
+    });
+    if (!resolveRes.ok) {
+        console.error(`❌ Resolve failed:`, await resolveRes.text());
+        process.exit(1);
+    }
+    console.log(`   ✅ ${expectedType} resolved.`);
+  };
+
+  // 3. Resolve Manual Approval
+  await resolveAction('approval', `Approve ${TEST_PROJECT}`, { status: 'approved', comment: 'Looks good' });
+
+  // 4. Resolve Manual Selection
+  await resolveAction('selection', `Select Role for ${TEST_USER}`, { selection: 'Admin' });
+
+  // 5. Resolve Manual Form
+  await resolveAction('form', `Metadata for ${TEST_PROJECT}`, { environment: 'prod', nodes: 5 });
+
+  // 6. Wait for Completion
+  console.log('\n⏳ Waiting for completion...');
+  let status = 'RUNNING';
+  for (let i = 0; i < 20; i++) {
+    await sleep(1000);
+    const statusRes = await fetch(`${API_BASE}/workflows/runs/${runId}/status`, { headers: HEADERS });
+    const data = await statusRes.json();
+    status = data.status;
+    if (status !== 'RUNNING') break;
+    process.stdout.write('.');
+  }
+  console.log('\n🏁 Final Status:', status);
+
+  if (status === 'COMPLETED') {
+    console.log('\n🎉🎉🎉 E2E TEST PASSED! All manual actions interpolated and resolved correctly.');
+    console.log(`\nWorkflow ID: ${workflowId}`);
+    console.log(`Run ID: ${runId}`);
+  } else {
+    console.error('\n❌ Test failed with status:', status);
+    const resultRes = await fetch(`${API_BASE}/workflows/runs/${runId}/result`, { headers: HEADERS });
+    console.log('Error info:', await resultRes.text());
+    process.exit(1);
+  }
+
+  // Cleanup skipped as requested
+  console.log('\n🏁 Test finished. Cleanup skipped by user request.');
+}
+
+main().catch(e => {
+    console.error('Fatal error:', e);
+    process.exit(1);
+});

backend/drizzle/0017_create-approval-requests.sql

@@ -0,0 +1,59 @@
+-- Drop the old approval_requests table (v1, no legacy data)
+DROP TABLE IF EXISTS approval_requests;
+
+-- Drop old enum
+DROP TYPE IF EXISTS approval_status;
+
+-- Create new enum for human input status
+CREATE TYPE human_input_status AS ENUM ('pending', 'resolved', 'expired', 'cancelled');
+
+-- Create new enum for input types
+CREATE TYPE human_input_type AS ENUM ('approval', 'form', 'selection', 'review', 'acknowledge');
+
+-- Human Input Requests table - generalized HITL system
+CREATE TABLE human_input_requests (
+  -- Primary key
+  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
+
+  -- Workflow context
+  run_id TEXT NOT NULL,
+  workflow_id UUID NOT NULL,
+  node_ref TEXT NOT NULL,
+
+  -- Status
+  status human_input_status NOT NULL DEFAULT 'pending',
+
+  -- Input type and schema
+  input_type human_input_type NOT NULL DEFAULT 'approval',
+  input_schema JSONB NOT NULL DEFAULT '{}',
+
+  -- Display metadata
+  title TEXT NOT NULL,
+  description TEXT,
+  context JSONB DEFAULT '{}',
+
+  -- Secure token for public links
+  resolve_token TEXT NOT NULL UNIQUE,
+
+  -- Timeout handling
+  timeout_at TIMESTAMPTZ,
+
+  -- Response tracking
+  response_data JSONB,
+  responded_at TIMESTAMPTZ,
+  responded_by TEXT,
+
+  -- Multi-tenancy
+  organization_id VARCHAR(191),
+
+  -- Audit timestamps
+  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+-- Indexes for common queries
+CREATE INDEX idx_human_input_requests_status ON human_input_requests(status);
+CREATE INDEX idx_human_input_requests_run_id ON human_input_requests(run_id);
+CREATE INDEX idx_human_input_requests_workflow_id ON human_input_requests(workflow_id);
+CREATE INDEX idx_human_input_requests_organization_id ON human_input_requests(organization_id);
+CREATE INDEX idx_human_input_requests_resolve_token ON human_input_requests(resolve_token);

backend/scripts/generate-openapi.ts

@@ -10,12 +10,15 @@ import { cleanupOpenApiDoc } from 'nestjs-zod';
 async function generateOpenApi() {
   // Skip ingest services that require external connections during OpenAPI generation
   process.env.SKIP_INGEST_SERVICES = 'true';
+  process.env.SHIPSEC_SKIP_MIGRATION_CHECK = 'true';
 
   const { AppModule } = await import('../src/app.module');
 
+  console.log('Creating Nest app...');
   const app = await NestFactory.create(AppModule, {
-    logger: false,
+    logger: ['error', 'warn'],
   });
+  console.log('Nest app created');
 
   // Set global prefix to match production
   app.setGlobalPrefix('api/v1');
@@ -28,6 +31,7 @@ async function generateOpenApi() {
     .build();
 
   const document = SwaggerModule.createDocument(app, config);
+  console.log('Document paths keys:', Object.keys(document.paths).filter(k => k.includes('human')));
   const cleaned = cleanupOpenApiDoc(document);
   const repoRootSpecPath = join(__dirname, '..', '..', 'openapi.json');
   const payload = JSON.stringify(cleaned, null, 2);
@@ -36,7 +40,8 @@ async function generateOpenApi() {
   await app.close();
 }
 
-generateOpenApi().catch((error) => {
+console.log('Script started');
+generateOpenApi().then(() => console.log('Script finished successfully')).catch((error) => {
   console.error('Failed to generate OpenAPI spec', error);
   process.exit(1);
 });

backend/src/app.module.ts

@@ -19,6 +19,10 @@ import { IntegrationsModule } from './integrations/integrations.module';
 import { SchedulesModule } from './schedules/schedules.module';
 import { AnalyticsModule } from './analytics/analytics.module';
 
+import { ApiKeysModule } from './api-keys/api-keys.module';
+import { WebhooksModule } from './webhooks/webhooks.module';
+import { HumanInputsModule } from './human-inputs/human-inputs.module';
+
 const coreModules = [
   AgentsModule,
   AnalyticsModule,
@@ -32,13 +36,12 @@ const coreModules = [
   SchedulesModule,
   ApiKeysModule,
   WebhooksModule,
+  HumanInputsModule,
 ];
+
 const testingModules =
   process.env.NODE_ENV === 'production' ? [] : [TestingSupportModule];
 
-import { ApiKeysModule } from './api-keys/api-keys.module';
-import { WebhooksModule } from './webhooks/webhooks.module';
-
 @Module({
   imports: [
     ConfigModule.forRoot({

backend/src/components/utils/categorization.ts

@@ -8,7 +8,7 @@ interface ComponentCategoryConfig {
   icon: string;
 }
 
-const SUPPORTED_CATEGORIES: ReadonlyArray<ComponentCategory> = ['input', 'transform', 'ai', 'security', 'it_ops', 'notification', 'output'];
+const SUPPORTED_CATEGORIES: ReadonlyArray<ComponentCategory> = ['input', 'transform', 'ai', 'security', 'it_ops', 'notification', 'manual_action', 'output'];
 
 const COMPONENT_CATEGORY_CONFIG: Record<ComponentCategory, ComponentCategoryConfig> = {
   input: {
@@ -53,6 +53,13 @@ const COMPONENT_CATEGORY_CONFIG: Record<ComponentCategory, ComponentCategoryConf
     emoji: '🔔',
     icon: 'Bell',
   },
+  manual_action: {
+    label: 'Manual Action',
+    color: 'text-amber-600',
+    description: 'Human-in-the-loop interactions, approvals, and manual tasks',
+    emoji: '👤',
+    icon: 'UserCheck',
+  },
   output: {
     label: 'Output',
     color: 'text-green-600',