diff --git a/.github/workflows/multi-model-eval.yml b/.github/workflows/multi-model-eval.yml index bc54c3ca..a528fe5d 100644 --- a/.github/workflows/multi-model-eval.yml +++ b/.github/workflows/multi-model-eval.yml @@ -25,8 +25,8 @@ jobs: concurrency: 20 - model: opus concurrency: 20 - - model: gpt-4.1 - concurrency: 5 + - model: gpt-5.4 + concurrency: 1 - model: gemini-2.5-pro concurrency: 20 steps: diff --git a/.gitignore b/.gitignore index 8fac3b22..c634653c 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ CLAUDE.local.md resources/deno/ /scripts/dist evals/promptfoo/results.json +evals/promptfoo/promptfooconfig.yaml evals/promptfoo/node_modules/ diff --git a/deno.json b/deno.json index edc1b161..1d9390d6 100644 --- a/deno.json +++ b/deno.json @@ -15,7 +15,7 @@ "audit": "deno run --allow-read --allow-net=api.osv.dev --allow-env=GITHUB_STEP_SUMMARY --allow-write scripts/audit_deps.ts", "audit-actions": "deno run --allow-read --allow-net=api.github.com --allow-env=GITHUB_STEP_SUMMARY,GITHUB_TOKEN --allow-write scripts/audit_actions.ts", "review-skills": "deno run --allow-read --allow-run --allow-env=GITHUB_STEP_SUMMARY --allow-write scripts/review_skills.ts", - "eval-skill-triggers": "deno run --allow-read --allow-run --allow-env --allow-write scripts/eval_skill_triggers_promptfoo.ts" + "eval-skill-triggers": "deno run --allow-read --allow-run --allow-env --allow-write --allow-net scripts/eval_skill_triggers_promptfoo.ts" }, "imports": { "@std/assert": "jsr:@std/assert@^1.0.18", diff --git a/evals/promptfoo/generate_config.ts b/evals/promptfoo/generate_config.ts index cf0181d6..3988a521 100644 --- a/evals/promptfoo/generate_config.ts +++ b/evals/promptfoo/generate_config.ts @@ -35,6 +35,7 @@ import { parse as parseYaml, stringify as stringifyYaml } from "@std/yaml"; interface ProviderDefinition { id: string; apiKeyEnv: string; + delay?: number; } const PROVIDER_REGISTRY: Record = { @@ -46,9 +47,10 @@ const PROVIDER_REGISTRY: Record = { id: "anthropic:messages:claude-opus-4-6", apiKeyEnv: "ANTHROPIC_API_KEY", }, - "gpt-4.1": { - id: "openai:gpt-4.1", + "gpt-5.4": { + id: "openai:gpt-5.4", apiKeyEnv: "OPENAI_API_KEY", + delay: 500, }, "gemini-2.5-pro": { id: "google:gemini-2.5-pro", @@ -206,7 +208,7 @@ async function main(): Promise { `try {`, ` const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str);`, ` const calls = Array.isArray(parsed) ? parsed : [parsed];`, - ` return calls.some(c => c.function?.name === needle || c.name === needle);`, + ` return calls.some(c => c.function?.name === needle || c.name === needle || c.functionCall?.name === needle);`, `} catch {}`, `return str.includes('"' + needle + '"');`, ].join("\n"), @@ -226,7 +228,7 @@ async function main(): Promise { `try {`, ` const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str);`, ` const calls = Array.isArray(parsed) ? parsed : [parsed];`, - ` return !calls.some(c => c.function?.name === needle || c.name === needle);`, + ` return !calls.some(c => c.function?.name === needle || c.name === needle || c.functionCall?.name === needle);`, `} catch {}`, `return !str.includes('"' + needle + '"');`, ].join("\n"), @@ -252,6 +254,7 @@ async function main(): Promise { systemMessage, tools, }, + ...(provider.delay ? { delay: provider.delay } : {}), }, ], tests, diff --git a/evals/promptfoo/package.json b/evals/promptfoo/package.json index 6b3ec606..5ce20dc6 100644 --- a/evals/promptfoo/package.json +++ b/evals/promptfoo/package.json @@ -8,6 +8,8 @@ "overrides": { "@anthropic-ai/sdk": "^0.81.0", "hono": ">=4.12.12", - "@hono/node-server": ">=1.19.13" + "@hono/node-server": ">=1.19.13", + "axios": ">=1.15.0", + "basic-ftp": ">=5.2.1" } } diff --git a/evals/promptfoo/promptfooconfig.yaml b/evals/promptfoo/promptfooconfig.yaml deleted file mode 100644 index ece0f79f..00000000 --- a/evals/promptfoo/promptfooconfig.yaml +++ /dev/null @@ -1,3031 +0,0 @@ -description: Swamp skill trigger routing evaluation (sonnet) -prompts: - - "{{query}}" -providers: - - id: "anthropic:messages:claude-sonnet-4-5" - config: - temperature: 0 - max_tokens: 200 - systemMessage: >- - You are a skill router for swamp, an AI-native automation framework. Your ONLY job is to route user requests to the correct skill by making a tool call. You MUST call exactly one tool for - every request. NEVER respond with text. NEVER ask clarifying questions. Even if the request is vague or missing details, route it to the best-matching skill based on the topic and keywords. - The skill itself will handle gathering any missing information from the user. - tools: - - type: function - function: - name: swamp-data - description: >- - Manage swamp model data — list data artifacts, view version history, delete expired versions, and run garbage collection. Use when working with swamp model data lifecycle, retention - policies, or version cleanup. Triggers on "swamp data", "model data", "data list", "data get", "data versions", "garbage collection", "gc", "data gc", "data retention", "data lifecycle", - "version history", "data cleanup", "prune data", "expire data", "ephemeral data". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-extension-datastore - description: >- - Create user-defined TypeScript datastores for swamp — implement DatastoreProvider, configure locking, health checks, and optional sync. Use when users want to extend swamp with custom - datastore backends. Triggers on "custom datastore", "extension datastore", "DatastoreProvider", "extensions/datastores", "create datastore", "new datastore type", "datastore plugin", - "datastore implementation". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-extension-driver - description: >- - Create user-defined TypeScript execution drivers for swamp — implement ExecutionDriver to control where and how model methods run. Use when users want custom execution environments - (remote servers, cloud functions, custom containers). Triggers on "custom driver", "extension driver", "execution driver", "ExecutionDriver", "extensions/drivers", "create driver", "new - driver type", "driver plugin", "remote execution", "driver implementation". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-extension-model - description: >- - Create, test, and publish extension models for swamp — define Zod schemas, implement model interfaces, smoke test against live APIs, write manifest.yaml, and push extensions to the - registry. Use when creating models, writing manifest.yaml, publishing/pushing extensions, testing extensions, or preparing models for the registry. Covers all extension types (models, - vaults, drivers, datastores, reports). Triggers on "create model", "new model type", "custom model", "extension model", "user model", "typescript model", "extend swamp", "build - integration", "zod schema", "model plugin", "deno model", "extensions/models", "model development", "implement model", "smoke test", "test extension", "verify model", "test against API", - "before push test", "push extension", "publish extension", "extension push", "release extension", "bump version", "publish to registry", "test extension from another repo", "source - extension loading", "manifest", "manifest.yaml", "write manifest", "prepare for publishing". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-extension-vault - description: >- - Create user-defined TypeScript vaults for swamp — implement VaultProvider to securely store and retrieve secrets. Use when users want to extend swamp with custom vault backends. Triggers - on "custom vault", "extension vault", "VaultProvider", "extensions/vaults", "create vault", "new vault type", "vault plugin", "vault implementation", "user-defined vault", "vault - provider". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-issue - description: >- - Create GitHub issues for swamp — file bug reports with reproduction steps or submit feature requests with implementation context. Use when the user wants to report a bug, request a - feature, or provide feedback about swamp. Triggers on "bug report", "feature request", "report bug", "request feature", "file bug", "submit bug", "swamp bug", "swamp feature", - "feedback", "report issue", "file issue". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-model - description: > - Work with existing swamp models — structured automation units that define typed schemas, methods (validate, transform, enrich), and outputs for data processing. Use when searching for - model types, describing schemas, creating inputs, running or executing methods, viewing outputs, or managing lifecycle (edit, delete). Do NOT use when the user wants to build, create, or - implement a custom model type, Zod schema, or TypeScript model — that is swamp-extension-model. Triggers on "swamp model", "model type", "model schema", "create input", "type search", - "type describe", "run method", "execute method", "validation method", "transform method", "enrichment model", "model validate", "model delete", "model edit", "model output", "output - logs", "output format", "CEL expression". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-repo - description: >- - Manage swamp repositories, datastores, and extension sources — initializing repos, upgrading swamp, syncing data, releasing stuck locks, loading extensions from external paths, and - installing swamp in CI. Use when initializing repos, upgrading swamp, starting the webapp, configuring datastores, managing extension sources, or setting up swamp in CI/CD pipelines. - Triggers on "repo", "repository", "init", "initialize", "swamp init", "setup swamp", "new swamp project", "upgrade swamp", "webapp", "swamp webapp", "repository structure", ".swamp - folder", "datastore", "datastore setup", "datastore status", "datastore sync", "datastore lock", "s3 datastore", "filesystem datastore", "stuck lock", "lock release", "install swamp", - "CI", "CI/CD", "GitHub Actions", "install in CI", "setup CI", "extension source", "extension sources", ".swamp-sources.yaml", "load extensions from", "local extension source", "add - source", "source add", "source rm", "source list". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-report - description: >- - Create, register, configure, and run reports for swamp models and workflows. Use when creating report extensions, configuring reports in definition YAML, running reports via CLI, or - viewing report output. Triggers on "report", "swamp report", "model report", "create report", "run report", "report extension", "report label", "skip report", "report output", "cost - report", "audit report", "workflow report", "report results". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-troubleshooting - description: > - Navigate swamp source code to trace error origins, identify execution flows, inspect internal data structures, and understand CLI behavior when --help is insufficient. Use this skill - whenever something is broken ("error", "failing", "not working", "crash", "timeout", "bug", "fix", "debug", "troubleshoot", "root cause", "stack trace", "isn't being found", "giving me - an error", "slow", "performance", "latency") OR when you need to understand how swamp works internally ("how does", "what happens when", "where is", "internals", "under the hood"). - Applies even when the query mentions a specific domain (e.g., "vault expressions aren't resolving" or "how does extension push work") — fetch swamp source to find the answer. - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-vault - description: >- - Manage swamp vaults for secure secret storage. Use when creating vaults, storing secrets, retrieving secrets, listing vault keys, or working with vault expressions in workflows. Triggers - on "vault", "secret", "secrets", "swamp vault", "store secret", "get secret", "vault expression", "aws secrets manager", "credential storage", "user-defined vault", "custom vault", - "vault implementation", "extensions/vaults", "vault provider", or vault-related CLI commands. - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query - - type: function - function: - name: swamp-workflow - description: >- - Work with swamp workflows for AI-native automation — define jobs and steps in YAML, wire models together with dependencies, validate DAGs, and inspect run history. Use when searching for - workflows, creating new workflows, validating workflow definitions, running workflows, or viewing run history. Triggers on "swamp workflow", "run workflow", "create workflow", - "automate", "automation", "orchestrate", "run history", "execute workflow", "workflow logs", "workflow failure", "debug workflow". - parameters: - type: object - properties: - query: - type: string - description: The user's request to process - required: - - query -tests: - - description: "[swamp-data] SHOULD trigger: Show me all the model data currently stored in swamp" - vars: - query: Show me all the model data currently stored in swamp - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: List the version history for the Customer model" - vars: - query: List the version history for the Customer model - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: How do I delete old data versions that are no longer needed?" - vars: - query: How do I delete old data versions that are no longer needed? - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: Run garbage collection to clean up expired data" - vars: - query: Run garbage collection to clean up expired data - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: What data retention policies are in place?" - vars: - query: What data retention policies are in place? - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: Get the latest version of the Payment data" - vars: - query: Get the latest version of the Payment data - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-data] SHOULD trigger: How do I prune ephemeral data that''s past its expiration?' - vars: - query: How do I prune ephemeral data that's past its expiration? - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: Configure data gc to run on a schedule" - vars: - query: Configure data gc to run on a schedule - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] SHOULD trigger: View the data lifecycle and retention settings" - vars: - query: View the data lifecycle and retention settings - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-data] should NOT trigger: I want to understand the schema fields in this model type" - vars: - query: I want to understand the schema fields in this model type - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-data] should NOT trigger: How do I create a custom input for the validation method?" - vars: - query: How do I create a custom input for the validation method? - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-data] should NOT trigger: Run the transform method on this dataset" - vars: - query: Run the transform method on this dataset - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-data] should NOT trigger: Help me set up a new datastore backend" - vars: - query: Help me set up a new datastore backend - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: '[swamp-data] should NOT trigger: Debug why my workflow isn''t reading the right data version' - vars: - query: Debug why my workflow isn't reading the right data version - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-data] should NOT trigger: Create a custom datastore for MongoDB" - vars: - query: Create a custom datastore for MongoDB - assert: - - type: javascript - value: |- - const needle = 'swamp-data'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: How do I implement a custom DatastoreProvider for MongoDB?" - vars: - query: How do I implement a custom DatastoreProvider for MongoDB? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Create a custom datastore backend for PostgreSQL" - vars: - query: Create a custom datastore backend for PostgreSQL - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Build a datastore that stores data in S3 buckets" - vars: - query: Build a datastore that stores data in S3 buckets - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: I want to extend swamp with a custom vector database datasto" - vars: - query: I want to extend swamp with a custom vector database datastore - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Guide me through creating a DatastoreProvider interface impl" - vars: - query: Guide me through creating a DatastoreProvider interface implementation - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Create a custom datastore for graph database support" - vars: - query: Create a custom datastore for graph database support - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Build a datastore backend for DynamoDB with locking support" - vars: - query: Build a datastore backend for DynamoDB with locking support - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Implement a datastore plugin for Redis with sync capabilitie" - vars: - query: Implement a datastore plugin for Redis with sync capabilities - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] SHOULD trigger: Create a custom datastore for Cassandra" - vars: - query: Create a custom datastore for Cassandra - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: Initialize a new swamp repository with the default datastore" - vars: - query: Initialize a new swamp repository with the default datastore - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: List all versions of my data stored in the datastore" - vars: - query: List all versions of my data stored in the datastore - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: Implement a custom vault provider for storing secrets" - vars: - query: Implement a custom vault provider for storing secrets - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: Datastore writes are failing intermittently" - vars: - query: Datastore writes are failing intermittently - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: How do I store my API key securely?" - vars: - query: How do I store my API key securely? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-datastore] should NOT trigger: Run garbage collection to clean up old data" - vars: - query: Run garbage collection to clean up old data - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-datastore'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: How do I create a custom execution driver for SSH?" - vars: - query: How do I create a custom execution driver for SSH? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Build a Lambda execution driver for running jobs on AWS" - vars: - query: Build a Lambda execution driver for running jobs on AWS - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: I want to create a custom driver that runs on Kubernetes" - vars: - query: I want to create a custom driver that runs on Kubernetes - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Guide me through implementing a remote execution driver" - vars: - query: Guide me through implementing a remote execution driver - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Create a custom driver for executing tasks on GCP Cloud Run" - vars: - query: Create a custom driver for executing tasks on GCP Cloud Run - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: How do I implement the ExecutionDriver interface in TypeScri" - vars: - query: How do I implement the ExecutionDriver interface in TypeScript? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Build a driver that executes workloads on a Docker Swarm clu" - vars: - query: Build a driver that executes workloads on a Docker Swarm cluster - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Create an extension driver for running code on a custom VM" - vars: - query: Create an extension driver for running code on a custom VM - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] SHOULD trigger: Implement a driver plugin for Nomad job orchestration" - vars: - query: Implement a driver plugin for Nomad job orchestration - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: Create a model type for processing Docker containers" - vars: - query: Create a model type for processing Docker containers - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: How do I orchestrate multiple remote jobs in a workflow?" - vars: - query: How do I orchestrate multiple remote jobs in a workflow? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: What custom datastores can I build?" - vars: - query: What custom datastores can I build? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: My SSH driver is timing out when executing remote tasks" - vars: - query: My SSH driver is timing out when executing remote tasks - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: Run this workflow step on a remote Kubernetes cluster" - vars: - query: Run this workflow step on a remote Kubernetes cluster - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-driver] should NOT trigger: Store the execution credentials in a secure vault" - vars: - query: Store the execution credentials in a secure vault - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-driver'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: How do I create a custom model using TypeScript and Zod?" - vars: - query: How do I create a custom model using TypeScript and Zod? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: I want to build a new model type that integrates with our in" - vars: - query: I want to build a new model type that integrates with our internal API - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Create a user-defined model for processing healthcare record" - vars: - query: Create a user-defined model for processing healthcare records - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: How do I extend swamp with a custom transformation model?" - vars: - query: How do I extend swamp with a custom transformation model? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Guide me through building a model that works with GraphQL" - vars: - query: Guide me through building a model that works with GraphQL - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Create a Zod schema for my custom data validation model" - vars: - query: Create a Zod schema for my custom data validation model - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Build a new model type for extracting entities from text" - vars: - query: Build a new model type for extracting entities from text - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Implement the model interface in TypeScript for managing Str" - vars: - query: Implement the model interface in TypeScript for managing Stripe subscriptions - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Create a deno model plugin for Salesforce CRM integration" - vars: - query: Create a deno model plugin for Salesforce CRM integration - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] should NOT trigger: What are the available model types already in swamp?" - vars: - query: What are the available model types already in swamp? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] should NOT trigger: How do I retrieve secrets from a vault in my model?" - vars: - query: How do I retrieve secrets from a vault in my model? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] should NOT trigger: Can I run this custom model as part of a scheduled workflow?" - vars: - query: Can I run this custom model as part of a scheduled workflow? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: '[swamp-extension-model] should NOT trigger: I''m getting a type error in my custom model code' - vars: - query: I'm getting a type error in my custom model code - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] should NOT trigger: Execute my custom model with this input data" - vars: - query: Execute my custom model with this input data - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] should NOT trigger: List all the data stored from running my models" - vars: - query: List all the data stored from running my models - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: How do I smoke test my extension model before pushing?" - vars: - query: How do I smoke test my extension model before pushing? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: I need to verify my extension model works before I push it" - vars: - query: I need to verify my extension model works before I push it - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: How do I test my extension model against a real API?" - vars: - query: How do I test my extension model against a real API? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-extension-model] SHOULD trigger: What''s the process for testing an extension model before pub' - vars: - query: What's the process for testing an extension model before publishing? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Help me create and test a CRUD extension model" - vars: - query: Help me create and test a CRUD extension model - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Make a manifest.yaml so I can push my extension" - vars: - query: Make a manifest.yaml so I can push my extension - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Publish my model to the registry" - vars: - query: Publish my model to the registry - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Help me push this extension to swamp-club" - vars: - query: Help me push this extension to swamp-club - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: I need to write a manifest for my model" - vars: - query: I need to write a manifest for my model - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: Prepare my extension for publishing" - vars: - query: Prepare my extension for publishing - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: How do I swamp extension push my model?" - vars: - query: How do I swamp extension push my model? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-model] SHOULD trigger: I want to release a new version of my extension" - vars: - query: I want to release a new version of my extension - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: How do I implement a custom VaultProvider for HashiCorp Vaul" - vars: - query: How do I implement a custom VaultProvider for HashiCorp Vault? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Create a vault backend that integrates with AWS Secrets Mana" - vars: - query: Create a vault backend that integrates with AWS Secrets Manager - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Build a custom vault implementation for Azure Key Vault" - vars: - query: Build a custom vault implementation for Azure Key Vault - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: I want to extend swamp with a custom vault provider" - vars: - query: I want to extend swamp with a custom vault provider - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Guide me through creating a VaultProvider interface implemen" - vars: - query: Guide me through creating a VaultProvider interface implementation - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Implement a custom vault that uses environment variables" - vars: - query: Implement a custom vault that uses environment variables - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Build a vault plugin for GCP Secret Manager" - vars: - query: Build a vault plugin for GCP Secret Manager - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Create a user-defined vault backend for Kubernetes secrets" - vars: - query: Create a user-defined vault backend for Kubernetes secrets - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] SHOULD trigger: Implement a vault provider for Hashicorp Consul" - vars: - query: Implement a vault provider for Hashicorp Consul - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: How do I store and retrieve secrets in the built-in vault?" - vars: - query: How do I store and retrieve secrets in the built-in vault? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: List all the secrets in my production vault" - vars: - query: List all the secrets in my production vault - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: Create a custom datastore backend for Cassandra" - vars: - query: Create a custom datastore backend for Cassandra - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: The vault provider is not connecting to Vault" - vars: - query: The vault provider is not connecting to Vault - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: How do I use vault expressions in my workflow?" - vars: - query: How do I use vault expressions in my workflow? - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-extension-vault] should NOT trigger: Build a Lambda driver for running on AWS" - vars: - query: Build a Lambda driver for running on AWS - assert: - - type: javascript - value: |- - const needle = 'swamp-extension-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: File a bug report — swamp vault get crashes with a segfault " - vars: - query: File a bug report — swamp vault get crashes with a segfault when the key contains special characters - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Report a feature request for dashboard improvements" - vars: - query: Report a feature request for dashboard improvements - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Create a GitHub issue — swamp workflow run silently skips st" - vars: - query: Create a GitHub issue — swamp workflow run silently skips steps when a dependency model has no data - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: File a bug report — swamp model get takes 30+ seconds when t" - vars: - query: File a bug report — swamp model get takes 30+ seconds when there are more than 100 data versions - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Submit an issue requesting support for PostgreSQL" - vars: - query: Submit an issue requesting support for PostgreSQL - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Report that the CLI is not respecting vault expressions" - vars: - query: Report that the CLI is not respecting vault expressions - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: File a bug about workflows not executing on schedule" - vars: - query: File a bug about workflows not executing on schedule - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Submit a feature request for better error messages" - vars: - query: Submit a feature request for better error messages - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] SHOULD trigger: Report an issue with data version handling" - vars: - query: Report an issue with data version handling - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-issue] should NOT trigger: Create and submit a pull request with my fix" - vars: - query: Create and submit a pull request with my fix - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-issue] should NOT trigger: Debug why my workflow is failing" - vars: - query: Debug why my workflow is failing - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-issue] should NOT trigger: List all the data versions I have" - vars: - query: List all the data versions I have - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-issue] should NOT trigger: Help me plan the architecture for a new feature" - vars: - query: Help me plan the architecture for a new feature - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-issue] should NOT trigger: How do I retrieve secrets from a vault?" - vars: - query: How do I retrieve secrets from a vault? - assert: - - type: javascript - value: |- - const needle = 'swamp-issue'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: Search for model types that handle payments" - vars: - query: Search for model types that handle payments - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: Describe the schema for the Order model including all fields" - vars: - query: Describe the schema for the Order model including all fields - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: How do I create an input for the validate method?" - vars: - query: How do I create an input for the validate method? - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: Run the validate method on the payment-processor model with " - vars: - query: Run the validate method on the payment-processor model with amount=100 and currency=USD - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: Show me the model output from the last execution" - vars: - query: Show me the model output from the last execution - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: How do CEL expressions work for referencing model data?" - vars: - query: How do CEL expressions work for referencing model data? - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: Execute the enrichment model on the customer-profile instanc" - vars: - query: Execute the enrichment model on the customer-profile instance to add geo data - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-model] SHOULD trigger: Use type search to find models matching ''deploy''' - vars: - query: Use type search to find models matching 'deploy' - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: View the output logs from the last model execution" - vars: - query: View the output logs from the last model execution - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] SHOULD trigger: How do I delete a model definition?" - vars: - query: How do I delete a model definition? - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-model] should NOT trigger: List all the data I have stored in the system" - vars: - query: List all the data I have stored in the system - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-model] should NOT trigger: I want to create a completely custom model type using TypeSc" - vars: - query: I want to create a completely custom model type using TypeScript - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-model] should NOT trigger: How do I chain this model into an automated workflow?" - vars: - query: How do I chain this model into an automated workflow? - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: '[swamp-model] should NOT trigger: The model is throwing a type error I can''t understand' - vars: - query: The model is throwing a type error I can't understand - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-model] should NOT trigger: Run a report on model performance over time" - vars: - query: Run a report on model performance over time - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-model] should NOT trigger: Build a Zod schema for my custom validation model" - vars: - query: Build a Zod schema for my custom validation model - assert: - - type: javascript - value: |- - const needle = 'swamp-model'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: How do I initialize a new swamp repository?" - vars: - query: How do I initialize a new swamp repository? - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: I need to upgrade swamp to the latest version" - vars: - query: I need to upgrade swamp to the latest version - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: Sync the datastore with the remote repository" - vars: - query: Sync the datastore with the remote repository - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: What lock mechanisms are available for concurrent access?" - vars: - query: What lock mechanisms are available for concurrent access? - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: Set up a new datastore for production use" - vars: - query: Set up a new datastore for production use - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: How do I upgrade the web application?" - vars: - query: How do I upgrade the web application? - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-repo] SHOULD trigger: Release a stuck lock that''s blocking datastore writes' - vars: - query: Release a stuck lock that's blocking datastore writes - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: Configure an S3 datastore for my swamp project" - vars: - query: Configure an S3 datastore for my swamp project - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: Check the datastore status and health" - vars: - query: Check the datastore status and health - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: How do I install swamp in a GitHub Actions CI pipeline?" - vars: - query: How do I install swamp in a GitHub Actions CI pipeline? - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] SHOULD trigger: Create a GitHub Actions workflow that runs swamp on every PR" - vars: - query: Create a GitHub Actions workflow that runs swamp on every PR - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: Create a custom model type for AWS Lambda" - vars: - query: Create a custom model type for AWS Lambda - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: List all the data versions in my system" - vars: - query: List all the data versions in my system - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: How do I define a workflow that runs nightly?" - vars: - query: How do I define a workflow that runs nightly? - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: The datastore sync is failing with an error" - vars: - query: The datastore sync is failing with an error - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: Help me plan the architecture for a new feature" - vars: - query: Help me plan the architecture for a new feature - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-repo] should NOT trigger: Implement a custom datastore backend for PostgreSQL" - vars: - query: Implement a custom datastore backend for PostgreSQL - assert: - - type: javascript - value: |- - const needle = 'swamp-repo'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: How do I create a report on model performance metrics?" - vars: - query: How do I create a report on model performance metrics? - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Run a report to show all workflow executions from last week" - vars: - query: Run a report to show all workflow executions from last week - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Generate a report extension for custom data analysis" - vars: - query: Generate a report extension for custom data analysis - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Create a report that tracks data model changes over time" - vars: - query: Create a report that tracks data model changes over time - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Can I export a report as CSV or JSON?" - vars: - query: Can I export a report as CSV or JSON? - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Run a compliance report for all stored secrets" - vars: - query: Run a compliance report for all stored secrets - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Configure report labels in the model definition YAML" - vars: - query: Configure report labels in the model definition YAML - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Skip a report from running in a specific workflow" - vars: - query: Skip a report from running in a specific workflow - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Show me the report results from the last automation run" - vars: - query: Show me the report results from the last automation run - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: List the version history for the Payment model" - vars: - query: List the version history for the Payment model - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: I need to define a multi-step automation that runs daily" - vars: - query: I need to define a multi-step automation that runs daily - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: Help me debug an issue with my model output" - vars: - query: Help me debug an issue with my model output - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: Create a custom execution driver for running on Kubernetes" - vars: - query: Create a custom execution driver for running on Kubernetes - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: Execute the processing workflow and show logs" - vars: - query: Execute the processing workflow and show logs - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] should NOT trigger: The report generation is timing out" - vars: - query: The report generation is timing out - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: How do I read execution data in a report using dataHandles?" - vars: - query: How do I read execution data in a report using dataHandles? - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: Access dataRepository from report context to get model outpu" - vars: - query: Access dataRepository from report context to get model output - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-report] SHOULD trigger: What methods does UnifiedDataRepository expose in reports?" - vars: - query: What methods does UnifiedDataRepository expose in reports? - assert: - - type: javascript - value: |- - const needle = 'swamp-report'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: Debug why my deploy-infra workflow is failing on the second " - vars: - query: Debug why my deploy-infra workflow is failing on the second step with error 'model not found' - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: Swamp is throwing an unexpected type error" - vars: - query: Swamp is throwing an unexpected type error - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-troubleshooting] SHOULD trigger: Help me diagnose why vault expressions aren''t being resolved' - vars: - query: Help me diagnose why vault expressions aren't being resolved - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: The datastore sync is hanging and timing out" - vars: - query: The datastore sync is hanging and timing out - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-troubleshooting] SHOULD trigger: Fix the issue where my custom model isn''t being found' - vars: - query: Fix the issue where my custom model isn't being found - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-troubleshooting] SHOULD trigger: swamp model get is giving me ''Error: ENOENT: no such file or' - vars: - query: 'swamp model get is giving me ''Error: ENOENT: no such file or directory'' but the model exists' - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: Trace the swamp execution flow to find where the data pipeli" - vars: - query: Trace the swamp execution flow to find where the data pipeline failed - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: Read the swamp source code to understand why vault.get retur" - vars: - query: Read the swamp source code to understand why vault.get returns undefined for keys that exist - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: Why is the workflow not reading the correct data version?" - vars: - query: Why is the workflow not reading the correct data version? - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] SHOULD trigger: My model method is crashing with a stack trace, help me find" - vars: - query: "My model method is crashing with a stack trace, help me find the root cause" - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: File a bug report for this issue on GitHub" - vars: - query: File a bug report for this issue on GitHub - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: How do I define a workflow that handles errors gracefully?" - vars: - query: How do I define a workflow that handles errors gracefully? - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: List all the data stored in my system" - vars: - query: List all the data stored in my system - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: Create a pull request to fix the swamp core" - vars: - query: Create a pull request to fix the swamp core - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: How do I store my database password securely?" - vars: - query: How do I store my database password securely? - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: Show me the execution logs from the last workflow run" - vars: - query: Show me the execution logs from the last workflow run - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-troubleshooting] should NOT trigger: Format the output of this CLI command" - vars: - query: Format the output of this CLI command - assert: - - type: javascript - value: |- - const needle = 'swamp-troubleshooting'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: How do I store my database password securely in swamp?" - vars: - query: How do I store my database password securely in swamp? - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Retrieve the API key from my production vault" - vars: - query: Retrieve the API key from my production vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Create a new vault and add my AWS credentials to it" - vars: - query: Create a new vault and add my AWS credentials to it - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: List all secrets stored in the vault" - vars: - query: List all secrets stored in the vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: How do I reference a vault secret inside a workflow step?" - vars: - query: How do I reference a vault secret inside a workflow step? - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Delete an expired credential from the vault" - vars: - query: Delete an expired credential from the vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: I need to store a token so my models can authenticate with a" - vars: - query: I need to store a token so my models can authenticate with an external API - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: How do I use vault expressions like vault.get in my workflow" - vars: - query: How do I use vault expressions like vault.get in my workflow YAML? - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Put a secret called db-password into my vault" - vars: - query: Put a secret called db-password into my vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Which vault providers does swamp support out of the box?" - vars: - query: Which vault providers does swamp support out of the box? - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] SHOULD trigger: Get the OAuth token from my vault for the integration" - vars: - query: Get the OAuth token from my vault for the integration - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: I want to build a custom vault provider that talks to HashiC" - vars: - query: I want to build a custom vault provider that talks to HashiCorp Vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: Implement VaultProvider interface for Azure Key Vault" - vars: - query: Implement VaultProvider interface for Azure Key Vault - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: Show me the version history of my stored model data" - vars: - query: Show me the version history of my stored model data - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: Create a new TypeScript model type for DNS management" - vars: - query: Create a new TypeScript model type for DNS management - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: The vault command keeps crashing with a stack trace" - vars: - query: The vault command keeps crashing with a stack trace - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-vault] should NOT trigger: Run the deploy workflow and show me the logs" - vars: - query: Run the deploy workflow and show me the logs - assert: - - type: javascript - value: |- - const needle = 'swamp-vault'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: How do I define a workflow in YAML that orchestrates multipl" - vars: - query: How do I define a workflow in YAML that orchestrates multiple models? - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Show me the run history for the deployment workflow" - vars: - query: Show me the run history for the deployment workflow - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Wire together the validation and processing models in a DAG" - vars: - query: Wire together the validation and processing models in a DAG - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: '[swamp-workflow] SHOULD trigger: Validate that my workflow doesn''t have circular dependencies' - vars: - query: Validate that my workflow doesn't have circular dependencies - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Create an automated job that runs the data sync every night" - vars: - query: Create an automated job that runs the data sync every night - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Inspect the execution logs from the last workflow run" - vars: - query: Inspect the execution logs from the last workflow run - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Run the production workflow and monitor for failures" - vars: - query: Run the production workflow and monitor for failures - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Define dependencies between workflow steps" - vars: - query: Define dependencies between workflow steps - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] SHOULD trigger: Execute the nightly-sync workflow and show me the results" - vars: - query: Execute the nightly-sync workflow and show me the results - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: List all stored secrets and their expiration dates" - vars: - query: List all stored secrets and their expiration dates - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: I want to create a custom SSH execution driver" - vars: - query: I want to create a custom SSH execution driver - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: How do I describe the schema for this data model?" - vars: - query: How do I describe the schema for this data model? - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: The workflow is erroring on the second step" - vars: - query: The workflow is erroring on the second step - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: Clean up old data versions from the system" - vars: - query: Clean up old data versions from the system - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: Generate a cost report for workflow executions" - vars: - query: Generate a cost report for workflow executions - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); - - description: "[swamp-workflow] should NOT trigger: How do I install swamp in CI to run workflows on every PR?" - vars: - query: How do I install swamp in CI to run workflows on every PR? - assert: - - type: javascript - value: |- - const needle = 'swamp-workflow'; - const str = typeof output === 'string' ? output : JSON.stringify(output); - try { - const parsed = typeof output === 'object' ? (Array.isArray(output) ? output : [output]) : JSON.parse(str); - const calls = Array.isArray(parsed) ? parsed : [parsed]; - return !calls.some(c => c.function?.name === needle || c.name === needle); - } catch {} - return !str.includes('"' + needle + '"'); diff --git a/scripts/eval_skill_triggers_promptfoo.ts b/scripts/eval_skill_triggers_promptfoo.ts index 3fb368c9..c128cbcb 100644 --- a/scripts/eval_skill_triggers_promptfoo.ts +++ b/scripts/eval_skill_triggers_promptfoo.ts @@ -27,11 +27,11 @@ * * Usage: deno run eval-skill-triggers [--model ] [--concurrency ] [--threshold <0.0-1.0>] * - * Supported models: sonnet (default), opus, gpt-4.1, gemini-2.5-pro + * Supported models: sonnet (default), opus, gpt-5.4, gemini-2.5-pro * * Environment: * ANTHROPIC_API_KEY - Required for sonnet/opus models. - * OPENAI_API_KEY - Required for gpt-4.1 model. + * OPENAI_API_KEY - Required for gpt-5.4 model. * GOOGLE_API_KEY - Required for gemini-2.5-pro model. */ @@ -41,15 +41,105 @@ import { join } from "@std/path"; const API_KEY_ENV: Record = { "sonnet": "ANTHROPIC_API_KEY", "opus": "ANTHROPIC_API_KEY", - "gpt-4.1": "OPENAI_API_KEY", + "gpt-5.4": "OPENAI_API_KEY", "gemini-2.5-pro": "GOOGLE_API_KEY", }; +// Maps model aliases to the API model ID used for preflight checks. +const PREFLIGHT_MODEL_ID: Record = { + "sonnet": "claude-sonnet-4-5", + "opus": "claude-opus-4-6", + "gpt-5.4": "gpt-5.4", + "gemini-2.5-pro": "gemini-2.5-pro", +}; + +interface PreflightConfig { + url: string; + headers: Record; + body: string; +} + +function buildPreflightRequest( + model: string, + apiKey: string, +): PreflightConfig { + const modelId = PREFLIGHT_MODEL_ID[model]; + if (model === "sonnet" || model === "opus") { + return { + url: "https://api.anthropic.com/v1/messages", + headers: { + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + "content-type": "application/json", + }, + body: JSON.stringify({ + model: modelId, + max_tokens: 1, + messages: [{ role: "user", content: "hi" }], + }), + }; + } else if (model === "gpt-5.4") { + return { + url: "https://api.openai.com/v1/chat/completions", + headers: { + "Authorization": `Bearer ${apiKey}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + model: modelId, + max_completion_tokens: 10, + messages: [{ role: "user", content: "hi" }], + }), + }; + } else { + return { + url: + `https://generativelanguage.googleapis.com/v1beta/models/${modelId}:generateContent?key=${apiKey}`, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + contents: [{ parts: [{ text: "hi" }] }], + generationConfig: { maxOutputTokens: 1 }, + }), + }; + } +} + +async function preflightCheck(model: string): Promise { + const apiKey = Deno.env.get(API_KEY_ENV[model])!; + const config = buildPreflightRequest(model, apiKey); + + console.log(`Preflight: verifying ${model} API access…`); + const signal = AbortSignal.timeout(15_000); + const resp = await fetch(config.url, { + method: "POST", + headers: config.headers, + body: config.body, + signal, + }); + + if (!resp.ok) { + const body = await resp.text(); + console.error( + `Preflight failed for ${model} (HTTP ${resp.status}): ${body.slice(0, 300)}`, + ); + if (resp.status === 429) { + console.error( + "Hint: a 429 before any eval calls usually means insufficient credits or a billing issue, not a rate limit.", + ); + } + Deno.exit(1); + } + + // Drain the response body + await resp.text(); + console.log(`Preflight: ${model} API access verified.`); +} + // Per-million-token pricing for cost estimation const TOKEN_PRICING: Record = { "sonnet": { prompt: 3.0, completion: 15.0 }, "opus": { prompt: 15.0, completion: 75.0 }, - "gpt-4.1": { prompt: 2.0, completion: 8.0 }, + "gpt-5.4": { prompt: 2.0, completion: 8.0 }, "gemini-2.5-pro": { prompt: 1.25, completion: 10.0 }, }; @@ -182,6 +272,11 @@ async function main(): Promise { return; } + // Preflight: make a single API call to verify the key works before + // launching 202 eval calls. Catches billing/credit issues immediately + // instead of retrying for hours. + await preflightCheck(model); + // Regenerate config for the selected model await regenerateConfig(projectRoot, model);