cli-evaluate/action.yml at main · orq-ai/cli-evaluate · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
name: "Orq Evaluate"
description: "Run evaluatorq evaluation files using the @orq-ai/cli"
author: "Orq AI"

branding:
  icon: "check-circle"
  color: "purple"

inputs:
  pattern:
    description: "Glob pattern for evaluation files (e.g., '**/*.eval.ts')"
    required: true
  orq-api-key:
    description: "Orq API key for authentication and tracing"
    required: false
  orq-base-url:
    description: "Orq API base URL (defaults to https://my.orq.ai)"
    required: false
  working-directory:
    description: "Working directory to run evaluations from"
    required: false
    default: "."
  bun-version:
    description: "Bun version to use"
    required: false
    default: "latest"
  disable-tracing:
    description: "Disable OTEL tracing even when API key is set"
    required: false
    default: "false"
  otel-endpoint:
    description: "Custom OTEL endpoint for traces (overrides auto-detection from orq-base-url)"
    required: false

runs:
  using: "composite"
  steps:
    - name: Setup Bun
      uses: oven-sh/setup-bun@v2
      with:
        bun-version: ${{ inputs.bun-version }}

    - name: Run evaluations
      shell: bash
      working-directory: ${{ inputs.working-directory }}
      env:
        ORQ_API_KEY: ${{ inputs.orq-api-key }}
        ORQ_BASE_URL: ${{ inputs.orq-base-url }}
        ORQ_DISABLE_TRACING: ${{ inputs.disable-tracing }}
        OTEL_EXPORTER_OTLP_ENDPOINT: ${{ inputs.otel-endpoint }}
      run: |
        bunx @orq-ai/cli evaluate "${{ inputs.pattern }}" || exit 1