ASAPQuery/.github/workflows/accuracy_performance.yml at main · ProjectASAP/ASAPQuery · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
name: PR Evaluation

# NOTE: GitHub-hosted runners are noisy. Latency numbers are indicative only.
# For precise benchmarks, register a self-hosted runner once asap-tools infra
# is decoupled from Cloudlab. See PDF eval guide Phase 3.

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    branches:
      - main
    paths:
      - 'asap-query-engine/**'
      - 'asap-planner-rs/**'
      - 'asap-quickstart/**'
      - '.github/workflows/accuracy_performance.yml'
      - 'benchmarks/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read
  packages: write
  pull-requests: write

jobs:
  # ---------------------------------------------------------------------------
  # Job 1: build images once from branch code and push with a SHA-based tag.
  # All downstream jobs pull these images instead of rebuilding.
  # ---------------------------------------------------------------------------
  build:
    if: github.event_name != 'pull_request' || github.event.pull_request.draft == false
    name: Build CI images
    runs-on: ubuntu-latest
    outputs:
      image-tag: ${{ steps.tag.outputs.value }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Compute image tag
        id: tag
        run: echo "value=sha-$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Build and push asap-planner-rs
        uses: docker/build-push-action@v6
        with:
          context: .
          file: asap-planner-rs/Dockerfile
          push: true
          tags: ghcr.io/projectasap/asap-planner-rs:${{ steps.tag.outputs.value }}
          cache-from: type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache
          cache-to: type=registry,ref=ghcr.io/projectasap/asap-planner-rs:buildcache,mode=max

      - name: Build and push asap-query-engine
        uses: docker/build-push-action@v6
        with:
          context: .
          file: asap-query-engine/Dockerfile
          push: true
          tags: ghcr.io/projectasap/asap-query-engine:${{ steps.tag.outputs.value }}
          cache-from: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache
          cache-to: type=registry,ref=ghcr.io/projectasap/asap-query-engine:buildcache,mode=max

      - name: Build and push asap-fake-exporter
        uses: docker/build-push-action@v6
        with:
          context: asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter
          file: asap-tools/data-sources/prometheus-exporters/fake_exporter/fake_exporter_rust/fake_exporter/Dockerfile
          push: true
          tags: ghcr.io/projectasap/asap-fake-exporter:${{ steps.tag.outputs.value }}
          cache-from: type=registry,ref=ghcr.io/projectasap/asap-fake-exporter:buildcache
          cache-to: type=registry,ref=ghcr.io/projectasap/asap-fake-exporter:buildcache,mode=max

  # ---------------------------------------------------------------------------
  # Job 2: pull the images built above, deploy the full stack, and evaluate.
  # ---------------------------------------------------------------------------
  eval:
    name: Full-stack PR evaluation
    needs: build
    runs-on: ubuntu-latest
    timeout-minutes: 60
    env:
      ASAP_IMAGE_TAG: ${{ needs.build.outputs.image-tag }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Log in to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Pull and start full stack
        run: |
          docker compose \
            -f asap-quickstart/docker-compose.yml \
            -f benchmarks/docker-compose.yml \
            up -d

      - name: Show running containers
        run: |
          docker compose \
            -f asap-quickstart/docker-compose.yml \
            -f benchmarks/docker-compose.yml \
            ps

      - name: Wait for all services to be healthy
        run: bash benchmarks/scripts/wait_for_stack.sh

      - name: Wait for pipeline and data ingestion
        run: bash benchmarks/scripts/ingest_wait.sh

      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install requests

      - name: Run baseline queries (Prometheus)
        run: python benchmarks/scripts/run_baseline.py

      - name: Run ASAP queries (query engine)
        run: python benchmarks/scripts/run_asap.py

      - name: Compare results and evaluate
        run: python benchmarks/scripts/compare.py

      - name: Upload evaluation reports
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: eval-reports-${{ github.run_id }}
          path: benchmarks/reports/

      - name: Print docker logs on failure
        if: failure()
        run: |
          docker compose \
            -f asap-quickstart/docker-compose.yml \
            -f benchmarks/docker-compose.yml \
            logs --no-color

      - name: Teardown stack
        if: always()
        run: |
          docker compose \
            -f asap-quickstart/docker-compose.yml \
            -f benchmarks/docker-compose.yml \
            down -v