Model validation

.github/workflows/model_validation.yml

@@ -0,0 +1,88 @@
+name: Model Validation
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Runs every Sunday at 14:00 UTC.
+    - cron: "0 14 * * 0"
+
+concurrency:
+  group: model-validation-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  validate-models:
+    name: Weekly Model Validation
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    env:
+      HARP_ENABLE_NETWORK_VALIDATION: "1"
+      HARP_HUGGINGFACE_TOKEN: ${{ secrets.HARP_HUGGINGFACE_TOKEN }}
+      HARP_STABILITY_API_KEY: ${{ secrets.HARP_STABILITY_API_KEY }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+          cache: "pip"
+          cache-dependency-path: |
+            requirements-model-validation.txt
+            pyharp/setup.py
+
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ffmpeg libsndfile1
+
+      - name: Install Python dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements-model-validation.txt
+          python -m pip install torch torchaudio --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Run model validation
+        id: validation
+        shell: bash
+        run: |
+          mkdir -p artifacts/model_validation
+          set +e
+          python -m pytest tests/model_validation -rA --run-network-validation \
+            --model-validation-report-dir artifacts/model_validation \
+            2>&1 | tee artifacts/model_validation/pytest.log
+          exit_code=${PIPESTATUS[0]}
+          echo "exit_code=${exit_code}" >> "$GITHUB_OUTPUT"
+          exit 0
+
+      - name: Publish dashboard to workflow summary
+        if: always()
+        shell: bash
+        run: |
+          if [ -f artifacts/model_validation/dashboard.md ]; then
+            cat artifacts/model_validation/dashboard.md >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "## Model Validation" >> "$GITHUB_STEP_SUMMARY"
+            echo "" >> "$GITHUB_STEP_SUMMARY"
+            echo "No dashboard was generated." >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload model validation report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: model-validation-report-${{ github.run_number }}
+          path: artifacts/model_validation/
+          if-no-files-found: warn
+          retention-days: 90
+
+      - name: Fail workflow on validation failures
+        if: steps.validation.outputs.exit_code != '0'
+        run: exit 1

.gitignore

@@ -58,6 +58,10 @@ dist/
 Miniconda*/
 Miniconda*.sh
 
+# model validation outputs
+artifacts/model_validation/
+__pycache__/
+
 # website
 node_modules
 temp

CMakeLists.txt

@@ -167,6 +167,7 @@ juce_add_binary_data(stability_controls
     SOURCES
         ${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/text-to-audio.json
         ${CMAKE_SOURCE_DIR}/src/clients/providers/stability/models/audio-to-audio.json
+        ${CMAKE_SOURCE_DIR}/resources/models/model_registry.json
 )
 
 # `target_link_libraries` links libraries and JUCE modules to other libraries or executables. Here,

requirements-model-validation.txt

@@ -0,0 +1,3 @@
+pytest>=8,<9
+requests>=2,<3
+-e ./pyharp

resources/models/model_registry.json

@@ -0,0 +1,285 @@
+{
+  "version": 1,
+  "models": [
+    {
+      "id": "custom-path",
+      "name": "Custom Path",
+      "path": "click here to enter a custom path...",
+      "provider": "custom",
+      "source": "ui",
+      "featured": true,
+      "validation": {
+        "enabled": false,
+        "mode": "manual",
+        "reason": "UI placeholder entry used to add custom model paths."
+      }
+    },
+    {
+      "id": "stability-text-to-audio",
+      "name": "Stable Audio Text to Audio",
+      "path": "stability/text-to-audio",
+      "provider": "stability",
+      "source": "stability_api",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_api",
+        "requires_network": true,
+        "requires_env": "HARP_STABILITY_API_KEY"
+      }
+    },
+    {
+      "id": "stability-audio-to-audio",
+      "name": "Stable Audio Audio to Audio",
+      "path": "stability/audio-to-audio",
+      "provider": "stability",
+      "source": "stability_api",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_api",
+        "requires_network": true,
+        "requires_env": "HARP_STABILITY_API_KEY"
+      }
+    },
+    {
+      "id": "text2midi-symbolic-music-generation",
+      "name": "Text2MIDI Symbolic Music Generation",
+      "path": "teamup-tech/text2midi-symbolic-music-generation",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "demucs-source-separation",
+      "name": "Demucs Source Separation",
+      "path": "teamup-tech/demucs-source-separation",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "solo-piano-audio-to-midi-transcription",
+      "name": "Solo Piano Audio to MIDI Transcription",
+      "path": "teamup-tech/solo-piano-audio-to-midi-transcription",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "transkun",
+      "name": "Transkun",
+      "path": "teamup-tech/transkun",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "tria",
+      "name": "TRIA",
+      "path": "teamup-tech/TRIA",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "anticipatory-music-transformer",
+      "name": "Anticipatory Music Transformer",
+      "path": "teamup-tech/anticipatory-music-transformer",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "vampnet-conditional-music-generation",
+      "name": "VampNet Conditional Music Generation",
+      "path": "teamup-tech/vampnet-conditional-music-generation",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "harmonic-percussive-separation",
+      "name": "Harmonic Percussive Separation",
+      "path": "teamup-tech/harmonic-percussive-separation",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "kokoro-tts",
+      "name": "Kokoro TTS",
+      "path": "teamup-tech/Kokoro-TTS",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "megatts3-voice-cloning",
+      "name": "MegaTTS3 Voice Cloning",
+      "path": "teamup-tech/MegaTTS3-Voice-Cloning",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "midi-synthesizer",
+      "name": "MIDI Synthesizer",
+      "path": "teamup-tech/midi-synthesizer",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "audioseal",
+      "name": "AudioSeal",
+      "path": "teamup-tech/audioseal",
+      "provider": "huggingface",
+      "source": "huggingface_space",
+      "featured": true,
+      "validation": {
+        "enabled": true,
+        "mode": "remote_gradio",
+        "requires_network": true
+      }
+    },
+    {
+      "id": "pitch-shifter-example",
+      "name": "Pitch Shifter Example",
+      "path": null,
+      "provider": "pyharp",
+      "source": "local_example",
+      "featured": false,
+      "validation": {
+        "enabled": true,
+        "mode": "local_pyharp_example",
+        "app_path": "pyharp/examples/pitch_shifter/app.py",
+        "python_modules": [
+          "gradio",
+          "audiotools",
+          "torch",
+          "torchaudio"
+        ],
+        "inputs": [
+          {
+            "kind": "audio_fixture",
+            "path": "resources/media/test.wav"
+          },
+          {
+            "kind": "literal",
+            "value": 7
+          }
+        ],
+        "expected_outputs": [
+          {
+            "kind": "audio_file",
+            "extension": ".wav",
+            "min_size_bytes": 128
+          }
+        ]
+      }
+    },
+    {
+      "id": "midi-pitch-shifter-example",
+      "name": "MIDI Pitch Shifter Example",
+      "path": null,
+      "provider": "pyharp",
+      "source": "local_example",
+      "featured": false,
+      "validation": {
+        "enabled": true,
+        "mode": "local_pyharp_example",
+        "app_path": "pyharp/examples/midi_pitch_shifter/app.py",
+        "python_modules": [
+          "gradio",
+          "symusic"
+        ],
+        "inputs": [
+          {
+            "kind": "midi_fixture",
+            "path": "resources/media/test.mid"
+          },
+          {
+            "kind": "literal",
+            "value": 7
+          }
+        ],
+        "expected_outputs": [
+          {
+            "kind": "midi_file",
+            "extension": ".mid",
+            "min_size_bytes": 32
+          }
+        ]
+      }
+    },
+    {
+      "id": "ui-tester-example",
+      "name": "UI Tester Example",
+      "path": null,
+      "provider": "pyharp",
+      "source": "local_example",
+      "featured": false,
+      "validation": {
+        "enabled": false,
+        "mode": "local_pyharp_example",
+        "reason": "Downloads external fixture files and is better covered by a dedicated integration test."
+      }
+    }
+  ]
+}