[ML] Run PyTorch allowlist validation as a dedicated Buildkite step

The Linux build/test Docker images don't include Python 3 (it's only
used during image builds to compile PyTorch, then dropped in the
multi-stage final image). Move the validation to a dedicated pipeline
step using a python:3 agent image, triggered only for
run_pytorch_tests builds.

Made-with: Cursor

Author: edsavage

.buildkite/pipeline.json.py

@@ -84,6 +84,14 @@ def main():
                                                        ".buildkite/pipelines/check_build_regression.yml.sh",
                                                        soft_fail=True))
 
+    # Validate the PyTorch allowlist against HuggingFace models when
+    # triggered from the PyTorch edge pipeline.  Runs in a python:3
+    # container since the build/test images don't include Python.
+    if config.run_pytorch_tests:
+        pipeline_steps.append(pipeline_steps.generate_step("Upload PyTorch allowlist validation",
+                                                           ".buildkite/pipelines/validate_pytorch_allowlist.yml.sh",
+                                                           soft_fail=True))
+
     pipeline["env"] = env
     pipeline["steps"] = pipeline_steps
     print(json.dumps(pipeline, indent=2))

.buildkite/pipelines/validate_pytorch_allowlist.yml.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+
+cat <<'EOL'
+steps:
+  - label: "Validate PyTorch allowlist :torch:"
+    key: "validate_pytorch_allowlist"
+    command:
+        - "pip install -q -r dev-tools/extract_model_ops/requirements.txt"
+        - "python3 dev-tools/extract_model_ops/validate_allowlist.py --config dev-tools/extract_model_ops/validation_models.json --pt-dir dev-tools/extract_model_ops/es_it_models --verbose"
+EOL
+
+# Depend on the build steps so validation doesn't start before the
+# pipeline is fully generated.
+if [ -n "${ML_BUILD_STEP_KEYS:-}" ]; then
+    echo '    depends_on:'
+    IFS=',' read -ra STEP_KEYS <<< "$ML_BUILD_STEP_KEYS"
+    for key in "${STEP_KEYS[@]}"; do
+        echo "        - \"${key}\""
+    done
+fi
+
+cat <<'EOL'
+    allow_dependency_failure: true
+    soft_fail: true
+    agents:
+      image: "python:3"
+EOL