Complete local Minikube/Triton/LocalStack scaffolding for end-to-end S3 model deployment testing

README.md

@@ -32,6 +32,26 @@ The Model Deployment Operator is a prototype system designed to automate the dep
     helm install model-deployment-operator ./helm
     ```
 
+## Local End-to-End Testing (Minikube + Triton + LocalStack S3)
+
+Use the Kubernetes taskfile workflow to run operator + Triton and test model loading from S3 locally.
+
+```bash
+task kube:develop
+```
+
+This flow now:
+- Starts LocalStack (S3) and seeds `s3://bucket/model_repository/model.graphdef`
+- Starts Minikube and loads operator/job images
+- Installs Triton and operator charts
+- Applies local S3 config (`example/config-map-operator.yaml`) and example ModelDeployment (`example/md-source-s3.yaml`)
+
+To stop and clean up:
+
+```bash
+task kube:stop-develop
+```
+
 ## Contributing
 
 Contributions are welcome! Please open issues and pull requests to help improve this project.
@@ -44,4 +64,4 @@ Contributions are welcome! Please open issues and pull requests to help improve
 ## Research
 1. How to create CRD from pydantic model:
    * https://github.com/nolar/kopf/issues/524
-   * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py
+   * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py

docker-compose.yaml

@@ -1,14 +1,15 @@
 services:
   localstack:
     container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
-    network_mode: "host"
-    image: localstack/localstack:latest
+    # Pinned for reproducible local testing behavior across environments.
+    image: localstack/localstack:3.5.0
     ports:
-      - "4568:4568"
+      - "4566:4566"
     environment:
-      - SERVICES=s3  # Specify desired services (optional, adjust for other services)
+      - SERVICES=s3
       - SETUP_BUCKET_NAME=bucket
+      - AWS_DEFAULT_REGION=us-east-1
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
       - ${PWD}/tests/data/models_repository/simple/1:/models
-      - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh
+      - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh

example/config-map-operator.yaml

@@ -2,10 +2,10 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: operator-config
+  namespace: triton
 data:
-  awsConfig: |
-    access_key_id: XXX
-    secret_access_key: XXX
-    region_name: XXX
-    endpoint_url: XXX
-
+  # WARNING: Local development-only dummy credentials. Do not use in production.
+  aws_access_key_id: LOCALSTACK_DUMMY_KEY
+  aws_secret_access_key: LOCALSTACK_DUMMY_SECRET
+  aws_region_name: us-east-1
+  aws_endpoint_url: http://host.minikube.internal:4566

example/md-source-s3.yaml

@@ -27,20 +27,20 @@ spec:
           data_type: TYPE_INT32
           dims:
             - '16'
-    labels: ... # output-labels-file
+    # Labels expected by the bundled local test model.
+    labels:
+      - class_0
+      - class_1
     versions:
       - version: 1
         files:
-          - name: model.plan #  triton specific name
+          - name: model.graphdef
             source: s3://bucket/model_repository/model.graphdef
 
   jobTemplate:
-    provider: operator | user
+    provider: operator
     spec:
       source: s3
       auth:
         type: configMap
-        name: aws-creds
-
-      image: popovych-labs/model-deployment-job-s3:latest
-      configMap: aws-config
+        name: operator-config

kube.Taskfile.yaml

@@ -8,17 +8,48 @@ tasks:
   develop:
     desc: Start development process
     cmds:
+      - task: start-localstack
+      - task: wait-localstack
       - task: start
       - task: load-images
       - task: install-triton
       - task: install-operator-chart
+      - task: apply-s3-example
 
   stop-develop:
     desc: Stop development process
     cmds:
+      - task: delete-s3-example
       - task: delete-operator-chart
       - task: delete-triton
       - task: stop
+      - task: stop-localstack
+
+  start-localstack:
+    desc: Start localstack with seeded test bucket/object
+    cmds:
+      - docker compose up -d localstack
+
+  wait-localstack:
+    desc: Wait for localstack to become healthy
+    cmds:
+      - |
+        # 60 tries * 2s sleep = ~120s total timeout.
+        LOCALSTACK_HEALTH_MAX_TRIES=60
+        TRIES=0
+        until curl -sS http://localhost:4566/_localstack/health | python -c 'import json,sys; d=json.load(sys.stdin); s3=d.get("s3") or d.get("services", {}).get("s3"); raise SystemExit(0 if s3=="available" else 1)'; do
+          TRIES=$((TRIES + 1))
+          if [ "$TRIES" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then
+            echo "LocalStack did not become healthy in time"
+            exit 1
+          fi
+          sleep 2
+        done
+
+  stop-localstack:
+    desc: Stop localstack
+    cmds:
+      - docker compose down
   start:
       desc: Start minikube
       cmds:
@@ -83,4 +114,16 @@ tasks:
     desc: Delete helm release
     cmds:
       - |
-        helm delete triton -n triton
+        helm delete triton -n triton
+
+  apply-s3-example:
+    desc: Apply local S3 auth config and example ModelDeployment
+    cmds:
+      - kubectl apply -f ./example/config-map-operator.yaml
+      - kubectl apply -f ./example/md-source-s3.yaml
+
+  delete-s3-example:
+    desc: Delete local S3 auth config and example ModelDeployment
+    cmds:
+      - kubectl delete -f ./example/md-source-s3.yaml --ignore-not-found=true
+      - kubectl delete -f ./example/config-map-operator.yaml --ignore-not-found=true

scripts/localstack/init/setup_bucket.sh

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
 
-awslocal s3api create-bucket --bucket ${SETUP_BUCKET_NAME}  # Create bucket if it doesn't exist
-awslocal s3 cp /models/model.graphdef s3://${SETUP_BUCKET_NAME}/model_repository/  # Upload the file
-awslocal s3 ls --recursive s3://${SETUP_BUCKET_NAME}
+set -euo pipefail
+
+if awslocal s3api head-bucket --bucket "${SETUP_BUCKET_NAME}" >/dev/null 2>&1; then
+  echo "Bucket ${SETUP_BUCKET_NAME} already exists; continuing."
+else
+  awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}"
+fi
+
+awslocal s3 cp /models/model.graphdef "s3://${SETUP_BUCKET_NAME}/model_repository/model.graphdef"
+awslocal s3 ls --recursive "s3://${SETUP_BUCKET_NAME}"