From da7945a4f0a735f852c328799d8a9571943c53ef Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:43:05 +0000 Subject: [PATCH 01/10] feat: complete local minikube triton localstack test scaffolding Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- README.md | 22 ++++++++++++++- docker-compose.yaml | 8 +++--- example/config-map-operator.yaml | 11 ++++---- example/md-source-s3.yaml | 13 ++++----- kube.Taskfile.yaml | 37 ++++++++++++++++++++++++- scripts/localstack/init/setup_bucket.sh | 8 ++++-- 6 files changed, 77 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 7882b9c..a8f54b1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,26 @@ The Model Deployment Operator is a prototype system designed to automate the dep helm install model-deployment-operator ./helm ``` +## Local End-to-End Testing (Minikube + Triton + LocalStack S3) + +Use the Kubernetes taskfile workflow to run operator + Triton and test model loading from S3 locally. + +```bash +task kube:develop +``` + +This flow now: +- Starts LocalStack (S3) and seeds `s3://bucket/model_repository/model.graphdef` +- Starts Minikube and loads operator/job images +- Installs Triton and operator charts +- Applies local S3 config (`example/config-map-operator.yaml`) and example ModelDeployment (`example/md-source-s3.yaml`) + +To stop and clean up: + +```bash +task kube:stop-develop +``` + ## Contributing Contributions are welcome! Please open issues and pull requests to help improve this project. @@ -44,4 +64,4 @@ Contributions are welcome! Please open issues and pull requests to help improve ## Research 1. How to create CRD from pydantic model: * https://github.com/nolar/kopf/issues/524 - * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py \ No newline at end of file + * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py diff --git a/docker-compose.yaml b/docker-compose.yaml index 5928041..e54c69a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,14 @@ services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" - network_mode: "host" image: localstack/localstack:latest ports: - - "4568:4568" + - "4566:4566" environment: - - SERVICES=s3 # Specify desired services (optional, adjust for other services) + - SERVICES=s3 - SETUP_BUCKET_NAME=bucket + - AWS_DEFAULT_REGION=us-east-1 volumes: - "/var/run/docker.sock:/var/run/docker.sock" - ${PWD}/tests/data/models_repository/simple/1:/models - - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh \ No newline at end of file + - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh diff --git a/example/config-map-operator.yaml b/example/config-map-operator.yaml index 36797a9..f3fc5fd 100644 --- a/example/config-map-operator.yaml +++ b/example/config-map-operator.yaml @@ -2,10 +2,9 @@ apiVersion: v1 kind: ConfigMap metadata: name: operator-config + namespace: triton data: - awsConfig: | - access_key_id: XXX - secret_access_key: XXX - region_name: XXX - endpoint_url: XXX - \ No newline at end of file + aws_access_key_id: test + aws_secret_access_key: test + aws_region_name: us-east-1 + aws_endpoint_url: http://host.minikube.internal:4566 diff --git a/example/md-source-s3.yaml b/example/md-source-s3.yaml index 3591674..133b8a9 100644 --- a/example/md-source-s3.yaml +++ b/example/md-source-s3.yaml @@ -27,20 +27,19 @@ spec: data_type: TYPE_INT32 dims: - '16' - labels: ... # output-labels-file + labels: + - class_0 + - class_1 versions: - version: 1 files: - - name: model.plan # triton specific name + - name: model.graphdef source: s3://bucket/model_repository/model.graphdef jobTemplate: - provider: operator | user + provider: operator spec: source: s3 auth: type: configMap - name: aws-creds - - image: popovych-labs/model-deployment-job-s3:latest - configMap: aws-config \ No newline at end of file + name: operator-config diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 4a095bd..140f754 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -8,17 +8,40 @@ tasks: develop: desc: Start development process cmds: + - task: start-localstack + - task: wait-localstack - task: start - task: load-images - task: install-triton - task: install-operator-chart + - task: apply-s3-example stop-develop: desc: Stop development process cmds: + - task: delete-s3-example - task: delete-operator-chart - task: delete-triton - task: stop + - task: stop-localstack + + start-localstack: + desc: Start localstack with seeded test bucket/object + cmds: + - docker compose up -d localstack + + wait-localstack: + desc: Wait for localstack to become healthy + cmds: + - | + until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do + sleep 2 + done + + stop-localstack: + desc: Stop localstack + cmds: + - docker compose down start: desc: Start minikube cmds: @@ -83,4 +106,16 @@ tasks: desc: Delete helm release cmds: - | - helm delete triton -n triton \ No newline at end of file + helm delete triton -n triton + + apply-s3-example: + desc: Apply local S3 auth config and example ModelDeployment + cmds: + - kubectl apply -f ./example/config-map-operator.yaml + - kubectl apply -f ./example/md-source-s3.yaml + + delete-s3-example: + desc: Delete local S3 auth config and example ModelDeployment + cmds: + - kubectl delete -f ./example/md-source-s3.yaml --ignore-not-found=true + - kubectl delete -f ./example/config-map-operator.yaml --ignore-not-found=true diff --git a/scripts/localstack/init/setup_bucket.sh b/scripts/localstack/init/setup_bucket.sh index 4e9628e..ff3b982 100755 --- a/scripts/localstack/init/setup_bucket.sh +++ b/scripts/localstack/init/setup_bucket.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash -awslocal s3api create-bucket --bucket ${SETUP_BUCKET_NAME} # Create bucket if it doesn't exist -awslocal s3 cp /models/model.graphdef s3://${SETUP_BUCKET_NAME}/model_repository/ # Upload the file -awslocal s3 ls --recursive s3://${SETUP_BUCKET_NAME} \ No newline at end of file +set -euo pipefail + +awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" || true +awslocal s3 cp /models/model.graphdef "s3://${SETUP_BUCKET_NAME}/model_repository/model.graphdef" +awslocal s3 ls --recursive "s3://${SETUP_BUCKET_NAME}" From 4f1186a746f97f0777695f56d4a2d59bae0fac9f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:43:53 +0000 Subject: [PATCH 02/10] chore: harden localstack startup checks in local scaffolding Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- docker-compose.yaml | 2 +- kube.Taskfile.yaml | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e54c69a..87ec7c0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,7 +1,7 @@ services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" - image: localstack/localstack:latest + image: localstack/localstack:3.5.0 ports: - "4566:4566" environment: diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 140f754..e05b6e1 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -34,7 +34,14 @@ tasks: desc: Wait for localstack to become healthy cmds: - | + tries=0 + max_tries=60 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do + tries=$((tries + 1)) + if [ "$tries" -ge "$max_tries" ]; then + echo "LocalStack did not become healthy in time" + exit 1 + fi sleep 2 done From b8e1e1d2b6ca5ce2358f408a667071a1cf0e4e5f Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:44:35 +0000 Subject: [PATCH 03/10] chore: polish local dev config clarity for s3 scaffolding Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- example/config-map-operator.yaml | 1 + kube.Taskfile.yaml | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/example/config-map-operator.yaml b/example/config-map-operator.yaml index f3fc5fd..7b3478a 100644 --- a/example/config-map-operator.yaml +++ b/example/config-map-operator.yaml @@ -4,6 +4,7 @@ metadata: name: operator-config namespace: triton data: + # LocalStack uses dummy static credentials for local development/testing only. aws_access_key_id: test aws_secret_access_key: test aws_region_name: us-east-1 diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index e05b6e1..368c520 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -35,10 +35,10 @@ tasks: cmds: - | tries=0 - max_tries=60 + localstack_health_max_tries=60 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do tries=$((tries + 1)) - if [ "$tries" -ge "$max_tries" ]; then + if [ "$tries" -ge "$localstack_health_max_tries" ]; then echo "LocalStack did not become healthy in time" exit 1 fi From 7aeac112462a5662d27a5b6772788863b0eb06ce Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:45:11 +0000 Subject: [PATCH 04/10] chore: align shell variable naming in localstack health check Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- kube.Taskfile.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 368c520..6480e48 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -35,10 +35,10 @@ tasks: cmds: - | tries=0 - localstack_health_max_tries=60 + LOCALSTACK_HEALTH_MAX_TRIES=60 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do tries=$((tries + 1)) - if [ "$tries" -ge "$localstack_health_max_tries" ]; then + if [ "$tries" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then echo "LocalStack did not become healthy in time" exit 1 fi From b5bd4f0ef5a66452be36c6849b9a48ecf52620ba Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:45:50 +0000 Subject: [PATCH 05/10] chore: keep localstack wait loop variable naming consistent Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- kube.Taskfile.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 6480e48..7bbf2e4 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -34,11 +34,11 @@ tasks: desc: Wait for localstack to become healthy cmds: - | - tries=0 + TRIES=0 LOCALSTACK_HEALTH_MAX_TRIES=60 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do - tries=$((tries + 1)) - if [ "$tries" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then + TRIES=$((TRIES + 1)) + if [ "$TRIES" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then echo "LocalStack did not become healthy in time" exit 1 fi From 1d727f5ebdc30b6df67bc843a27860b8b9eb00dd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:46:46 +0000 Subject: [PATCH 06/10] chore: improve localstack init robustness and clarify local examples Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- example/md-source-s3.yaml | 1 + kube.Taskfile.yaml | 1 + scripts/localstack/init/setup_bucket.sh | 11 ++++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/example/md-source-s3.yaml b/example/md-source-s3.yaml index 133b8a9..03fb465 100644 --- a/example/md-source-s3.yaml +++ b/example/md-source-s3.yaml @@ -27,6 +27,7 @@ spec: data_type: TYPE_INT32 dims: - '16' + # Labels expected by the bundled local test model. labels: - class_0 - class_1 diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 7bbf2e4..e78a9f1 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -35,6 +35,7 @@ tasks: cmds: - | TRIES=0 + # 60 tries * 2s sleep = ~120s total timeout. LOCALSTACK_HEALTH_MAX_TRIES=60 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do TRIES=$((TRIES + 1)) diff --git a/scripts/localstack/init/setup_bucket.sh b/scripts/localstack/init/setup_bucket.sh index ff3b982..93dc8a1 100755 --- a/scripts/localstack/init/setup_bucket.sh +++ b/scripts/localstack/init/setup_bucket.sh @@ -2,6 +2,15 @@ set -euo pipefail -awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" || true +CREATE_BUCKET_ERR_FILE="/tmp/localstack-create-bucket.err" +if ! awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" 2>"${CREATE_BUCKET_ERR_FILE}"; then + if grep -Eq "BucketAlreadyOwnedByYou|BucketAlreadyExists" "${CREATE_BUCKET_ERR_FILE}"; then + echo "Bucket ${SETUP_BUCKET_NAME} already exists; continuing." + else + cat "${CREATE_BUCKET_ERR_FILE}" >&2 + exit 1 + fi +fi + awslocal s3 cp /models/model.graphdef "s3://${SETUP_BUCKET_NAME}/model_repository/model.graphdef" awslocal s3 ls --recursive "s3://${SETUP_BUCKET_NAME}" From faba9d55d960a22deed50ffb02b4aa62e8f001bd Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:47:23 +0000 Subject: [PATCH 07/10] chore: clarify dummy local creds and tidy wait-loop variable order Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- example/config-map-operator.yaml | 6 +++--- kube.Taskfile.yaml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/example/config-map-operator.yaml b/example/config-map-operator.yaml index 7b3478a..bfb3f43 100644 --- a/example/config-map-operator.yaml +++ b/example/config-map-operator.yaml @@ -4,8 +4,8 @@ metadata: name: operator-config namespace: triton data: - # LocalStack uses dummy static credentials for local development/testing only. - aws_access_key_id: test - aws_secret_access_key: test + # WARNING: Local development-only dummy credentials. Do not use in production. + aws_access_key_id: LOCALSTACK_DUMMY_KEY + aws_secret_access_key: LOCALSTACK_DUMMY_SECRET aws_region_name: us-east-1 aws_endpoint_url: http://host.minikube.internal:4566 diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index e78a9f1..a157951 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -34,9 +34,9 @@ tasks: desc: Wait for localstack to become healthy cmds: - | - TRIES=0 # 60 tries * 2s sleep = ~120s total timeout. LOCALSTACK_HEALTH_MAX_TRIES=60 + TRIES=0 until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do TRIES=$((TRIES + 1)) if [ "$TRIES" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then From 7f1febfbb30cee9f9dec4950689aafd6e1ebdd4c Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:47:58 +0000 Subject: [PATCH 08/10] chore: use head-bucket check for idempotent localstack bucket setup Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- scripts/localstack/init/setup_bucket.sh | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/scripts/localstack/init/setup_bucket.sh b/scripts/localstack/init/setup_bucket.sh index 93dc8a1..b63a27a 100755 --- a/scripts/localstack/init/setup_bucket.sh +++ b/scripts/localstack/init/setup_bucket.sh @@ -2,14 +2,10 @@ set -euo pipefail -CREATE_BUCKET_ERR_FILE="/tmp/localstack-create-bucket.err" -if ! awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" 2>"${CREATE_BUCKET_ERR_FILE}"; then - if grep -Eq "BucketAlreadyOwnedByYou|BucketAlreadyExists" "${CREATE_BUCKET_ERR_FILE}"; then - echo "Bucket ${SETUP_BUCKET_NAME} already exists; continuing." - else - cat "${CREATE_BUCKET_ERR_FILE}" >&2 - exit 1 - fi +if awslocal s3api head-bucket --bucket "${SETUP_BUCKET_NAME}" >/dev/null 2>&1; then + echo "Bucket ${SETUP_BUCKET_NAME} already exists; continuing." +else + awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" fi awslocal s3 cp /models/model.graphdef "s3://${SETUP_BUCKET_NAME}/model_repository/model.graphdef" From da4a6b99b47bfd32196c4d3060adbf88bf4e8fa9 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:48:37 +0000 Subject: [PATCH 09/10] chore: make localstack health check parse json robustly Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- kube.Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index a157951..2913b41 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -37,7 +37,7 @@ tasks: # 60 tries * 2s sleep = ~120s total timeout. LOCALSTACK_HEALTH_MAX_TRIES=60 TRIES=0 - until curl -sS http://localhost:4566/_localstack/health | grep -q '"s3": "available"'; do + until curl -sS http://localhost:4566/_localstack/health | python -c 'import json,sys; d=json.load(sys.stdin); s3=d.get("s3") or d.get("services", {}).get("s3"); raise SystemExit(0 if s3=="available" else 1)'; do TRIES=$((TRIES + 1)) if [ "$TRIES" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then echo "LocalStack did not become healthy in time" From 3ed257f2915cba339d9a3a64d27c983cbe8349cc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 12 May 2026 06:49:16 +0000 Subject: [PATCH 10/10] docs: add localstack version pin rationale in compose config Agent-Logs-Url: https://github.com/ogvalt/model-deployment-operator/sessions/0f5b85c1-5625-4cc0-b8f2-67fea4dc418c Co-authored-by: ogvalt <10532355+ogvalt@users.noreply.github.com> --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 87ec7c0..aed5ce3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,6 +1,7 @@ services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" + # Pinned for reproducible local testing behavior across environments. image: localstack/localstack:3.5.0 ports: - "4566:4566"