diff --git a/README.md b/README.md index 7882b9c..a8f54b1 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,26 @@ The Model Deployment Operator is a prototype system designed to automate the dep helm install model-deployment-operator ./helm ``` +## Local End-to-End Testing (Minikube + Triton + LocalStack S3) + +Use the Kubernetes taskfile workflow to run operator + Triton and test model loading from S3 locally. + +```bash +task kube:develop +``` + +This flow now: +- Starts LocalStack (S3) and seeds `s3://bucket/model_repository/model.graphdef` +- Starts Minikube and loads operator/job images +- Installs Triton and operator charts +- Applies local S3 config (`example/config-map-operator.yaml`) and example ModelDeployment (`example/md-source-s3.yaml`) + +To stop and clean up: + +```bash +task kube:stop-develop +``` + ## Contributing Contributions are welcome! Please open issues and pull requests to help improve this project. @@ -44,4 +64,4 @@ Contributions are welcome! Please open issues and pull requests to help improve ## Research 1. How to create CRD from pydantic model: * https://github.com/nolar/kopf/issues/524 - * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py \ No newline at end of file + * https://github.com/asteven/kopf_resources/blob/master/kopf_resources/registry.py diff --git a/docker-compose.yaml b/docker-compose.yaml index 5928041..aed5ce3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,14 +1,15 @@ services: localstack: container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}" - network_mode: "host" - image: localstack/localstack:latest + # Pinned for reproducible local testing behavior across environments. + image: localstack/localstack:3.5.0 ports: - - "4568:4568" + - "4566:4566" environment: - - SERVICES=s3 # Specify desired services (optional, adjust for other services) + - SERVICES=s3 - SETUP_BUCKET_NAME=bucket + - AWS_DEFAULT_REGION=us-east-1 volumes: - "/var/run/docker.sock:/var/run/docker.sock" - ${PWD}/tests/data/models_repository/simple/1:/models - - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh \ No newline at end of file + - ${PWD}/scripts/localstack/init/setup_bucket.sh:/etc/localstack/init/ready.d/init.sh diff --git a/example/config-map-operator.yaml b/example/config-map-operator.yaml index 36797a9..bfb3f43 100644 --- a/example/config-map-operator.yaml +++ b/example/config-map-operator.yaml @@ -2,10 +2,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: operator-config + namespace: triton data: - awsConfig: | - access_key_id: XXX - secret_access_key: XXX - region_name: XXX - endpoint_url: XXX - \ No newline at end of file + # WARNING: Local development-only dummy credentials. Do not use in production. + aws_access_key_id: LOCALSTACK_DUMMY_KEY + aws_secret_access_key: LOCALSTACK_DUMMY_SECRET + aws_region_name: us-east-1 + aws_endpoint_url: http://host.minikube.internal:4566 diff --git a/example/md-source-s3.yaml b/example/md-source-s3.yaml index 3591674..03fb465 100644 --- a/example/md-source-s3.yaml +++ b/example/md-source-s3.yaml @@ -27,20 +27,20 @@ spec: data_type: TYPE_INT32 dims: - '16' - labels: ... # output-labels-file + # Labels expected by the bundled local test model. + labels: + - class_0 + - class_1 versions: - version: 1 files: - - name: model.plan # triton specific name + - name: model.graphdef source: s3://bucket/model_repository/model.graphdef jobTemplate: - provider: operator | user + provider: operator spec: source: s3 auth: type: configMap - name: aws-creds - - image: popovych-labs/model-deployment-job-s3:latest - configMap: aws-config \ No newline at end of file + name: operator-config diff --git a/kube.Taskfile.yaml b/kube.Taskfile.yaml index 4a095bd..2913b41 100644 --- a/kube.Taskfile.yaml +++ b/kube.Taskfile.yaml @@ -8,17 +8,48 @@ tasks: develop: desc: Start development process cmds: + - task: start-localstack + - task: wait-localstack - task: start - task: load-images - task: install-triton - task: install-operator-chart + - task: apply-s3-example stop-develop: desc: Stop development process cmds: + - task: delete-s3-example - task: delete-operator-chart - task: delete-triton - task: stop + - task: stop-localstack + + start-localstack: + desc: Start localstack with seeded test bucket/object + cmds: + - docker compose up -d localstack + + wait-localstack: + desc: Wait for localstack to become healthy + cmds: + - | + # 60 tries * 2s sleep = ~120s total timeout. + LOCALSTACK_HEALTH_MAX_TRIES=60 + TRIES=0 + until curl -sS http://localhost:4566/_localstack/health | python -c 'import json,sys; d=json.load(sys.stdin); s3=d.get("s3") or d.get("services", {}).get("s3"); raise SystemExit(0 if s3=="available" else 1)'; do + TRIES=$((TRIES + 1)) + if [ "$TRIES" -ge "$LOCALSTACK_HEALTH_MAX_TRIES" ]; then + echo "LocalStack did not become healthy in time" + exit 1 + fi + sleep 2 + done + + stop-localstack: + desc: Stop localstack + cmds: + - docker compose down start: desc: Start minikube cmds: @@ -83,4 +114,16 @@ tasks: desc: Delete helm release cmds: - | - helm delete triton -n triton \ No newline at end of file + helm delete triton -n triton + + apply-s3-example: + desc: Apply local S3 auth config and example ModelDeployment + cmds: + - kubectl apply -f ./example/config-map-operator.yaml + - kubectl apply -f ./example/md-source-s3.yaml + + delete-s3-example: + desc: Delete local S3 auth config and example ModelDeployment + cmds: + - kubectl delete -f ./example/md-source-s3.yaml --ignore-not-found=true + - kubectl delete -f ./example/config-map-operator.yaml --ignore-not-found=true diff --git a/scripts/localstack/init/setup_bucket.sh b/scripts/localstack/init/setup_bucket.sh index 4e9628e..b63a27a 100755 --- a/scripts/localstack/init/setup_bucket.sh +++ b/scripts/localstack/init/setup_bucket.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash -awslocal s3api create-bucket --bucket ${SETUP_BUCKET_NAME} # Create bucket if it doesn't exist -awslocal s3 cp /models/model.graphdef s3://${SETUP_BUCKET_NAME}/model_repository/ # Upload the file -awslocal s3 ls --recursive s3://${SETUP_BUCKET_NAME} \ No newline at end of file +set -euo pipefail + +if awslocal s3api head-bucket --bucket "${SETUP_BUCKET_NAME}" >/dev/null 2>&1; then + echo "Bucket ${SETUP_BUCKET_NAME} already exists; continuing." +else + awslocal s3api create-bucket --bucket "${SETUP_BUCKET_NAME}" +fi + +awslocal s3 cp /models/model.graphdef "s3://${SETUP_BUCKET_NAME}/model_repository/model.graphdef" +awslocal s3 ls --recursive "s3://${SETUP_BUCKET_NAME}"