diff --git a/.github/workflows/test-ls-on-k8s.yml b/.github/workflows/test-ls-on-k8s.yml new file mode 100644 index 0000000..32c77d5 --- /dev/null +++ b/.github/workflows/test-ls-on-k8s.yml @@ -0,0 +1,72 @@ +name: LocalStack on Kubernetes + +on: + pull_request: + branches: [main] + paths: + - 'ls-on-k8s/**' + - '.github/workflows/test-ls-on-k8s.yml' + push: + branches: [main] + paths: + - 'ls-on-k8s/**' + - '.github/workflows/test-ls-on-k8s.yml' + workflow_dispatch: + +env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.K8S_LOCALSTACK_AUTH_TOKEN }} + LOCALSTACK_DISABLE_EVENTS: "1" + +jobs: + test-ls-on-k8s: + name: k3d + LocalStack + Lambda pod executor + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: ls-on-k8s + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install k3d + run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash + + - name: Spin up cluster and deploy LocalStack + run: make cluster-up + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Debug LocalStack pod k8s setup + run: | + PODNAME=$(kubectl --context k3d-ls-k8s-cluster -n localstack get pod -l app=localstack -o jsonpath='{.items[0].metadata.name}') + echo "Pod: $PODNAME" + echo "--- env ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- env | grep -E 'KUBERNETES|KUBECONFIG|LAMBDA' | sort + echo "--- /kube/config ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- cat /kube/config 2>/dev/null || echo "(not found)" + echo "--- serviceaccount files ---" + kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- ls /var/run/secrets/kubernetes.io/serviceaccount/ 2>/dev/null || echo "(not found)" + + - name: Run Lambda integration tests + run: make test + + - name: Dump logs on failure + if: failure() + run: | + make logs 2>/dev/null || true + make pods 2>/dev/null || true + echo "--- lambda pod logs ---" + kubectl --context k3d-ls-k8s-cluster -n localstack get pods | grep lambda || true + for pod in $(kubectl --context k3d-ls-k8s-cluster -n localstack get pods -o name | grep lambda); do + echo "=== $pod ===" + kubectl --context k3d-ls-k8s-cluster -n localstack logs $pod --all-containers 2>/dev/null || true + done + + - name: Tear down cluster + if: always() + run: make cluster-down diff --git a/ls-k8s/k8s/values.yaml b/ls-k8s/k8s/values.yaml deleted file mode 100644 index df9b58d..0000000 --- a/ls-k8s/k8s/values.yaml +++ /dev/null @@ -1,26 +0,0 @@ -replicaCount: 1 - -image: - repository: localstack/localstack - tag: "latest" - pullPolicy: IfNotPresent - -# Role + ServiceAccount give LocalStack permission to create/watch pods, -# which is required by the Kubernetes Lambda executor. -serviceAccount: - create: true - -role: - create: true - -service: - type: NodePort - edgeService: - name: edge - targetPort: 4566 - nodePort: 31566 - -# Use the Kubernetes executor so every Lambda invocation runs in its own pod. -lambdaExecutor: "kubernetes" - -debug: false diff --git a/ls-k8s/Makefile b/ls-on-k8s/Makefile similarity index 74% rename from ls-k8s/Makefile rename to ls-on-k8s/Makefile index 4e7da29..3c7d81b 100644 --- a/ls-k8s/Makefile +++ b/ls-on-k8s/Makefile @@ -18,13 +18,10 @@ cluster-up: ## Create k3d cluster and deploy LocalStack with Kubernetes Lambda e cluster-down: ## Tear down the k3d cluster @bash scripts/cluster-down.sh $(CLUSTER) -deploy-ls: ## Re-deploy / upgrade LocalStack (cluster must already be running) - helm repo update localstack - helm upgrade --install $(RELEASE) localstack/localstack \ - --kube-context $(CONTEXT) \ - --namespace $(NAMESPACE) \ - --values k8s/values.yaml \ - --wait --timeout 180s +deploy-ls: ## Re-deploy LocalStack (cluster must already be running) + kubectl --context $(CONTEXT) apply -f k8s/localstack.yaml + kubectl --context $(CONTEXT) rollout status deployment/$(RELEASE) \ + --namespace $(NAMESPACE) --timeout=180s $(VENV): python3 -m venv $(VENV) @@ -33,7 +30,10 @@ $(VENV): test: $(VENV) ## Run Lambda-on-Kubernetes integration tests $(VENV)/bin/pytest tests/ -v -s -logs: ## Tail LocalStack pod logs +logs: ## Dump LocalStack pod logs + kubectl --context $(CONTEXT) -n $(NAMESPACE) logs deploy/$(RELEASE) --tail=200 + +logs-follow: ## Tail LocalStack pod logs (interactive) kubectl --context $(CONTEXT) -n $(NAMESPACE) logs -f deploy/$(RELEASE) pods: ## List all pods in the LocalStack namespace diff --git a/ls-on-k8s/k8s/localstack.yaml b/ls-on-k8s/k8s/localstack.yaml new file mode 100644 index 0000000..9e6f307 --- /dev/null +++ b/ls-on-k8s/k8s/localstack.yaml @@ -0,0 +1,91 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: localstack + namespace: localstack +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: localstack + namespace: localstack +rules: + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["create", "delete", "get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: localstack + namespace: localstack +subjects: + - kind: ServiceAccount + name: localstack + namespace: localstack +roleRef: + kind: Role + name: localstack + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: localstack + namespace: localstack +spec: + replicas: 1 + selector: + matchLabels: + app: localstack + template: + metadata: + labels: + app: localstack + spec: + serviceAccountName: localstack + containers: + - name: localstack + image: localstack/localstack:latest + imagePullPolicy: IfNotPresent + ports: + - containerPort: 4566 + env: + - name: LAMBDA_RUNTIME_EXECUTOR + value: kubernetes + - name: LOCALSTACK_K8S_NAMESPACE + value: localstack + - name: LOCALSTACK_K8S_SERVICE_NAME + value: localstack + - name: LAMBDA_K8S_IMAGE_PREFIX + value: localstack/lambda- + - name: LOCALSTACK_DISABLE_EVENTS + value: "1" + - name: LOCALSTACK_AUTH_TOKEN + valueFrom: + secretKeyRef: + name: localstack-auth + key: token + optional: true + readinessProbe: + httpGet: + path: /_localstack/health + port: 4566 + initialDelaySeconds: 15 + periodSeconds: 5 + failureThreshold: 24 +--- +apiVersion: v1 +kind: Service +metadata: + name: localstack + namespace: localstack +spec: + type: NodePort + selector: + app: localstack + ports: + - port: 4566 + targetPort: 4566 + nodePort: 31566 diff --git a/ls-k8s/scripts/cluster-down.sh b/ls-on-k8s/scripts/cluster-down.sh similarity index 100% rename from ls-k8s/scripts/cluster-down.sh rename to ls-on-k8s/scripts/cluster-down.sh diff --git a/ls-k8s/scripts/cluster-up.sh b/ls-on-k8s/scripts/cluster-up.sh similarity index 66% rename from ls-k8s/scripts/cluster-up.sh rename to ls-on-k8s/scripts/cluster-up.sh index 3f8864c..319d212 100755 --- a/ls-k8s/scripts/cluster-up.sh +++ b/ls-on-k8s/scripts/cluster-up.sh @@ -5,7 +5,6 @@ CLUSTER="${1:-ls-k8s-cluster}" NODE_PORT="${2:-31566}" HOST_PORT="${3:-4566}" NAMESPACE="localstack" -RELEASE="localstack" CONTEXT="k3d-${CLUSTER}" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -14,7 +13,7 @@ if k3d cluster list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "${CLUSTER}"; echo " Cluster '${CLUSTER}' already exists – skipping creation." else k3d cluster create "${CLUSTER}" \ - --port "${HOST_PORT}:${NODE_PORT}@server[0]" \ + --port "${HOST_PORT}:${NODE_PORT}@server:0" \ --wait fi @@ -22,17 +21,20 @@ echo "==> Ensuring namespace '${NAMESPACE}'..." kubectl --context "${CONTEXT}" create namespace "${NAMESPACE}" \ --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f - -echo "==> Adding/updating LocalStack Helm repo..." -helm repo add localstack https://localstack.github.io/helm-charts 2>/dev/null || true -helm repo update localstack 2>/dev/null - -echo "==> Deploying LocalStack via Helm..." -helm upgrade --install "${RELEASE}" localstack/localstack \ - --kube-context "${CONTEXT}" \ - --namespace "${NAMESPACE}" \ - --values "${SCRIPT_DIR}/../k8s/values.yaml" \ - --wait \ - --timeout 180s +if [[ -n "${LOCALSTACK_AUTH_TOKEN:-}" ]]; then + echo "==> Creating auth token secret..." + kubectl --context "${CONTEXT}" create secret generic localstack-auth \ + --namespace "${NAMESPACE}" \ + --from-literal=token="${LOCALSTACK_AUTH_TOKEN}" \ + --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f - +fi + +echo "==> Deploying LocalStack..." +kubectl --context "${CONTEXT}" apply -f "${SCRIPT_DIR}/../k8s/localstack.yaml" + +echo "==> Waiting for LocalStack to be ready..." +kubectl --context "${CONTEXT}" rollout status deployment/localstack \ + --namespace "${NAMESPACE}" --timeout=180s echo "==> Waiting for LocalStack health endpoint on localhost:${HOST_PORT}..." for i in $(seq 1 60); do diff --git a/ls-k8s/tests/requirements.txt b/ls-on-k8s/tests/requirements.txt similarity index 100% rename from ls-k8s/tests/requirements.txt rename to ls-on-k8s/tests/requirements.txt diff --git a/ls-k8s/tests/test_lambda.py b/ls-on-k8s/tests/test_lambda.py similarity index 100% rename from ls-k8s/tests/test_lambda.py rename to ls-on-k8s/tests/test_lambda.py