ci: LocalStack on Kubernetes — k3d cluster + Lambda pod executor test

.github/workflows/test-ls-on-k8s.yml

@@ -0,0 +1,72 @@
+name: LocalStack on Kubernetes
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - 'ls-on-k8s/**'
+      - '.github/workflows/test-ls-on-k8s.yml'
+  push:
+    branches: [main]
+    paths:
+      - 'ls-on-k8s/**'
+      - '.github/workflows/test-ls-on-k8s.yml'
+  workflow_dispatch:
+
+env:
+  LOCALSTACK_AUTH_TOKEN: ${{ secrets.K8S_LOCALSTACK_AUTH_TOKEN }}
+  LOCALSTACK_DISABLE_EVENTS: "1"
+
+jobs:
+  test-ls-on-k8s:
+    name: k3d + LocalStack + Lambda pod executor
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    defaults:
+      run:
+        working-directory: ls-on-k8s
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install k3d
+        run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
+
+      - name: Spin up cluster and deploy LocalStack
+        run: make cluster-up
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Debug LocalStack pod k8s setup
+        run: |
+          PODNAME=$(kubectl --context k3d-ls-k8s-cluster -n localstack get pod -l app=localstack -o jsonpath='{.items[0].metadata.name}')
+          echo "Pod: $PODNAME"
+          echo "--- env ---"
+          kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- env | grep -E 'KUBERNETES|KUBECONFIG|LAMBDA' | sort
+          echo "--- /kube/config ---"
+          kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- cat /kube/config 2>/dev/null || echo "(not found)"
+          echo "--- serviceaccount files ---"
+          kubectl --context k3d-ls-k8s-cluster -n localstack exec "$PODNAME" -- ls /var/run/secrets/kubernetes.io/serviceaccount/ 2>/dev/null || echo "(not found)"
+
+      - name: Run Lambda integration tests
+        run: make test
+
+      - name: Dump logs on failure
+        if: failure()
+        run: |
+          make logs 2>/dev/null || true
+          make pods 2>/dev/null || true
+          echo "--- lambda pod logs ---"
+          kubectl --context k3d-ls-k8s-cluster -n localstack get pods | grep lambda || true
+          for pod in $(kubectl --context k3d-ls-k8s-cluster -n localstack get pods -o name | grep lambda); do
+            echo "=== $pod ==="
+            kubectl --context k3d-ls-k8s-cluster -n localstack logs $pod --all-containers 2>/dev/null || true
+          done
+
+      - name: Tear down cluster
+        if: always()
+        run: make cluster-down

ls-k8s/Makefile → ls-on-k8s/Makefile

@@ -18,13 +18,10 @@ cluster-up: ## Create k3d cluster and deploy LocalStack with Kubernetes Lambda e
 cluster-down: ## Tear down the k3d cluster
 	@bash scripts/cluster-down.sh $(CLUSTER)
 
-deploy-ls: ## Re-deploy / upgrade LocalStack (cluster must already be running)
-	helm repo update localstack
-	helm upgrade --install $(RELEASE) localstack/localstack \
-	  --kube-context $(CONTEXT) \
-	  --namespace $(NAMESPACE) \
-	  --values k8s/values.yaml \
-	  --wait --timeout 180s
+deploy-ls: ## Re-deploy LocalStack (cluster must already be running)
+	kubectl --context $(CONTEXT) apply -f k8s/localstack.yaml
+	kubectl --context $(CONTEXT) rollout status deployment/$(RELEASE) \
+	  --namespace $(NAMESPACE) --timeout=180s
 
 $(VENV):
 	python3 -m venv $(VENV)
@@ -33,7 +30,10 @@ $(VENV):
 test: $(VENV) ## Run Lambda-on-Kubernetes integration tests
 	$(VENV)/bin/pytest tests/ -v -s
 
-logs: ## Tail LocalStack pod logs
+logs: ## Dump LocalStack pod logs
+	kubectl --context $(CONTEXT) -n $(NAMESPACE) logs deploy/$(RELEASE) --tail=200
+
+logs-follow: ## Tail LocalStack pod logs (interactive)
 	kubectl --context $(CONTEXT) -n $(NAMESPACE) logs -f deploy/$(RELEASE)
 
 pods: ## List all pods in the LocalStack namespace

ls-on-k8s/k8s/localstack.yaml

@@ -0,0 +1,91 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: localstack
+  namespace: localstack
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: localstack
+  namespace: localstack
+rules:
+  - apiGroups: [""]
+    resources: ["pods", "pods/log"]
+    verbs: ["create", "delete", "get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: localstack
+  namespace: localstack
+subjects:
+  - kind: ServiceAccount
+    name: localstack
+    namespace: localstack
+roleRef:
+  kind: Role
+  name: localstack
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: localstack
+  namespace: localstack
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: localstack
+  template:
+    metadata:
+      labels:
+        app: localstack
+    spec:
+      serviceAccountName: localstack
+      containers:
+        - name: localstack
+          image: localstack/localstack:latest
+          imagePullPolicy: IfNotPresent
+          ports:
+            - containerPort: 4566
+          env:
+            - name: LAMBDA_RUNTIME_EXECUTOR
+              value: kubernetes
+            - name: LOCALSTACK_K8S_NAMESPACE
+              value: localstack
+            - name: LOCALSTACK_K8S_SERVICE_NAME
+              value: localstack
+            - name: LAMBDA_K8S_IMAGE_PREFIX
+              value: localstack/lambda-
+            - name: LOCALSTACK_DISABLE_EVENTS
+              value: "1"
+            - name: LOCALSTACK_AUTH_TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: localstack-auth
+                  key: token
+                  optional: true
+          readinessProbe:
+            httpGet:
+              path: /_localstack/health
+              port: 4566
+            initialDelaySeconds: 15
+            periodSeconds: 5
+            failureThreshold: 24
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: localstack
+  namespace: localstack
+spec:
+  type: NodePort
+  selector:
+    app: localstack
+  ports:
+    - port: 4566
+      targetPort: 4566
+      nodePort: 31566

ls-k8s/scripts/cluster-up.sh → ls-on-k8s/scripts/cluster-up.sh

@@ -5,7 +5,6 @@ CLUSTER="${1:-ls-k8s-cluster}"
 NODE_PORT="${2:-31566}"
 HOST_PORT="${3:-4566}"
 NAMESPACE="localstack"
-RELEASE="localstack"
 CONTEXT="k3d-${CLUSTER}"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
@@ -14,25 +13,28 @@ if k3d cluster list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "${CLUSTER}";
   echo "    Cluster '${CLUSTER}' already exists – skipping creation."
 else
   k3d cluster create "${CLUSTER}" \
-    --port "${HOST_PORT}:${NODE_PORT}@server[0]" \
+    --port "${HOST_PORT}:${NODE_PORT}@server:0" \
     --wait
 fi
 
 echo "==> Ensuring namespace '${NAMESPACE}'..."
 kubectl --context "${CONTEXT}" create namespace "${NAMESPACE}" \
   --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f -
 
-echo "==> Adding/updating LocalStack Helm repo..."
-helm repo add localstack https://localstack.github.io/helm-charts 2>/dev/null || true
-helm repo update localstack 2>/dev/null
-
-echo "==> Deploying LocalStack via Helm..."
-helm upgrade --install "${RELEASE}" localstack/localstack \
-  --kube-context "${CONTEXT}" \
-  --namespace "${NAMESPACE}" \
-  --values "${SCRIPT_DIR}/../k8s/values.yaml" \
-  --wait \
-  --timeout 180s
+if [[ -n "${LOCALSTACK_AUTH_TOKEN:-}" ]]; then
+  echo "==> Creating auth token secret..."
+  kubectl --context "${CONTEXT}" create secret generic localstack-auth \
+    --namespace "${NAMESPACE}" \
+    --from-literal=token="${LOCALSTACK_AUTH_TOKEN}" \
+    --dry-run=client -o yaml | kubectl --context "${CONTEXT}" apply -f -
+fi
+
+echo "==> Deploying LocalStack..."
+kubectl --context "${CONTEXT}" apply -f "${SCRIPT_DIR}/../k8s/localstack.yaml"
+
+echo "==> Waiting for LocalStack to be ready..."
+kubectl --context "${CONTEXT}" rollout status deployment/localstack \
+  --namespace "${NAMESPACE}" --timeout=180s
 
 echo "==> Waiting for LocalStack health endpoint on localhost:${HOST_PORT}..."
 for i in $(seq 1 60); do