From a92ad65144797828713739bc1789cf6533d236f7 Mon Sep 17 00:00:00 2001 From: Tomasz Bykowski Date: Wed, 12 Nov 2025 10:17:46 +0100 Subject: [PATCH 1/2] fix: Set device on individual models when DEVICE parameter is provided When using `make install` with DEVICE parameter (e.g., DEVICE=hpu), this error is displayed: ```Error: template: rag/charts/llm-service/templates/serving-runtime.yaml:47:14: executing "rag/charts/llm-service/templates/serving-runtime.yaml" at : error calling index: index of nil pointer make: *** [Makefile:470: install] Error 1 ``` The device was only being set at the llm-service level but not on the individual model being enabled. This caused the Helm template to fail when looking up deviceConfigs because the model didn't have a device specified. Now when DEVICE is provided with LLM or SAFETY parameters, the device is explicitly set on the model itself (global.models.$(LLM).device), ensuring the serving-runtime template can correctly look up the appropriate deviceConfigs. Fixes the error: "index of nil pointer" when accessing deviceConfigs in serving-runtime.yaml template. Example usage: make install NAMESPACE=my-ns LLM=llama-3-1-8b-instruct DEVICE=hpu --- deploy/helm/Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/deploy/helm/Makefile b/deploy/helm/Makefile index 44f6b100..8de4caed 100644 --- a/deploy/helm/Makefile +++ b/deploy/helm/Makefile @@ -471,6 +471,10 @@ install: ## Install the RAG deployment if [ -n "$(LLM)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Enabling LLM model: $(LLM)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).enabled=true"; \ + if [ -n "$(DEVICE)" ]; then \ + echo -e "$(BLUE)[INFO]$(NC) Setting LLM model device: $(DEVICE)"; \ + HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).device=$(DEVICE)"; \ + fi; \ if [ -n "$(LLM_TOLERATION)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting LLM toleration: $(LLM_TOLERATION)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).tolerations[0].key=$(LLM_TOLERATION)"; \ @@ -489,6 +493,10 @@ install: ## Install the RAG deployment if [ -n "$(SAFETY)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Enabling SAFETY model: $(SAFETY)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).enabled=true"; \ + if [ -n "$(DEVICE)" ]; then \ + echo -e "$(BLUE)[INFO]$(NC) Setting SAFETY model device: $(DEVICE)"; \ + HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).device=$(DEVICE)"; \ + fi; \ if [ -n "$(SAFETY_TOLERATION)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting SAFETY toleration: $(SAFETY_TOLERATION)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).tolerations[0].key=$(SAFETY_TOLERATION)"; \ From e0769b5f6b1826f941d0fa99f028f7bcb1e2105d Mon Sep 17 00:00:00 2001 From: Tomasz Bykowski Date: Fri, 14 Nov 2025 10:05:15 +0100 Subject: [PATCH 2/2] Simplifying code --- deploy/helm/Makefile | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/deploy/helm/Makefile b/deploy/helm/Makefile index 8de4caed..356d941f 100644 --- a/deploy/helm/Makefile +++ b/deploy/helm/Makefile @@ -471,10 +471,6 @@ install: ## Install the RAG deployment if [ -n "$(LLM)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Enabling LLM model: $(LLM)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).enabled=true"; \ - if [ -n "$(DEVICE)" ]; then \ - echo -e "$(BLUE)[INFO]$(NC) Setting LLM model device: $(DEVICE)"; \ - HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).device=$(DEVICE)"; \ - fi; \ if [ -n "$(LLM_TOLERATION)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting LLM toleration: $(LLM_TOLERATION)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).tolerations[0].key=$(LLM_TOLERATION)"; \ @@ -493,10 +489,6 @@ install: ## Install the RAG deployment if [ -n "$(SAFETY)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Enabling SAFETY model: $(SAFETY)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).enabled=true"; \ - if [ -n "$(DEVICE)" ]; then \ - echo -e "$(BLUE)[INFO]$(NC) Setting SAFETY model device: $(DEVICE)"; \ - HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).device=$(DEVICE)"; \ - fi; \ if [ -n "$(SAFETY_TOLERATION)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting SAFETY toleration: $(SAFETY_TOLERATION)"; \ HELM_ARGS="$$HELM_ARGS --set global.models.$(SAFETY).tolerations[0].key=$(SAFETY_TOLERATION)"; \ @@ -515,6 +507,7 @@ install: ## Install the RAG deployment if [ -n "$(DEVICE)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting device: $(DEVICE)"; \ HELM_ARGS="$$HELM_ARGS --set llm-service.device='$(DEVICE)'"; \ + HELM_ARGS="$$HELM_ARGS --set global.models.$(LLM).device=$(DEVICE)"; \ fi; \ if [ -n "$(HF_TOKEN)" ]; then \ echo -e "$(BLUE)[INFO]$(NC) Setting HF_TOKEN from command line"; \