feat: part 2 Env new behavior: OS Fallback and alert

proto/gen/rill/admin/v1/openapi.yaml

@@ -2302,7 +2302,7 @@ externalDocs:
 info:
     description: Rill Admin API enables programmatic management of Rill Cloud resources, including organizations, projects, and user access. It provides endpoints for creating, updating, and deleting these resources, as well as managing authentication and permissions.
     title: Rill Admin API
-    version: v0.78.2
+    version: v0.79.1
 openapi: 3.0.3
 paths:
     /v1/ai/complete:

proto/gen/rill/admin/v1/public.openapi.yaml

@@ -2302,7 +2302,7 @@ externalDocs:
 info:
     description: Rill Admin API enables programmatic management of Rill Cloud resources, including organizations, projects, and user access. It provides endpoints for creating, updating, and deleting these resources, as well as managing authentication and permissions.
     title: Rill Admin API
-    version: v0.78.2
+    version: v0.79.1
 openapi: 3.0.3
 paths:
     /v1/ai/complete: {}

proto/gen/rill/runtime/v1/runtime.swagger.yaml

@@ -4130,6 +4130,13 @@ definitions:
       errorMessage:
         type: string
         title: Error message if the connector is misconfigured
+      osEnvVariables:
+        type: array
+        items:
+          type: string
+        description: |-
+          Variables that were resolved from OS environment instead of .env files.
+          Used to show warnings in the UI when credentials come from OS env.
     description: AnalyzedConnector contains information about a connector that is referenced in the project files.
   v1AnalyzedVariable:
     type: object

proto/rill/runtime/v1/api.proto

@@ -1000,6 +1000,9 @@ message AnalyzedConnector {
   repeated ResourceName used_by = 8;
   // Error message if the connector is misconfigured
   string error_message = 9;
+  // Variables that were resolved from OS environment instead of .env files.
+  // Used to show warnings in the UI when credentials come from OS env.
+  repeated string os_env_variables = 12;
 }
 
 // Request message for RuntimeService.ListConnectorDrivers

runtime/connections.go

@@ -3,6 +3,7 @@ package runtime
 import (
 	"context"
 	"fmt"
+	"os"
 	"strconv"
 	"strings"
 
@@ -216,10 +217,18 @@ func (r *Runtime) ConnectorConfig(ctx context.Context, instanceID, name string)
 		}
 
 		res.Driver = c.Type
-		res.Project, err = resolveConnectorProperties(inst.Environment, inst.ResolveVariables(false), c)
+		var osEnvVars map[string]bool
+		res.Project, osEnvVars, err = resolveConnectorProperties(inst.Environment, inst.ResolveVariables(false), c)
 		if err != nil {
 			return nil, err
 		}
+		// Merge OS env vars from template resolution
+		for k := range osEnvVars {
+			if res.OSEnvVars == nil {
+				res.OSEnvVars = make(map[string]bool)
+			}
+			res.OSEnvVars[k] = true
+		}
 		if c.Provision {
 			res.Provision = c.Provision
 			res.ProvisionArgs = c.ProvisionArgs.AsMap()
@@ -255,22 +264,23 @@ func (r *Runtime) ConnectorConfig(ctx context.Context, instanceID, name string)
 
 	// For backwards compatibility, certain root-level variables apply to certain implicit connectors.
 	// NOTE: This switches on connector.Name, not connector.Type, because this only applies to implicit connectors.
+	// Uses OS environment variable fallback for cloud credentials.
 	switch name {
 	case "s3", "athena", "redshift":
-		res.setPreset("aws_access_key_id", vars["aws_access_key_id"], false)
-		res.setPreset("aws_secret_access_key", vars["aws_secret_access_key"], false)
-		res.setPreset("aws_session_token", vars["aws_session_token"], false)
+		res.setPresetWithOSFallback("aws_access_key_id", vars["aws_access_key_id"])
+		res.setPresetWithOSFallback("aws_secret_access_key", vars["aws_secret_access_key"])
+		res.setPresetWithOSFallback("aws_session_token", vars["aws_session_token"])
 	case "azure":
-		res.setPreset("azure_storage_account", vars["azure_storage_account"], false)
-		res.setPreset("azure_storage_key", vars["azure_storage_key"], false)
-		res.setPreset("azure_storage_sas_token", vars["azure_storage_sas_token"], false)
-		res.setPreset("azure_storage_connection_string", vars["azure_storage_connection_string"], false)
+		res.setPresetWithOSFallback("azure_storage_account", vars["azure_storage_account"])
+		res.setPresetWithOSFallback("azure_storage_key", vars["azure_storage_key"])
+		res.setPresetWithOSFallback("azure_storage_sas_token", vars["azure_storage_sas_token"])
+		res.setPresetWithOSFallback("azure_storage_connection_string", vars["azure_storage_connection_string"])
 	case "gcs":
-		res.setPreset("google_application_credentials", vars["google_application_credentials"], false)
+		res.setPresetWithOSFallback("google_application_credentials", vars["google_application_credentials"])
 	case "bigquery":
-		res.setPreset("google_application_credentials", vars["google_application_credentials"], false)
+		res.setPresetWithOSFallback("google_application_credentials", vars["google_application_credentials"])
 	case "motherduck":
-		res.setPreset("token", vars["token"], false)
+		res.setPresetWithOSFallback("token", vars["token"])
 		res.setPreset("dsn", "", true)
 	case "local_file":
 		// The "local_file" connector needs to know the repo root.
@@ -299,15 +309,17 @@ func (r *Runtime) ConnectorConfig(ctx context.Context, instanceID, name string)
 
 // resolveConnectorProperties resolves templating in the provided connector's properties.
 // It always returns a clone of the properties, even if no templating is found, so the output is safe for further mutations.
-func resolveConnectorProperties(environment string, vars map[string]string, c *runtimev1.Connector) (map[string]any, error) {
+// Also returns a map of OS env vars that were used during template resolution.
+func resolveConnectorProperties(environment string, vars map[string]string, c *runtimev1.Connector) (map[string]any, map[string]bool, error) {
 	if c.Config == nil {
-		return make(map[string]any), nil
+		return make(map[string]any), nil, nil
 	}
 	res := c.Config.AsMap()
 
 	td := parser.TemplateData{
 		Environment: environment,
 		Variables:   vars,
+		OSEnvVars:   make(map[string]bool),
 	}
 
 	for _, k := range c.TemplatedProperties {
@@ -317,12 +329,12 @@ func resolveConnectorProperties(environment string, vars map[string]string, c *r
 		}
 		v, err := parser.ResolveTemplateRecursively(v, td, true)
 		if err != nil {
-			return nil, fmt.Errorf("failed to resolve template: %w", err)
+			return nil, nil, fmt.Errorf("failed to resolve template: %w", err)
 		}
 		res[k] = v
 	}
 
-	return res, nil
+	return res, td.OSEnvVars, nil
 }
 
 // ConnectorConfig holds and resolves connector configuration.
@@ -339,6 +351,9 @@ type ConnectorConfig struct {
 	Provision bool
 	// ProvisionArgs provide provisioning args for when ProvisionName is set.
 	ProvisionArgs map[string]any
+	// OSEnvVars tracks variables that were resolved from OS environment instead of .env files.
+	// This is used to show warnings in the UI when credentials come from OS env.
+	OSEnvVars map[string]bool
 }
 
 // Resolve returns the final resolved connector configuration.
@@ -372,3 +387,46 @@ func (c *ConnectorConfig) setPreset(k, v string, force bool) {
 	}
 	c.Preset[k] = v
 }
+
+// setPresetWithOSFallback sets a preset value, falling back to OS environment variable if the value is empty.
+// It tracks whether the value came from OS env in the OSEnvVars map.
+func (c *ConnectorConfig) setPresetWithOSFallback(k, v string) {
+	if v != "" {
+		// Value found in .env or project variables
+		if c.Preset == nil {
+			c.Preset = make(map[string]any)
+		}
+		c.Preset[k] = v
+		return
+	}
+
+	// Try OS environment variable fallback
+	// Check exact match first
+	if osVal := os.Getenv(k); osVal != "" {
+		if c.Preset == nil {
+			c.Preset = make(map[string]any)
+		}
+		c.Preset[k] = osVal
+		c.trackOSEnvVar(k)
+		return
+	}
+
+	// Try uppercase variant (common for env vars like AWS_ACCESS_KEY_ID)
+	upperKey := strings.ToUpper(k)
+	if osVal := os.Getenv(upperKey); osVal != "" {
+		if c.Preset == nil {
+			c.Preset = make(map[string]any)
+		}
+		c.Preset[k] = osVal
+		c.trackOSEnvVar(upperKey)
+		return
+	}
+}
+
+// trackOSEnvVar marks a variable as coming from OS environment.
+func (c *ConnectorConfig) trackOSEnvVar(name string) {
+	if c.OSEnvVars == nil {
+		c.OSEnvVars = make(map[string]bool)
+	}
+	c.OSEnvVars[name] = true
+}

runtime/parser/template.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"reflect"
 	"strings"
 	"text/template"
@@ -34,6 +35,7 @@ import (
 //     dependency [`kind`] `name`: register a dependency (parse time)
 //     ref [`kind`] `name`: register a dependency at parse-time, resolve it to a name at resolve time (parse time and resolve time)
 //     lookup [`kind`] `name`: lookup another resource (resolve time)
+//     env `name`: access a project "environment" variable (parse and resolve time)
 //     .env.name: access a project "environment" variable (resolve time)
 //     .user.attribute: access an attribute from auth claims (resolve time)
 //     .meta: access the current resource's metadata (resolve time)
@@ -54,6 +56,9 @@ type TemplateData struct {
 	Self        TemplateResource
 	Resolve     func(ref ResourceName) (string, error)
 	Lookup      func(name ResourceName) (TemplateResource, error)
+	// OSEnvVars tracks variables that were resolved from OS environment instead of .env files.
+	// This is used to show warnings in the UI when credentials come from OS env.
+	OSEnvVars map[string]bool
 }
 
 // TemplateResource contains data for a resource for injection into a template.
@@ -128,6 +133,13 @@ func AnalyzeTemplate(tmpl string) (*TemplateMetadata, error) {
 		refs[name] = true
 		return map[string]any{}, nil
 	}
+	funcMap["env"] = func(name string) (string, error) {
+		if name == "" {
+			return "", fmt.Errorf(`"env" requires a variable name argument`)
+		}
+		// At parse time, just return a placeholder
+		return "", nil
+	}
 
 	// Parse template
 	t, err := template.New("").Funcs(funcMap).Option("missingkey=default").Parse(tmpl)
@@ -261,6 +273,42 @@ func ResolveTemplate(tmpl string, data TemplateData, errOnMissingTemplKeys bool)
 		}, nil
 	}
 
+	// Add func to access environment variables (case-insensitive)
+	// Falls back to OS environment variables if not found in .env files
+	funcMap["env"] = func(name string) (string, error) {
+		if name == "" {
+			return "", fmt.Errorf(`"env" requires a variable name argument`)
+		}
+		// Try exact match first
+		if value, ok := data.Variables[name]; ok {
+			return value, nil
+		}
+		// Try case-insensitive match
+		for key, value := range data.Variables {
+			if strings.EqualFold(key, name) {
+				return value, nil
+			}
+		}
+		// Fallback to OS environment variable
+		if value := os.Getenv(name); value != "" {
+			// Track that this variable came from OS env
+			if data.OSEnvVars != nil {
+				data.OSEnvVars[name] = true
+			}
+			return value, nil
+		}
+		// Try case-insensitive OS env lookup (check common variations)
+		for _, variant := range []string{strings.ToUpper(name), strings.ToLower(name)} {
+			if value := os.Getenv(variant); value != "" {
+				if data.OSEnvVars != nil {
+					data.OSEnvVars[variant] = true
+				}
+				return value, nil
+			}
+		}
+		return "", fmt.Errorf(`environment variable "%s" not found`, name)
+	}
+
 	// Parse template (error on missing keys)
 	// TODO: missingkey=error may be problematic for claims.
 	var opt string
@@ -274,12 +322,61 @@ func ResolveTemplate(tmpl string, data TemplateData, errOnMissingTemplKeys bool)
 		return "", err
 	}
 
-	// Split variables that contain dots into nested maps.
-	var vars map[string]any
-	if len(data.Variables) > 0 {
-		vars = map[string]any{}
-	}
+	// Extract variable references from the template to check for OS env fallback.
+	// Variables like "env.AWS_ACCESS_KEY_ID" need to be populated from OS env if not in data.Variables.
+	referencedVars := extractVariablesFromTemplate(t.Tree)
+
+	varsWithOSFallback := make(map[string]string, len(data.Variables))
 	for k, v := range data.Variables {
+		varsWithOSFallback[k] = v
+	}
+
+	// Check for OS env fallback for referenced variables not in data.Variables
+	for _, refVar := range referencedVars {
+		// Handle env.VAR_NAME references
+		if !strings.HasPrefix(refVar, "env.") {
+			continue
+		}
+		varName := strings.TrimPrefix(refVar, "env.")
+		// Check if already in variables (case-insensitive check)
+		// If found with different case, also add with the case the template expects
+		foundKey := ""
+		for k := range varsWithOSFallback {
+			if strings.EqualFold(k, varName) {
+				foundKey = k
+				break
+			}
+		}
+		if foundKey != "" {
+			// If the case doesn't match exactly, also add with the expected case
+			if foundKey != varName {
+				varsWithOSFallback[varName] = varsWithOSFallback[foundKey]
+			}
+			continue
+		}
+		// Try OS env fallback (try exact case, uppercase, and lowercase)
+		if osVal := os.Getenv(varName); osVal != "" {
+			varsWithOSFallback[varName] = osVal
+			if data.OSEnvVars != nil {
+				data.OSEnvVars[varName] = true
+			}
+		} else if osVal := os.Getenv(strings.ToUpper(varName)); osVal != "" {
+			varsWithOSFallback[varName] = osVal
+			if data.OSEnvVars != nil {
+				data.OSEnvVars[strings.ToUpper(varName)] = true
+			}
+		} else if osVal := os.Getenv(strings.ToLower(varName)); osVal != "" {
+			varsWithOSFallback[varName] = osVal
+			if data.OSEnvVars != nil {
+				data.OSEnvVars[strings.ToLower(varName)] = true
+			}
+		}
+	}
+
+	// Split variables that contain dots into nested maps.
+	// Always initialize vars to ensure .env.X access doesn't fail on nil map.
+	vars := map[string]any{}
+	for k, v := range varsWithOSFallback {
 		// Note: We always add the full variable name (including dots) at the top level.
 		vars[k] = v