diff --git a/README.md b/README.md index cb25ddb..fa22e58 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Designed for a GitOps workflow where infrastructure operators commit `haproxy.cf ## Architecture ``` -┌─────────────────────────┐ poll / webhook ┌──────────────────────┐ +┌─────────────────────────┐ poll ┌──────────────────────┐ │ GitHub Repo │◄────────────────────│ FluxCD │ │ (haproxy.cfg) │ GitRepository │ (K8s cluster) │ └─────────────────────────┘ │ Kustomization │ @@ -30,10 +30,11 @@ Designed for a GitOps workflow where infrastructure operators commit `haproxy.cf ## Features -- **GitOps-native**: FluxCD handles Git polling, webhooks, and Secret synchronization +- **GitOps-native**: FluxCD polls Git and syncs haproxy.cfg into a Kubernetes Secret +- **One-way architecture**: Clusters are not publicly accessible — Flux pulls from GitHub; no direct cluster-to-external communication is required - **Config validation**: Pre-validates `haproxy.cfg` via the Dataplane API `only_validate` endpoint before applying - **SPIFFE/SPIRE mTLS**: Automatic workload identity and certificate rotation between K8s and the bare-metal load balancer -- **Vault integration**: VSO (Vault Secrets Operator) for PKI certificate issuance and credential syncing +- **Static TLS**: Alternative to SPIRE — mount cert files from a pre-provisioned Secret - **Environment promotion**: `latest` (dev) → `stable` (production) using Flux Kustomize overlays - **Kubernetes Events**: Accept/reject status emitted as Events on the config Secret for observability - **Leader election**: Safe multi-replica deployment with controller-runtime leader election @@ -79,9 +80,9 @@ See `charts/haproxy-operator/values.yaml` for the full set of Helm values. When SPIRE is enabled (`spire.enabled=true`), the operator obtains X.509 SVIDs from the local SPIRE Agent for mTLS with the Dataplane API. No manual certificate distribution required — both the operator pod and the HAProxy host authenticate via their SPIFFE identities. -### Vault PKI (via VSO) +### Static TLS Certificates -When Vault is enabled (`vault.enabled=true`), the Helm chart creates VSO `VaultPKISecret` CRs that issue and auto-rotate TLS certificates from the `pki-sica-v2` intermediate CA. +When SPIRE is not enabled, the operator mounts TLS certificates from a Kubernetes Secret (`haproxy-operator-tls`). Pre-provision this Secret with `ca.crt`, `tls.crt`, and `tls.key` for mTLS communication with the Dataplane API. ## Project Structure diff --git a/charts/haproxy-operator/templates/deployment.yaml b/charts/haproxy-operator/templates/deployment.yaml index d1118f0..f975776 100644 --- a/charts/haproxy-operator/templates/deployment.yaml +++ b/charts/haproxy-operator/templates/deployment.yaml @@ -100,11 +100,7 @@ spec: {{- if not .Values.spire.enabled }} - name: tls secret: - {{- if .Values.vault.enabled }} - secretName: {{ .Values.vault.pki.secretName }} - {{- else }} secretName: haproxy-operator-tls - {{- end }} {{- end }} {{- if .Values.spire.enabled }} - name: spire-agent-socket diff --git a/charts/haproxy-operator/templates/vault-pki.yaml b/charts/haproxy-operator/templates/vault-pki.yaml deleted file mode 100644 index 331b9bc..0000000 --- a/charts/haproxy-operator/templates/vault-pki.yaml +++ /dev/null @@ -1,23 +0,0 @@ -{{- if and .Values.vault.enabled .Values.vault.pki.mount }} -# VSO VaultPKISecret – automatically issues and rotates TLS certificates -# from the Vault PKI Secondary Intermediate CA (pki-sica-v2). -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultPKISecret -metadata: - name: {{ include "haproxy-operator.fullname" . }}-tls - namespace: {{ .Release.Namespace }} - labels: - {{- include "haproxy-operator.labels" . | nindent 4 }} -spec: - vaultAuthRef: {{ .Values.vault.authRef }} - mount: {{ .Values.vault.pki.mount }} - role: {{ .Values.vault.pki.role }} - commonName: {{ .Values.vault.pki.commonName }} - ttl: {{ .Values.vault.pki.ttl }} - destination: - name: {{ .Values.vault.pki.secretName }} - create: true - rolloutRestartTargets: - - kind: Deployment - name: {{ include "haproxy-operator.fullname" . }} -{{- end }} diff --git a/charts/haproxy-operator/templates/vault-static.yaml b/charts/haproxy-operator/templates/vault-static.yaml deleted file mode 100644 index 64912f5..0000000 --- a/charts/haproxy-operator/templates/vault-static.yaml +++ /dev/null @@ -1,19 +0,0 @@ -{{- if and .Values.vault.enabled .Values.vault.static.enabled }} -# VSO VaultStaticSecret – syncs Dataplane API credentials from Vault KV. -apiVersion: secrets.hashicorp.com/v1beta1 -kind: VaultStaticSecret -metadata: - name: {{ include "haproxy-operator.fullname" . }}-dataplane-auth - namespace: {{ .Release.Namespace }} - labels: - {{- include "haproxy-operator.labels" . | nindent 4 }} -spec: - vaultAuthRef: {{ .Values.vault.authRef }} - mount: {{ .Values.vault.static.mount }} - path: {{ .Values.vault.static.path }} - type: kv-v2 - refreshAfter: 1h - destination: - name: {{ .Values.vault.static.secretName }} - create: true -{{- end }} diff --git a/charts/haproxy-operator/values.yaml b/charts/haproxy-operator/values.yaml index dde6a39..22d7cff 100644 --- a/charts/haproxy-operator/values.yaml +++ b/charts/haproxy-operator/values.yaml @@ -22,7 +22,7 @@ secretKey: haproxy.cfg dataplane: url: "https://haproxy:5555/v3" insecure: false - # Paths inside the pod where certs are mounted (by VSO or SPIRE). + # Paths inside the pod where static TLS certs are mounted (used when SPIRE is disabled). caCertPath: /etc/haproxy-operator/tls/ca.crt clientCertPath: /etc/haproxy-operator/tls/tls.crt clientKeyPath: /etc/haproxy-operator/tls/tls.key @@ -38,25 +38,6 @@ spire: enabled: false socketPath: "unix:///run/spire/agent.sock" -# Vault Secrets Operator (VSO) integration. -# Creates VaultStaticSecret / VaultPKISecret CRs that sync certs into K8s Secrets. -vault: - enabled: false - # VaultAuth CR name (created by flux-fleet base manifests). - authRef: vault-auth - pki: - mount: pki-sica-v2 - role: pki - commonName: haproxy-operator.tlu.bcit.ca - ttl: 72h - # Secret name where VSO writes the issued certificate. - secretName: haproxy-operator-tls - static: - # For Dataplane API basic-auth credentials stored in Vault KV. - enabled: false - mount: tlu-infrastructure - path: haproxy-operator/dataplane-credentials - secretName: haproxy-operator-dataplane-auth leaderElection: enabled: true diff --git a/internal/haproxy/dataplane_client.go b/internal/haproxy/dataplane_client.go index bb5eaa5..e09711c 100644 --- a/internal/haproxy/dataplane_client.go +++ b/internal/haproxy/dataplane_client.go @@ -173,219 +173,6 @@ func (c *Client) ValidateRawConfiguration(ctx context.Context, raw string) error return c.doRequestPlain(ctx, http.MethodPost, endpoint, raw, nil) } -// ApplyConfiguration pushes structured configuration (backends, frontends). -func (c *Client) ApplyConfiguration(ctx context.Context, cfg *Config) error { - if err := c.waitForReady(ctx, 10*time.Second); err != nil { - return fmt.Errorf("dataplane api not ready: %w", err) - } - for _, be := range cfg.Backends { - if err := c.applyBackend(ctx, &be); err != nil { - return err - } - } - for _, fe := range cfg.Frontends { - if err := c.applyFrontend(ctx, &fe); err != nil { - return err - } - } - return nil -} - -func (c *Client) applyBackend(ctx context.Context, be *Backend) error { - exists, err := c.backendExists(ctx, be.Name) - if err != nil { - return err - } - data := map[string]any{ - "name": be.Name, - "mode": be.Mode, - "balance": map[string]any{ - "algorithm": firstNonEmpty(be.Balance, "roundrobin"), - }, - } - if exists { - if err := c.doRequestVersioned(ctx, http.MethodPut, fmt.Sprintf("/services/haproxy/configuration/backends/%s", url.PathEscape(be.Name)), data, nil); err != nil { - return fmt.Errorf("update backend %s: %w", be.Name, err) - } - } else { - if err := c.doRequestVersioned(ctx, http.MethodPost, "/services/haproxy/configuration/backends", data, nil); err != nil { - return fmt.Errorf("create backend %s: %w", be.Name, err) - } - } - - for _, srv := range be.Servers { - if err := c.applyServer(ctx, be.Name, &srv); err != nil { - return err - } - } - return nil -} - -func (c *Client) applyServer(ctx context.Context, backendName string, srv *Server) error { - exists, err := c.serverExists(ctx, backendName, srv.Name) - if err != nil { - return err - } - - data := map[string]any{ - "name": srv.Name, - "address": srv.Address, - "port": srv.Port, - } - if srv.Check { - data["check"] = "enabled" - } - if srv.SSL { - data["ssl"] = "enabled" - if srv.Verify != "" { - data["verify"] = srv.Verify - } - } - - base := fmt.Sprintf("/services/haproxy/configuration/backends/%s/servers", url.PathEscape(backendName)) - - if exists { - endpoint := fmt.Sprintf("%s/%s", base, url.PathEscape(srv.Name)) - if err := c.doRequestVersioned(ctx, http.MethodPut, endpoint, data, nil); err != nil { - return fmt.Errorf("update server %s/%s: %w", backendName, srv.Name, err) - } - return nil - } - - if err := c.doRequestVersioned(ctx, http.MethodPost, base, data, nil); err != nil { - return fmt.Errorf("create server %s/%s: %w", backendName, srv.Name, err) - } - return nil -} - -func (c *Client) applyFrontend(ctx context.Context, fe *Frontend) error { - exists, err := c.frontendExists(ctx, fe.Name) - if err != nil { - return err - } - data := map[string]any{ - "name": fe.Name, - "mode": fe.Mode, - } - if fe.DefaultBackend != "" { - data["default_backend"] = fe.DefaultBackend - } - if exists { - if err := c.doRequestVersioned(ctx, http.MethodPut, fmt.Sprintf("/services/haproxy/configuration/frontends/%s", url.PathEscape(fe.Name)), data, nil); err != nil { - return fmt.Errorf("update frontend %s: %w", fe.Name, err) - } - } else { - if err := c.doRequestVersioned(ctx, http.MethodPost, "/services/haproxy/configuration/frontends", data, nil); err != nil { - return fmt.Errorf("create frontend %s: %w", fe.Name, err) - } - } - - for i := range fe.Binds { - b := fe.Binds[i] - b.Name = firstNonEmpty(b.Name, fmt.Sprintf("%s-bind-%d", fe.Name, i)) - if err := c.applyBind(ctx, fe.Name, &b); err != nil { - return err - } - } - - for i := range fe.UseBackends { - r := fe.UseBackends[i] - if err := c.applyBackendSwitchRule(ctx, fe.Name, i, &r); err != nil { - return err - } - } - - return nil -} - -func (c *Client) applyBind(ctx context.Context, frontendName string, b *Bind) error { - exists, err := c.bindExists(ctx, frontendName, b.Name) - if err != nil { - return err - } - - data := map[string]any{ - "name": b.Name, - "address": b.Address, - "port": b.Port, - } - - if b.SSL { - data["ssl"] = true - if b.SSLCertificate != "" { - data["ssl_certificate"] = b.SSLCertificate - } - if b.Verify != "" { - data["verify"] = b.Verify - } - } - if b.Alpn != "" { - data["alpn"] = b.Alpn - } - - base := fmt.Sprintf("/services/haproxy/configuration/frontends/%s/binds", url.PathEscape(frontendName)) - - if exists { - endpoint := fmt.Sprintf("%s/%s", base, url.PathEscape(b.Name)) - if err := c.doRequestVersioned(ctx, http.MethodPut, endpoint, data, nil); err != nil { - return fmt.Errorf("update bind %s/%s: %w", frontendName, b.Name, err) - } - return nil - } - - if err := c.doRequestVersioned(ctx, http.MethodPost, base, data, nil); err != nil { - return fmt.Errorf("create bind %s/%s: %w", frontendName, b.Name, err) - } - return nil -} - -func (c *Client) bindExists(ctx context.Context, frontendName, bindName string) (bool, error) { - var out map[string]any - endpoint := fmt.Sprintf( - "/services/haproxy/configuration/frontends/%s/binds/%s", - url.PathEscape(frontendName), - url.PathEscape(bindName), - ) - err := c.doRequest(ctx, http.MethodGet, endpoint, nil, &out) - if err != nil { - if isNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -func (c *Client) applyBackendSwitchRule(ctx context.Context, frontendName string, index int, r *UseBackendRule) error { - data := map[string]any{ - "index": index, - "name": r.Name, - } - if r.Condition != "" { - data["cond"] = r.Condition - } - if r.CondTest != "" { - data["cond_test"] = r.CondTest - } - - fe := url.PathEscape(frontendName) - nestedPut := fmt.Sprintf("/services/haproxy/configuration/frontends/%s/backend_switching_rules/%d", fe, index) - nestedPost := fmt.Sprintf("/services/haproxy/configuration/frontends/%s/backend_switching_rules", fe) - if err := c.doRequestVersioned(ctx, http.MethodPut, nestedPut, data, nil); err != nil { - if err2 := c.doRequestVersioned(ctx, http.MethodPost, nestedPost, data, nil); err2 == nil { - return nil - } - legacyPut := fmt.Sprintf("/services/haproxy/configuration/backend_switching_rules?frontend=%s&index=%d", url.QueryEscape(frontendName), index) - if err3 := c.doRequestVersioned(ctx, http.MethodPut, legacyPut, data, nil); err3 != nil { - legacyPost := fmt.Sprintf("/services/haproxy/configuration/backend_switching_rules?frontend=%s", url.QueryEscape(frontendName)) - if err4 := c.doRequestVersioned(ctx, http.MethodPost, legacyPost, data, nil); err4 != nil { - return fmt.Errorf("apply backend_switching_rule %s[%d]: %w", frontendName, index, err4) - } - } - } - return nil -} - func (c *Client) getConfigVersion(ctx context.Context) (int, error) { var n int if err := c.doRequest(ctx, http.MethodGet, "/services/haproxy/configuration/version", nil, &n); err != nil { @@ -394,58 +181,8 @@ func (c *Client) getConfigVersion(ctx context.Context) (int, error) { return n, nil } -func (c *Client) backendExists(ctx context.Context, name string) (bool, error) { - var out map[string]any - err := c.doRequest(ctx, http.MethodGet, fmt.Sprintf("/services/haproxy/configuration/backends/%s", url.PathEscape(name)), nil, &out) - if err != nil { - if isNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -func (c *Client) serverExists(ctx context.Context, backendName, serverName string) (bool, error) { - var out map[string]any - endpoint := fmt.Sprintf( - "/services/haproxy/configuration/backends/%s/servers/%s", - url.PathEscape(backendName), - url.PathEscape(serverName), - ) - err := c.doRequest(ctx, http.MethodGet, endpoint, nil, &out) - if err != nil { - if isNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - -func (c *Client) frontendExists(ctx context.Context, name string) (bool, error) { - var out map[string]any - err := c.doRequest(ctx, http.MethodGet, fmt.Sprintf("/services/haproxy/configuration/frontends/%s", url.PathEscape(name)), nil, &out) - if err != nil { - if isNotFound(err) { - return false, nil - } - return false, err - } - return true, nil -} - // --- HTTP plumbing --- -func (c *Client) doRequestVersioned(ctx context.Context, method, p string, body any, result any) error { - ver, err := c.getConfigVersion(ctx) - if err != nil { - return err - } - withVersion := addOrReplaceQuery(p, "version", fmt.Sprintf("%d", ver)) - return c.doRequest(ctx, method, withVersion, body, result) -} - func (c *Client) doRequest(ctx context.Context, method, p string, body any, result any) error { ref := *c.baseURL ref.Path = path.Join(strings.TrimSuffix(c.baseURL.Path, "/"), strings.TrimPrefix(p, "/")) @@ -542,13 +279,6 @@ func (e *APIError) Error() string { return fmt.Sprintf("dataplane api error (status %d): %s", e.StatusCode, e.Message) } -func isNotFound(err error) bool { - if e, ok := err.(*APIError); ok { - return e.StatusCode == http.StatusNotFound - } - return false -} - func addOrReplaceQuery(p, key, value string) string { base := p q := "" @@ -561,9 +291,4 @@ func addOrReplaceQuery(p, key, value string) string { return base + "?" + vals.Encode() } -func firstNonEmpty(v, d string) string { - if strings.TrimSpace(v) == "" { - return d - } - return v -} + diff --git a/internal/haproxy/dataplane_client_test.go b/internal/haproxy/dataplane_client_test.go index 3b6f651..6a5bbd7 100644 --- a/internal/haproxy/dataplane_client_test.go +++ b/internal/haproxy/dataplane_client_test.go @@ -131,20 +131,4 @@ func TestAPIError(t *testing.T) { } } -func TestIsNotFound(t *testing.T) { - t.Run("404 is not found", func(t *testing.T) { - if !isNotFound(&APIError{StatusCode: 404}) { - t.Error("expected true for 404") - } - }) - t.Run("500 is not not-found", func(t *testing.T) { - if isNotFound(&APIError{StatusCode: 500}) { - t.Error("expected false for 500") - } - }) - t.Run("non-APIError is not not-found", func(t *testing.T) { - if isNotFound(context.Canceled) { - t.Error("expected false for non-APIError") - } - }) -} + diff --git a/internal/haproxy/types.go b/internal/haproxy/types.go deleted file mode 100644 index faf6f98..0000000 --- a/internal/haproxy/types.go +++ /dev/null @@ -1,63 +0,0 @@ -package haproxy - -import ( - "crypto/sha256" - "encoding/hex" -) - -// Config is the internal representation pushed via the HAProxy Data Plane API. -type Config struct { - Frontends []Frontend - Backends []Backend -} - -// Backend models an HAProxy backend section. -type Backend struct { - Name string - Mode string - Balance string - Servers []Server -} - -// Server models a single server line inside a backend. -type Server struct { - Name string - Address string - Port int - Check bool - SSL bool - Verify string -} - -// Frontend models an HAProxy frontend section. -type Frontend struct { - Name string - Mode string - Binds []Bind - DefaultBackend string - UseBackends []UseBackendRule -} - -// Bind models a bind line inside a frontend. -type Bind struct { - Name string - Address string - Port int - SSL bool - SSLCertificate string - Alpn string - Verify string -} - -// UseBackendRule models a use_backend directive. -type UseBackendRule struct { - Name string - Condition string // "if" / "unless" - CondTest string // the condition expression -} - -// HashConfig returns a stable SHA256 hash of raw haproxy.cfg content. -func HashConfig(cfg string) string { - sum := sha256.Sum256([]byte(cfg)) - return hex.EncodeToString(sum[:]) -} diff --git a/internal/haproxy/types_test.go b/internal/haproxy/types_test.go deleted file mode 100644 index 9cc1d4b..0000000 --- a/internal/haproxy/types_test.go +++ /dev/null @@ -1,29 +0,0 @@ -package haproxy - -import "testing" - -func TestHashConfig(t *testing.T) { - t.Run("deterministic", func(t *testing.T) { - cfg := "global\n daemon\n" - h1 := HashConfig(cfg) - h2 := HashConfig(cfg) - if h1 != h2 { - t.Errorf("hashes differ: %s != %s", h1, h2) - } - }) - - t.Run("different configs different hashes", func(t *testing.T) { - h1 := HashConfig("config-a") - h2 := HashConfig("config-b") - if h1 == h2 { - t.Error("expected different hashes") - } - }) - - t.Run("returns 64 char hex", func(t *testing.T) { - h := HashConfig("test") - if len(h) != 64 { - t.Errorf("expected 64 chars, got %d", len(h)) - } - }) -}