From 44f4f591bba450db2fa1719181281356f6b832ab Mon Sep 17 00:00:00 2001 From: Michael Heller <21163552+mdheller@users.noreply.github.com> Date: Wed, 1 Jul 2026 23:49:13 -0400 Subject: [PATCH] =?UTF-8?q?feat(pkg):=20step=205=20=E2=80=94=20edge=20<->?= =?UTF-8?q?=20cloud-twin=20sync=20(finishes=20the=20person-graph=20stack)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diagram-4: the per-person graph is edge-resident (HellGraph on TopoLVM flash, #675); a cloud twin holds canonical. This wires the sync. - deployment-profiles.yaml: + p-edge profile with an edge_twin_sync block (edge_to_twin, 30m cadence, reference_only privacy). - edge-twin-sync/base: rclone CronJob (every 30m, co-located on the HellGraph node via podAffinity, mounts the store PVC read-only, pushes a timestamped copy to the S3-compatible twin) + topolvm-snapshots VolumeSnapshotClass (crash- consistent PITR upgrade path) + ServiceAccount + ExternalSecret example for the twin creds. Hardened podSec (runtime-baseline kyverno). Privacy: only non-private projections cross — the ProviderProjection membrane governs export; private nodes stay on-device. kustomize renders 3 resources; profiles parse (p-edge present). Depends on #675 (StatefulSet PVC). --- contracts/platform/deployment-profiles.yaml | 11 +++ infra/k8s/edge-twin-sync/README.md | 34 +++++++++ ...in-sync-secret.externalsecret.example.yaml | 34 +++++++++ .../edge-twin-sync/base/kustomization.yaml | 9 +++ .../edge-twin-sync/base/serviceaccount.yaml | 9 +++ .../k8s/edge-twin-sync/base/sync-cronjob.yaml | 73 +++++++++++++++++++ .../base/volumesnapshotclass.yaml | 12 +++ 7 files changed, 182 insertions(+) create mode 100644 infra/k8s/edge-twin-sync/README.md create mode 100644 infra/k8s/edge-twin-sync/base/edge-twin-sync-secret.externalsecret.example.yaml create mode 100644 infra/k8s/edge-twin-sync/base/kustomization.yaml create mode 100644 infra/k8s/edge-twin-sync/base/serviceaccount.yaml create mode 100644 infra/k8s/edge-twin-sync/base/sync-cronjob.yaml create mode 100644 infra/k8s/edge-twin-sync/base/volumesnapshotclass.yaml diff --git a/contracts/platform/deployment-profiles.yaml b/contracts/platform/deployment-profiles.yaml index d9454e9a..d9d4be39 100644 --- a/contracts/platform/deployment-profiles.yaml +++ b/contracts/platform/deployment-profiles.yaml @@ -8,6 +8,17 @@ profiles: purpose: multi-site private mesh with central hosted services - key: p3-regional purpose: regional or federated production posture + - key: p-edge + purpose: >- + on-device edge instance (k3s + TopoLVM flash). Per-person graph resident + locally; a cloud twin holds canonical. Intermittent link tolerated. + edge_twin_sync: + # The edge HellGraph store (RocksDB on TopoLVM) snapshots to the S3-compatible + # twin on a cadence; only non-private projections cross the link. + direction: edge_to_twin + cadence: 30m + mechanism: volume-snapshot + rclone-to-s3 + privacy: reference_only # private nodes never egress (ProviderProjection membrane) channels: - dev - stage diff --git a/infra/k8s/edge-twin-sync/README.md b/infra/k8s/edge-twin-sync/README.md new file mode 100644 index 00000000..2f4b9793 --- /dev/null +++ b/infra/k8s/edge-twin-sync/README.md @@ -0,0 +1,34 @@ +# PKG step 5 — edge ↔ cloud-twin sync + +Finishes the Personal Knowledge Graph stack (diagram-4). The per-person graph is +**edge-resident** (HellGraph on TopoLVM flash, PR #675); a **cloud twin** holds +canonical. This wires the sync between them. + +## Mechanism + +- **Profile** — `p-edge` added to `deployment-profiles.yaml` with an + `edge_twin_sync` block (direction `edge_to_twin`, 30m cadence, `reference_only` + privacy). +- **`sync-cronjob.yaml`** — every 30m an rclone Job co-locates on the HellGraph + node (TopoLVM volumes are node-local), mounts its RocksDB store **read-only**, + and pushes a timestamped copy to the S3-compatible twin. The twin keeps + point-in-time copies; the edge stays authoritative for the live graph. +- **`volumesnapshotclass.yaml`** — `topolvm-snapshots` for **crash-consistent** + point-in-time snapshots (the consistency-upgrade path: snapshot → restore to a + temp PVC → rclone that, instead of copying an open RocksDB store). +- **`serviceaccount.yaml`** + **`*.externalsecret.example.yaml`** — the sync + identity + the twin rclone creds (via external-secrets; example only). + +## Privacy + +Only **non-private projections** are meant to cross the link — the +`ProviderProjection` membrane (prophet-workspace `PersonalContextGraph`) governs +what may ever be exported. Private nodes stay on-device. + +## Needs a live cluster / follow-ups + +- Depends on the durable HellGraph StatefulSet ([#675](https://github.com/SocioProphet/prophet-platform/pull/675)) — the CronJob mounts its PVC `hellgraph-store-hellgraph-0`. +- Copying an open RocksDB store is best-effort consistent; use the VolumeSnapshot + path for strict PITR (a snapshot-rotation companion is the next refinement). +- Reverse path (twin → new edge bootstrap/restore) is the paired follow-up. +- rclone image + chart-free; the twin bucket/creds come from the ExternalSecret. diff --git a/infra/k8s/edge-twin-sync/base/edge-twin-sync-secret.externalsecret.example.yaml b/infra/k8s/edge-twin-sync/base/edge-twin-sync-secret.externalsecret.example.yaml new file mode 100644 index 00000000..e2bd959d --- /dev/null +++ b/infra/k8s/edge-twin-sync/base/edge-twin-sync-secret.externalsecret.example.yaml @@ -0,0 +1,34 @@ +# The twin credentials (rclone config for the S3-compatible twin + bucket name). +# Example ExternalSecret — pulls from the cluster's secret store; do not commit +# real creds. The rclone.conf defines a [twin] remote (s3/gcs). +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: edge-twin-sync + namespace: socioprophet + labels: + bundle: edge-twin-sync +spec: + refreshInterval: 1h + secretStoreRef: + name: REPLACE_WITH_SECRET_STORE + kind: ClusterSecretStore + target: + name: edge-twin-sync + template: + data: + # rclone remote named `twin` → the S3-compatible cloud store. + rclone.conf: | + [twin] + type = s3 + provider = GCS + endpoint = https://storage.googleapis.com + access_key_id = {{ .access_key }} + secret_access_key = {{ .secret_key }} + data: + - secretKey: bucket + remoteRef: { key: REPLACE_WITH_REMOTE_SECRET_KEY, property: twin_bucket } + - secretKey: access_key + remoteRef: { key: REPLACE_WITH_REMOTE_SECRET_KEY, property: twin_access_key } + - secretKey: secret_key + remoteRef: { key: REPLACE_WITH_REMOTE_SECRET_KEY, property: twin_secret_key } diff --git a/infra/k8s/edge-twin-sync/base/kustomization.yaml b/infra/k8s/edge-twin-sync/base/kustomization.yaml new file mode 100644 index 00000000..7260a309 --- /dev/null +++ b/infra/k8s/edge-twin-sync/base/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: socioprophet +# The SnapshotClass + SA + CronJob deploy on edge (p-edge). The ExternalSecret is +# an example (needs a real secret store) — not included here. +resources: + - volumesnapshotclass.yaml + - serviceaccount.yaml + - sync-cronjob.yaml diff --git a/infra/k8s/edge-twin-sync/base/serviceaccount.yaml b/infra/k8s/edge-twin-sync/base/serviceaccount.yaml new file mode 100644 index 00000000..a3594ae5 --- /dev/null +++ b/infra/k8s/edge-twin-sync/base/serviceaccount.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: hellgraph-edge-twin-sync + namespace: socioprophet + labels: + bundle: edge-twin-sync + # On GKE, annotate with the workload-identity SA that can write the twin bucket: + # iam.gke.io/gcp-service-account: edge-twin-sync@.iam.gserviceaccount.com diff --git a/infra/k8s/edge-twin-sync/base/sync-cronjob.yaml b/infra/k8s/edge-twin-sync/base/sync-cronjob.yaml new file mode 100644 index 00000000..fb466bc9 --- /dev/null +++ b/infra/k8s/edge-twin-sync/base/sync-cronjob.yaml @@ -0,0 +1,73 @@ +# Edge → cloud-twin sync (diagram-4). Every 30m the edge rclones the HellGraph +# RocksDB store to the S3-compatible twin. The twin holds canonical; the edge is +# authoritative for the live per-person graph. Only non-private projections are +# meant to cross — private nodes stay on-device (the ProviderProjection membrane +# governs what may ever be exported). +# +# The job co-locates on the node holding the HellGraph pod (TopoLVM volumes are +# node-local) and mounts its store PVC read-only. For strict point-in-time +# consistency, interpose a VolumeSnapshot (topolvm-snapshots class is provided) — +# see README. rclone remote + bucket come from the edge-twin-sync secret. +apiVersion: batch/v1 +kind: CronJob +metadata: + name: hellgraph-edge-twin-sync + namespace: socioprophet + labels: + bundle: edge-twin-sync +spec: + schedule: "*/30 * * * *" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + spec: + restartPolicy: Never + serviceAccountName: hellgraph-edge-twin-sync + securityContext: + runAsNonRoot: true + seccompProfile: { type: RuntimeDefault } + affinity: + # Land on the node that holds the HellGraph pod (its TopoLVM volume is + # local to that node). + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: { app: hellgraph } + topologyKey: kubernetes.io/hostname + containers: + - name: push-to-twin + image: rclone/rclone:1.68 + command: ["/bin/sh", "-c"] + args: + - | + set -eu + STAMP="$(date -u +%Y%m%dT%H%M%SZ)" + # Timestamped prefix per run → the twin keeps point-in-time copies. + rclone sync /store "twin:${TWIN_BUCKET}/hellgraph/${NODE_NAME}/${STAMP}" \ + --config /config/rclone.conf --transfers 4 --checksum + env: + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } + - name: TWIN_BUCKET + valueFrom: { secretKeyRef: { name: edge-twin-sync, key: bucket } } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: { drop: ["ALL"] } + volumeMounts: + - { name: store, mountPath: /store, readOnly: true } + - { name: rclone-config, mountPath: /config, readOnly: true } + volumes: + - name: store + persistentVolumeClaim: + claimName: hellgraph-store-hellgraph-0 # the StatefulSet's PVC (PR #675) + readOnly: true + - name: rclone-config + secret: + secretName: edge-twin-sync + items: + - { key: rclone.conf, path: rclone.conf } diff --git a/infra/k8s/edge-twin-sync/base/volumesnapshotclass.yaml b/infra/k8s/edge-twin-sync/base/volumesnapshotclass.yaml new file mode 100644 index 00000000..3adc4148 --- /dev/null +++ b/infra/k8s/edge-twin-sync/base/volumesnapshotclass.yaml @@ -0,0 +1,12 @@ +# Crash-consistent snapshots of the edge HellGraph RocksDB PVC (TopoLVM). A +# snapshot is point-in-time, so we export a snapshot rather than tar an open +# RocksDB store (which could be mid-write). Requires the external-snapshotter +# CRDs + TopoLVM's CSI snapshot support. +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: topolvm-snapshots + labels: + bundle: edge-twin-sync +driver: topolvm.cybozu.com +deletionPolicy: Delete