From 4287054ed675a1975a03df0e409bd58d48942c0f Mon Sep 17 00:00:00 2001 From: Justin Garrison Date: Mon, 16 Feb 2026 14:17:29 -0800 Subject: [PATCH] fix: update image copy commands Fix gaps when running through docs Signed-off-by: Justin Garrison --- .../self-hosted/run-image-factory-on-prem.mdx | 60 +++--- .../omni/self-hosted/run-omni-airgapped.mdx | 196 ++++++++++++------ 2 files changed, 166 insertions(+), 90 deletions(-) diff --git a/public/omni/self-hosted/run-image-factory-on-prem.mdx b/public/omni/self-hosted/run-image-factory-on-prem.mdx index 912058b2..22000d5f 100644 --- a/public/omni/self-hosted/run-image-factory-on-prem.mdx +++ b/public/omni/self-hosted/run-image-factory-on-prem.mdx @@ -11,7 +11,7 @@ The Image Factory is a critical component of Omni to generate installation media ## Prerequisites -* Machine to run Image Factory +* [`talosctl`](../../talos/latest/getting-started/talosctl) * [`crane`](https://github.com/google/go-containerregistry/blob/main/cmd/crane/README.md) * `docker` or `podman` @@ -197,41 +197,51 @@ done -If you don't have direct access to an internal container registry (e.g. air gapped environment) you need to download the container images while connected to the internet. This command will download every image to a tar file stored in `/images`: +If you don't have direct access to an internal container registry (e.g. air gapped environment) you need to download the container images while connected to the internet. -Create a folder for downloading the images. +Download all images from `images.txt`. Create file names that don't contain special characters. ```bash -mkdir images +cat images.txt \ + | talosctl images cache-create \ + --layout flat \ + --image-cache-path ./image-cache \ + --images=- ``` -Download all images from `images.txt`. Create file names that don't contain special characters. +Move to a network that has access to the `$REGISTRY_ENDPOINT` endpoint and push the local images to the registry. +This serves the container layers on your local admin machine port `:5000` so you can push them with crane. +Run this command command from your admin machine so you do not have port conflicts with the `$REGISTRY_ENDPOINT`. -If you run this from macOS please make sure to use [GNU sed](https://www.gnu.org/software/sed/) because the built-in BSD sed has different behavior. You can install it with `brew install gnu-sed` and you should have a `gsed` binary available. +This method of copying containers is important because Talos releases pin to a container digest which needs to match in your internal registry ```bash -while read -r IMAGE; do - FILE_NAME=$(echo "$IMAGE" | sed 's/[\/ :@]/_/g').tar - echo "Pulling $IMAGE to $FILE_NAME" - crane pull "$IMAGE" "./images/$FILE_NAME" -done < images.txt -``` +export IP=$(hostname -I | awk '{print $1}') -Move the `images` folder and `images.txt` file to an air gapped machine or a network where you have `push` access to the `$REGISTRY_ENDPOINT` and push them to the registry. +talosctl image cache-cert-gen \ + --advertised-address $IP -```bash -while read -r IMAGE; do - [ -z "$IMAGE" ] && continue +talosctl image cache-serve \ + --address $IP:5000 \ + --image-cache-path ./image-cache \ + --tls-cert-file tls.crt \ + --tls-key-file tls.key & +``` - # construct new registry/image name - FILE_NAME="./images/$(echo "$IMAGE" | sed 's/[\/ :@]/_/g').tar" - IMAGE_PATH=$(echo "$IMAGE" | cut -d'/' -f2-) - DEST_PATH=$(echo "$IMAGE_PATH" | sed 's/@sha256:.*//') - DEST_IMAGE="${REGISTRY_ENDPOINT}/${DEST_PATH}" +Push the container images with `crane`. - crane push "$FILE_NAME" "$DEST_IMAGE" -done < images.txt +```bash +for SOURCE_IMAGE in $(cat images.txt) + do + IMAGE_WITHOUT_DIGEST=${SOURCE_IMAGE%%@*} + IMAGE_WITH_NEW_REG="${REGISTRY_ENDPOINT}/${IMAGE_WITHOUT_DIGEST#*/}" + LOCALHOST_IMAGE="${IP}:5000/${IMAGE_WITHOUT_DIGEST#*/}" + crane copy --insecure \ + $LOCALHOST_IMAGE \ + $IMAGE_WITH_NEW_REG +done ``` + @@ -246,7 +256,7 @@ Generate a cosign key. ```bash docker run --rm -it \ - -v $PWD:/keys -w /keys \ + -v $PWD:/keys:ro,Z -w /keys \ -e COSIGN_PASSWORD="" \ --user $(id -u):$(id -g) \ ghcr.io/sigstore/cosign/cosign:v2.6.1 \ @@ -311,6 +321,8 @@ artifacts: registry: $REGISTRY_ENDPOINT external: registry: $REGISTRY_ENDPOINT + repository: "talos" + namespace: "siderolabs" containerSignature: publicKeyFile: /cosign.pub subjectRegExp: "" diff --git a/public/omni/self-hosted/run-omni-airgapped.mdx b/public/omni/self-hosted/run-omni-airgapped.mdx index 5389aa8a..1c2850c0 100644 --- a/public/omni/self-hosted/run-omni-airgapped.mdx +++ b/public/omni/self-hosted/run-omni-airgapped.mdx @@ -3,7 +3,7 @@ title: Run Omni Air-Gapped description: Set up the full Sidero stack in a fully offline environment --- -import { omni_release, version, release } from '/snippets/custom-variables.mdx'; +import { omni_release, version, release, k8s_release } from '/snippets/custom-variables.mdx'; This document will walk through each component to run the "Sidero stack" in an offline environment which includes the following components. @@ -78,13 +78,50 @@ chmod +x crane To make this guide easier to follow we will set global variables for each of the endpoints and ports we will use. Update the hostnames and ports if you change any of them from the defaults. +If you don't have DNS on your network it will be easiest to set these endpoints to the IP address of the machine running the services. + ```bash -REGISTRY_ENDPOINT=registry.internal:5000 -FACTORY_ENDPOINT=factory.internal:8080 -AUTH_ENDPOINT=auth.internal:5556 -OMNI_ENDPOINT=omni.internal +export REGISTRY_ENDPOINT=registry.internal:5000 +export FACTORY_ENDPOINT=factory.internal:8080 +export AUTH_ENDPOINT=auth.internal:5556 +export OMNI_ENDPOINT=omni.internal ``` +### Open firewall ports + +You'll need to open all of the above firewall ports on the machine you'll be running them on. + + + + + Red Hat and enterprise linux derivatives use `firewall-cmd` + + ```bash +sudo firewall-cmd --permanent --add-port=8100/tcp +sudo firewall-cmd --permanent --add-port=5000/tcp +sudo firewall-cmd --permanent --add-port=8080/tcp +sudo firewall-cmd --permanent --add-port=5556/tcp +sudo firewall-cmd --permanent --add-port=443/tcp +sudo firewall-cmd --permanent --add-port=8090/udp +sudo firewall-cmd --permanent --add-port=8091/tcp + ``` + + + + Debian and Ubuntu use `ufw` for firewall commands. + + ```bash +sudo ufw allow 8100/tcp +sudo ufw allow 5000/tcp +sudo ufw allow 8080/tcp +sudo ufw allow 5556/tcp +sudo ufw allow 443/tcp +sudo ufw allow 8090/udp +sudo ufw allow 8091/tcp + ``` + + + ## 1. Generate certificates In order to run services securely, even in an air gapped environment, you should run with encrypted data in transit and at rest. There are multiple certificates and keys needed to secure your infrastructure. @@ -329,9 +366,9 @@ Run dex with the provided configuration and certificate. docker run -d \ --name dex \ -p 5556:5556 \ - -v $(pwd)/dex.yaml:/etc/dex/dex.yaml \ - -v $(pwd)/server-key.pem:/etc/dex/tls/server-key.pem \ - -v $(pwd)/server-chain.pem:/etc/dex/tls/server-chain.pem \ + -v $(pwd)/dex.yaml:/etc/dex/dex.yaml:ro,Z \ + -v $(pwd)/server-key.pem:/etc/dex/tls/server-key.pem:ro,Z \ + -v $(pwd)/server-chain.pem:/etc/dex/tls/server-chain.pem:ro,Z \ ${REGISTRY_ENDPOINT}/dexidp/dex:v2.41.1 \ dex serve /etc/dex/dex.yaml ``` @@ -354,9 +391,13 @@ Data in Omni's database is encrypted and we need an encryption key to provide to Generate a GPG key: ```bash -gpg --quick-generate-key "Omni (Used for etcd data encryption) how-to-guide@siderolabs.com" rsa4096 cert never -FINGERPRINT=$(gpg --with-colons --list-keys "how-to-guide@siderolabs.com" | awk -F: '$1 == "fpr" {print $10; exit}') -gpg --quick-add-key ${FINGERPRINT} rsa4096 encr never +gpg --batch --passphrase '' \ + --quick-generate-key "Omni (Used for etcd data encryption) how-to-guide@siderolabs.com" \ + rsa4096 cert never +FINGERPRINT=$(gpg --with-colons --list-keys "how-to-guide@siderolabs.com" \ + | awk -F: '$1 == "fpr" {print $10; exit}') +gpg --batch --passphrase '' \ + --quick-add-key ${FINGERPRINT} rsa4096 encr never gpg --export-secret-key --armor how-to-guide@siderolabs.com > omni.asc ``` @@ -379,7 +420,7 @@ If your machine has access to the internal registry you can push the image direc If you need to send the image to a remote machine or transfer it via an offline method you can export the container image. -{`docker save -o omni.tar \${REGISTRY_ENDPOINT}/siderolabs/omni:${omni_release}`} +{`docker save -o omni.tar ghcr.io/siderolabs/omni:${omni_release}`} On the remote machine load the archive into the local image storage and then push it to the registry. @@ -392,14 +433,18 @@ On the remote machine load the archive into the local image storage and then pus This will run Omni with an embedded etcd database mounted to the host. It is not recommended for production use cases. The command assumes the certificates generated earlier are available in the local directory where you run this command. +If running Omni with podman you'll need to use `sudo` so the wireguard endpoints and low level ports can be mapped + -{`docker run \\\n --name omni \\\n -d --net=host \\\n --cap-add=NET_ADMIN \\\n --device /dev/net/tun:/dev/net/tun \\\n -v "\${PWD}/ca.pem:/etc/ssl/certs/ca-certificates.crt:ro" \\\n -v "\${PWD}/etcd:/_out/etcd" \\\n -v "\${PWD}/sqlite:/_out/sqlite:rw" \\\n -v "\${PWD}/server-key.pem:/server-key.pem:ro" \\\n -v "\${PWD}/server-chain.pem:/server-chain.pem:ro" \\\n -v "\${PWD}/omni.asc:/omni.asc:ro" \\\n \${REGISTRY_ENDPOINT}/siderolabs/omni:${omni_release} \\\n --name=air-gap-omni \\\n --cert=/server-chain.pem \\\n --key=/server-key.pem \\\n --siderolink-api-cert=/server-chain.pem \\\n --siderolink-api-key=/server-key.pem \\\n --private-key-source=file:///omni.asc \\\n --event-sink-port=8091 \\\n --bind-addr=0.0.0.0:443 \\\n --siderolink-api-bind-addr=0.0.0.0:8090 \\\n --k8s-proxy-bind-addr=0.0.0.0:8100 \\\n --advertised-api-url=https://omni.internal \\\n --siderolink-api-advertised-url=https://omni.internal:8090 \\\n --siderolink-wireguard-advertised-addr=\$(hostname -I | awk '{print \$1}'):50180 \\\n --advertised-kubernetes-proxy-url=https://omni.internal:8100 \\\n --auth-auth0-enabled=false \\\n --auth-oidc-enabled=true \\\n --auth-oidc-client-secret=aW50ZXJuYWwtc2lkZXJvLXN0YWNrCg== \\\n --auth-oidc-provider-url=https://\${AUTH_ENDPOINT} \\\n --auth-oidc-client-id=omni \\\n --auth-oidc-scopes=openid,profile,email \\\n --image-factory-address=https://\${FACTORY_ENDPOINT} \\\n --initial-users=admin@omni.internal \\\n --kubernetes-registry=\${REGISTRY_ENDPOINT}/siderolabs/kubelet \\\n --sqlite-storage-path=/_out/sqlite/omni.db \\\n --talos-installer-registry=\${REGISTRY_ENDPOINT}/siderolabs/installer \\\n --workload-proxying-enabled=false \\\n --metrics-bind-addr=0.0.0.0:2123`} +{`docker run \\\n --name omni \\\n -d --net=host \\\n --cap-add=NET_ADMIN \\\n --device /dev/net/tun:/dev/net/tun \\\n -v "\${PWD}/ca.pem:/etc/ssl/certs/ca-certificates.crt:ro,Z" \\\n -v "\${PWD}/etcd:/_out/etcd:rw,Z" \\\n -v "\${PWD}/sqlite:/_out/sqlite:rw,Z" \\\n -v "\${PWD}/server-key.pem:/server-key.pem:ro,Z" \\\n -v "\${PWD}/server-chain.pem:/server-chain.pem:ro,Z" \\\n -v "\${PWD}/omni.asc:/omni.asc:ro,Z" \\\n \${REGISTRY_ENDPOINT}/siderolabs/omni:${omni_release} \\\n --name=air-gap-omni \\\n --cert=/server-chain.pem \\\n --key=/server-key.pem \\\n --siderolink-api-cert=/server-chain.pem \\\n --siderolink-api-key=/server-key.pem \\\n --private-key-source=file:///omni.asc \\\n --event-sink-port=8091 \\\n --bind-addr=0.0.0.0:443 \\\n --siderolink-api-bind-addr=0.0.0.0:8090 \\\n --k8s-proxy-bind-addr=0.0.0.0:8100 \\\n --advertised-api-url=https://omni.internal \\\n --siderolink-api-advertised-url=https://omni.internal:8090 \\\n --siderolink-wireguard-advertised-addr=\$(hostname -I | awk '{print \$1}'):50180 \\\n --advertised-kubernetes-proxy-url=https://omni.internal:8100 \\\n --auth-auth0-enabled=false \\\n --auth-oidc-enabled=true \\\n --auth-oidc-client-secret=aW50ZXJuYWwtc2lkZXJvLXN0YWNrCg== \\\n --auth-oidc-provider-url=https://\${AUTH_ENDPOINT} \\\n --auth-oidc-client-id=omni \\\n --auth-oidc-scopes=openid,profile,email \\\n --image-factory-address=https://\${FACTORY_ENDPOINT} \\\n --initial-users=admin@omni.internal \\\n --kubernetes-registry=\${REGISTRY_ENDPOINT}/siderolabs/kubelet \\\n --sqlite-storage-path=/_out/sqlite/omni.db \\\n --talos-installer-registry=\${REGISTRY_ENDPOINT}/siderolabs/installer \\\n --workload-proxying-enabled=false \\\n --metrics-bind-addr=0.0.0.0:2123`} >We changed the `--metrics-bind-addr` to use port `:2123` to avoid port conflicts with Image Factory (if it's running on the same host). These flags mount the files and directories needed by Omni (e.g. certificates, etcd storage) and set flags to connect Omni to the upstream services (e.g. factory, authentication). +If you are running with SELinux enforcing you'll need to use `audit2allow` to allow the Omni container to create wireguard tunnels. Something like `sudo ausearch -m avc -ts recent | grep -v mlsconstrain | audit2allow -M omni-container` should work. + ## 5. Create a cluster Before you create a cluster you will need to generate installation media. When creating a cluster you'll need to make sure you patch the machine config to redirect container registries to your internal registry. @@ -420,7 +465,7 @@ Download the images and push them into the internal registry. If your machine can reach the public internet and the internal registry at the same time you can copy the images internally with this command. ```bash -for SOURCE_IMAGE in $(cat images.txt) +for SOURCE_IMAGE in $(cat k8s-images.txt) do IMAGE_WITHOUT_DIGEST=${SOURCE_IMAGE%%@*} IMAGE_WITH_NEW_REG="${REGISTRY_ENDPOINT}/${IMAGE_WITHOUT_DIGEST#*/}" @@ -435,24 +480,24 @@ done If you don't have direct access to an internal container registry (e.g. air gapped environment) you need to download the container images while connected to the internet with this command: ```bash -cat images.txt \ +cat k8s-images.txt \ | talosctl images cache-create \ --layout flat \ - --image-cache-path ./image-cache \ + --image-cache-path ./k8s-image-cache \ --images=- ``` Move the `image-cache` folder to an air gapped machine and serve the images on a read only, temporary container registry with: ```bash -IP=$(hostname -I | awk '{print $1}') +export IP=$(hostname -I | awk '{print $1}') talosctl image cache-cert-gen \ --advertised-address $IP talosctl image cache-serve \ --address $IP:5000 \ - --image-cache-path ./image-cache \ + --image-cache-path ./k8s-image-cache \ --tls-cert-file tls.crt \ --tls-key-file tls.key ``` @@ -460,12 +505,12 @@ talosctl image cache-serve \ A temporary image registry will run on your local machine IP address port 5000 with self-signed certificates. Copy the images to an internal, permanent container registry. ```bash -for SOURCE_IMAGE in $(cat images.txt) +for SOURCE_IMAGE in $(cat k8s-images.txt) do IMAGE_WITHOUT_DIGEST=${SOURCE_IMAGE%%@*} IMAGE_WITH_NEW_REG="${REGISTRY_ENDPOINT}/${IMAGE_WITHOUT_DIGEST#*/}" - LOCALHOST_IMAGE="localhost:5000/${IMAGE_WITHOUT_DIGEST#*/}" - crane copy \ + LOCALHOST_IMAGE="${IP}:5000/${IMAGE_WITHOUT_DIGEST#*/}" + crane copy --insecure \ $LOCALHOST_IMAGE \ $IMAGE_WITH_NEW_REG done @@ -498,36 +543,31 @@ You can use this config two different ways. Use kernel arguments for Talos 1.11 ```bash mkdir _out mv trustedrootsconfig.yaml _out/machine-config.yaml + echo "---" >> _out/machine-config.yaml + yq eval --null-input ' +.apiVersion = "v1alpha1" | +.kind = "TimeSyncConfig" | +.enabled = false +' >> _out/machine-config.yaml echo "---" >> _out/machine-config.yaml ``` Download machine join configuration from Omni. You can do this from the Omni web interface home page by clicking on the **Download Machine Join Config** button or if you have `omnictl` installed you can download it with - ```bash - omnictl jointoken machine-config >> _out/machine-config.yaml - ``` - - Create a static hosts configuration for Omni and the registry. +> If your `omni.internal` endpoint is not resolvable via DNS, update your machine config to set the SideroLink endpoint to the IP address of the Omni machine instead of a hostname. - ```bash - yq --null-input ' - .apiVersion = "v1alpha1" | - .kind = "StaticHostConfig" | - .name = "'$(hostname -I | awk '{print $1}')'" | - .hostnames = ["'${OMNI_ENDPOINT%:*}'", "'${REGISTRY_ENDPOINT%:*}'"] -' > hosts-config.yaml - ``` - Append this configuration to the others. +>If you're shell still has `$OMNI_ENDPOINT` configured you may need to `unset OMNI_ENDPOINT` for `omnictl` commands to work ```bash - echo "---" >> _out/machine-config.yaml - cat hosts-config.yaml >> _out/machine-config.yaml + omnictl jointoken machine-config >> _out/machine-config.yaml ``` Embed both configurations and create an installation ISO with `imager`. + + Imager cannot be run with SELinux in enforcing mode. - {`docker run --rm -t \\\n -v "\${PWD}/_out:/out" \\\n --privileged \\\n \${REGISTRY_ENDPOINT}/siderolabs/imager:${release} \\\n iso \\\n --embedded-config-path=/out/machine-config.yaml`} + {`docker run --rm -t \\\n -v "\${PWD}/_out:/out" \\\n \${REGISTRY_ENDPOINT}/siderolabs/imager:${release} \\\n iso \\\n --embedded-config-path=/out/machine-config.yaml`} @@ -535,10 +575,14 @@ You can use this config two different ways. Use kernel arguments for Talos 1.11 Get the kernel arguments from Omni with `omnictl`. You can also copy them from the Omni web interface with the **Copy Kernel Parameters** button on the home page. +>If you're shell still has `$OMNI_ENDPOINT` configured you may need to `unset OMNI_ENDPOINT` for `omnictl` commands to work + ```bash OMNI_KERNEL_ARGS=$(omnictl jointoken kernel-args) ``` +> If your `omni.internal` endpoint is not resolvable via DNS, update your machine config to set the SideroLink endpoint to the IP address of the Omni machine instead of a hostname. + Compress and base64 encode the configuration for a kernel argument. ```bash @@ -547,6 +591,8 @@ You can use this config two different ways. Use kernel arguments for Talos 1.11 Create an ISO with `imager` with both of the kernel arguments. + Imager cannot be run with SELinux in enforcing mode. + {`docker run --rm -t \\\n -v "\${PWD}/_out:/out" \\\n --privileged \\\n \${REGISTRY_ENDPOINT}/siderolabs/imager:${release} \\\n iso \\\n --extra-kernel-arg "talos.config.early=\$TRUSTED_ROOT_CONFIG $OMNI_KERNEL_ARGS"`} @@ -564,34 +610,52 @@ Because we're working in an airgapped environment we will need the following val > **NOTE:** In this example, cluster discovery is also disabled. You may also configure cluster discovery via Omni. More information on the Discovery Service can be found here. -```yaml -machine: - registries: - mirrors: - docker.io: - endpoints: - - https://${REGISTRY_ENDPOINT} - gcr.io: - endpoints: - - https://${REGISTRY_ENDPOINT} - ghcr.io: - endpoints: - - https://${REGISTRY_ENDPOINT} - registry.k8s.io: - endpoints: - - https://${REGISTRY_ENDPOINT} -cluster: - discovery: - enabled: false ---- -apiVersion: v1alpha1 -kind: RegistryTLSConfig -name: ${REGISTRY_ENDPOINT} -ca: |- - -----BEGIN CERTIFICATE----- - MIID...IDAQAB - -----END CERTIFICATE----- +Export the air-gap configuration patch to a file: + +```bash +(yq eval --null-input ' +.machine.registries.mirrors."docker.io".endpoints = ["https://" + env(REGISTRY_ENDPOINT)] | +.machine.registries.mirrors."gcr.io".endpoints = ["https://" + env(REGISTRY_ENDPOINT)] | +.machine.registries.mirrors."ghcr.io".endpoints = ["https://" + env(REGISTRY_ENDPOINT)] | +.machine.registries.mirrors."registry.k8s.io".endpoints = ["https://" + env(REGISTRY_ENDPOINT)] +' && echo "---" && yq eval --null-input ' +.apiVersion = "v1alpha1" | +.kind = "TimeSyncConfig" | +.enabled = false +' && echo "---" && yq eval --null-input ' +.apiVersion = "v1alpha1" | +.kind = "RegistryTLSConfig" | +.name = env(REGISTRY_ENDPOINT) | +.ca = load_str("ca.pem") +' && echo "---" && yq eval --null-input ' +.apiVersion = "v1alpha1" | +.kind = "TrustedRootsConfig" | +.name = "internal-ca" | +.certificates = load_str("ca.pem") +') > air-gap-patch.yaml +``` + +Get the first machine UUID from Omni: + + +```bash +export OMNI_CP_MACHINE=$(omnictl get machines -o yaml \ + | yq '.metadata.id') +``` + +This will list all available machine UUIDs. Note down the UUIDs you want to use for your cluster (you'll need at least one control plane node and optionally worker nodes). + +Generate a cluster template using the patch file and machine UUIDs: + + +{`cat < cluster-template.yaml\nkind: Cluster\nname: air-gap-cluster\nkubernetes:\n version: v${k8s_release}\ntalos:\n version: ${release}\npatches:\n - file: air-gap-patch.yaml\n---\nkind: ControlPlane\nmachines:\n - $OMNI_CP_MACHINE\nEOF`} + + +Create the cluster using the template: + +```bash +omnictl cluster template sync --file cluster-template.yaml ``` -The machine patch should be added to the cluster during cluster creation and should be applied cluster wide. +The machine patch will be applied cluster-wide to all nodes.