Qhouyee/83/backup service

stack-clients/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-client:
-    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.58.0
+    image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-clients/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-clients</artifactId>
-    <version>1.58.0</version>
+    <version>1.59.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Clients</name>
     <url>https://theworldavatar.io</url>

stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java

@@ -42,9 +42,11 @@
 import com.github.dockerjava.api.command.InspectContainerCmd;
 import com.github.dockerjava.api.command.InspectExecCmd;
 import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.command.ListConfigsCmd;
 import com.github.dockerjava.api.command.ListContainersCmd;
 import com.github.dockerjava.api.command.ListSecretsCmd;
+import com.github.dockerjava.api.command.ListVolumesCmd;
 import com.github.dockerjava.api.command.RemoveConfigCmd;
 import com.github.dockerjava.api.command.RemoveSecretCmd;
 import com.github.dockerjava.api.exception.NotFoundException;
@@ -639,6 +641,17 @@ public Optional<Config> getConfig(List<Config> configs, String configName) {
 
     }
 
+    public List<InspectVolumeResponse> getVolumes() {
+        try (ListVolumesCmd listVolumesCmd = internalClient.listVolumesCmd()) {
+            return listVolumesCmd
+                    .exec()
+                    .getVolumes()
+                    .stream()
+                    .filter(v -> v.getName() != null && v.getName().startsWith(StackClient.getStackName()))
+                    .collect(Collectors.toList());
+        }
+    }
+
     public List<Config> getConfigs() {
         try (ListConfigsCmd listConfigsCmd = internalClient.listConfigsCmd()) {
             return listConfigsCmd

stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java

@@ -1,11 +1,15 @@
 package com.cmclinnovations.stack.clients.utils;
 
+import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 
 import javax.annotation.Nonnull;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.json.JsonMapper;
 
@@ -31,4 +35,19 @@ public static final String handleFileValues(String value) {
         }
         return value;
     }
+
+    /**
+     * Parses a JSON file into a JSON object.
+     * 
+     * @param file Path to the file.
+     */
+    public static final JsonNode readFile(String file) {
+        try {
+            return getMapper().readTree(new File(file));
+        } catch (MalformedURLException ex) {
+            throw new IllegalArgumentException("Invalid file path: " + file, ex);
+        } catch (IOException ex) {
+            throw new UncheckedIOException("Failed to read file: " + file, ex);
+        }
+    }
 }

stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java

@@ -25,6 +25,7 @@
 import com.cmclinnovations.stack.services.config.ServiceConfig;
 import com.github.dockerjava.api.command.InspectImageCmd;
 import com.github.dockerjava.api.command.InspectImageResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
 import com.github.dockerjava.api.model.ContainerSpec;
 import com.github.dockerjava.api.model.ServiceSpec;
 import com.github.dockerjava.api.model.TaskSpec;
@@ -240,4 +241,7 @@ public String getDNSIPAddress() {
         return dockerClient.getDNSIPAddress();
     }
 
+    protected List<InspectVolumeResponse> getVolumes() {
+        return dockerClient.getVolumes();
+    }
 }

stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java

@@ -0,0 +1,174 @@
+package com.cmclinnovations.stack.services;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import org.apache.commons.lang3.exception.UncheckedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cmclinnovations.stack.clients.core.StackClient;
+import com.cmclinnovations.stack.clients.utils.JsonHelper;
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.model.ContainerSpec;
+import com.github.dockerjava.api.model.Mount;
+import com.github.dockerjava.api.model.MountType;
+
+public class KopiaService extends ContainerService {
+
+    public static final String TYPE = "kopia";
+
+    private static final String VOLUME_DIR = "/data/";
+    private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
+    private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+    private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key";
+    private static final String KNOWN_HOSTS_PATH = "/.ssh/known_hosts";
+    private static final String ROOT_KNOWN_HOSTS_PATH = "/root" + KNOWN_HOSTS_PATH;
+    private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
+
+    private static final String STORAGE_KEY = "storage";
+    private static final String CREATE_REPO_ACTION = "create";
+    private static final String CONNECT_REPO_ACTION = "connect";
+
+    private final String storageType;
+    private final String passwordFlag;
+    private final JsonNode storageConfig;
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
+
+    public KopiaService(String stackName, ServiceConfig config) {
+        super(stackName, config);
+        JsonNode repoConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+        this.storageConfig = repoConfig.get("config");
+        this.storageType = repoConfig.get("type").asText();
+        this.passwordFlag = "--password=\"$(cat " + KOPIA_PASSWORD_PATH + ")\"";
+    }
+
+    @Override
+    public void doPreStartUpConfiguration() {
+        // Mount all volumes on the stack (except kopia) for backups
+        List<InspectVolumeResponse> volumes = super.getVolumes();
+        List<Mount> mounts = new ArrayList<>();
+
+        for (InspectVolumeResponse vol : volumes) {
+            if (!vol.getName().endsWith(TYPE)) {
+                // Remove stack name prefix (STACK_) from volume name
+                String volName = vol.getName().substring(StackClient.getStackName().length() + 1);
+                String destinationPath = VOLUME_DIR + volName;
+                mounts.add(new Mount()
+                        .withType(MountType.VOLUME) // Use Docker volume for named volumes
+                        .withSource(volName)
+                        .withTarget(destinationPath)
+                        .withReadOnly(false));
+            }
+        }
+
+        ContainerSpec containerSpec = super.getContainerSpec();
+        containerSpec.withMounts(mounts);
+    }
+
+    @Override
+    public void doFirstTimePostStartUpConfiguration() {
+        this.genScheduledBackups();
+    }
+
+    private void genScheduledBackups() {
+        LOGGER.info("Generating scheduled backup script...");
+        String connectRepoCommand = this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION);
+        String createRepoCommand = this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION);
+        String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR,
+                this.passwordFlag);
+        String snapshotCommand = "kopia snapshot create --all " + this.passwordFlag;
+
+        // Add a warning that this file is auto-generated
+        StringJoiner scriptContents = new StringJoiner("\n")
+                .add("#!/bin/bash")
+                .add("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY")
+                .add("");
+
+        if (this.storageType.equals("sftp") && this.requiresExternalSSH()) {
+            scriptContents.add("eval `keychain --eval --agents ssh " + SFTP_SSH_KEY_PATH)
+                    .add("");
+        }
+
+        scriptContents.add("if ! " + connectRepoCommand + "; then")
+                .add("\t" + createRepoCommand)
+                .add("\t" + connectRepoCommand)
+                .add("\t" + initSnapshotCommand)
+                .add("else")
+                .add("\t" + snapshotCommand)
+                .add("fi");
+
+        super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
+        // Requires executable permission for root user for the crone job
+        super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
+                .withUser("root")
+                .exec();
+    }
+
+    private String genCreateOrConnectRepositoryCommand(String action) {
+        String storagePath = this.storageConfig.get("path").asText();
+        StringJoiner command = new StringJoiner(" ")
+                .add(TYPE).add("repository").add(action).add(this.storageType);
+        switch (this.storageType) {
+            case "sftp":
+                this.genSSHKeyFile(this.storageConfig.get("keyfile").asText());
+
+                String host = this.storageConfig.get("host").asText();
+                String user = this.storageConfig.get("username").asText();
+                if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) {
+                    super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH);
+                }
+                command.add("--path=" + storagePath)
+                        .add("--host=" + host)
+                        .add("--username=" + user)
+                        .add("--keyfile=" + SFTP_SSH_KEY_PATH)
+                        .add("--known-hosts=" + ROOT_KNOWN_HOSTS_PATH);
+                if (this.requiresExternalSSH()) {
+                    command.add("--external");
+                }
+                break;
+            case "filesystem":
+                command.add("--path=" + storagePath);
+                break;
+            default:
+                LOGGER.warn(
+                        "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage",
+                        this.storageType);
+                break;
+        }
+
+        command.add(this.passwordFlag);
+        return command.toString();
+    }
+
+    /**
+     * Checks if the repository requires external SSH key access due to passphrase
+     * protection.
+     */
+    private boolean requiresExternalSSH() {
+        return this.storageConfig.has("externalSSH") && this.storageConfig.get("externalSSH").asBoolean();
+    }
+
+    /**
+     * Generates an SSH key file in the Kopia container if it doesn't already exist.
+     * 
+     * @param keyFilePath Path to the SSH key file
+     */
+    private void genSSHKeyFile(String keyFilePath) {
+        if (!super.fileExists(keyFilePath)) {
+            try {
+                super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
+            } catch (IOException e) {
+                throw new UncheckedException(e);
+            }
+        }
+    }
+}

stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json

@@ -0,0 +1,16 @@
+{
+  "type": "kopia",
+  "ServiceSpec": {
+    "Name": "kopia",
+    "TaskTemplate": {
+      "ContainerSpec": {
+        "Image": "ghcr.io/theworldavatar/kopia:0.1.0-backup-service-SNAPSHOT",
+        "Secrets": [
+          {
+            "SecretName": "kopia_password"
+          }
+        ]
+      }
+    }
+  }
+}

stack-data-uploader/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-data-uploader:
-    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.58.0
+    image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
     secrets:
       - blazegraph_password
       - postgis_password

stack-data-uploader/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>com.cmclinnovations</groupId>
     <artifactId>stack-data-uploader</artifactId>
-    <version>1.58.0</version>
+    <version>1.59.0-backup-service-SNAPSHOT</version>
 
     <name>Stack Data Uploader</name>
     <url>https://theworldavatar.io</url>
@@ -38,7 +38,7 @@
         <dependency>
             <groupId>com.cmclinnovations</groupId>
             <artifactId>stack-clients</artifactId>
-            <version>1.58.0</version>
+            <version>1.59.0-backup-service-SNAPSHOT</version>
         </dependency>
 
         <dependency>

stack-manager/README.md

@@ -414,6 +414,69 @@ graph TB
 
 ```
 
+## Volume backup
+
+The stack includes an automated backup solution using [Kopia](https://kopia.io/). It is configured to perform daily snapshots of all volumes attached to the stack to a target destination.
+
+> [!IMPORTANT]  
+> Kopia can only access and back up volumes within the same stack
+
+### Prerequisites
+
+1) `stack.json`: Include the `kopia` service
+2) `kopia_password`: A [Docker secret](#secrets) containing the master password for the Kopia repository
+3) `./inputs/data/kopia/repository.config`: Contains the connection details for the backup destination (known as a Kopia repository); Note that this directory need not be mounted to the kopia service
+
+### Repository configuration
+
+Kopia requires the creation of a [repository](https://kopia.io/docs/repositories/) that specifies the destination of backups. Note that this automated implementation only supports a filesystem or sftp repository connections using the custom JSON-based `repository.config`. The skeleton template is as follows:
+
+```json
+{
+  "storage": {
+    "type": "sftp OR filesystem",
+    "config": {
+      ... // config options
+    }
+  }
+}
+```
+
+#### 1. Filesystem
+
+```json
+{
+  "storage": {
+    "type": "filesystem",
+    "config": {
+        "path": "/backup", // local or network mounted path to storage location
+    }
+  }
+}
+```
+
+#### 2. SFTP
+
+For the `SFTP` option, users must include a `ssh_key` file in the `./inputs/data/kopia/` directory containing the `SSH` key to access the repository.
+
+```json
+{
+  "storage": {
+    "type": "sftp",
+    "config": {
+        "path": "/backup", // path to destination
+        "host": "XXX.XXX.X.XX", // domain or ip address
+        "port": 22,
+        "username": "user",
+        "keyfile": "/inputs/data/kopia/ssh_key", // do NOT change
+        "externalSSH": false // optional and will default to false
+    }
+  }
+}
+```
+
+If the `SSH` key is encrypted by a passphrase, please set the `externalSSH` flag to true and manually add the private passphrase in a terminal via the `keychain /tmp/ssh_key` command.
+
 ## Example - including a visualisation
 
 This example explains how to spin up a TWA-VF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created.

stack-manager/docker-compose.yml

@@ -1,6 +1,6 @@
 services:
   stack-manager:
-    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.58.0
+    image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
     environment:
       EXTERNAL_PORT: "${EXTERNAL_PORT-3838}"
       STACK_BASE_DIR: "${STACK_BASE_DIR}"