diff --git a/stack-clients/docker-compose.yml b/stack-clients/docker-compose.yml
index 5097ef0c..206bdb9b 100644
--- a/stack-clients/docker-compose.yml
+++ b/stack-clients/docker-compose.yml
@@ -1,6 +1,6 @@
services:
stack-client:
- image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.58.0
+ image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
secrets:
- blazegraph_password
- postgis_password
diff --git a/stack-clients/pom.xml b/stack-clients/pom.xml
index 725c7281..7b86396d 100644
--- a/stack-clients/pom.xml
+++ b/stack-clients/pom.xml
@@ -7,7 +7,7 @@
com.cmclinnovations
stack-clients
- 1.58.0
+ 1.59.0-backup-service-SNAPSHOT
Stack Clients
https://theworldavatar.io
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
index 85b7bca1..d2db1160 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java
@@ -42,9 +42,11 @@
import com.github.dockerjava.api.command.InspectContainerCmd;
import com.github.dockerjava.api.command.InspectExecCmd;
import com.github.dockerjava.api.command.InspectExecResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.command.ListConfigsCmd;
import com.github.dockerjava.api.command.ListContainersCmd;
import com.github.dockerjava.api.command.ListSecretsCmd;
+import com.github.dockerjava.api.command.ListVolumesCmd;
import com.github.dockerjava.api.command.RemoveConfigCmd;
import com.github.dockerjava.api.command.RemoveSecretCmd;
import com.github.dockerjava.api.exception.NotFoundException;
@@ -639,6 +641,17 @@ public Optional getConfig(List configs, String configName) {
}
+ public List getVolumes() {
+ try (ListVolumesCmd listVolumesCmd = internalClient.listVolumesCmd()) {
+ return listVolumesCmd
+ .exec()
+ .getVolumes()
+ .stream()
+ .filter(v -> v.getName() != null && v.getName().startsWith(StackClient.getStackName()))
+ .collect(Collectors.toList());
+ }
+ }
+
public List getConfigs() {
try (ListConfigsCmd listConfigsCmd = internalClient.listConfigsCmd()) {
return listConfigsCmd
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
index 1a887cd4..f6ccdc25 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java
@@ -1,11 +1,15 @@
package com.cmclinnovations.stack.clients.utils;
+import java.io.File;
import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.MalformedURLException;
import java.nio.file.Files;
import java.nio.file.Path;
import javax.annotation.Nonnull;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.json.JsonMapper;
@@ -31,4 +35,19 @@ public static final String handleFileValues(String value) {
}
return value;
}
+
+ /**
+ * Parses a JSON file into a JSON object.
+ *
+ * @param file Path to the file.
+ */
+ public static final JsonNode readFile(String file) {
+ try {
+ return getMapper().readTree(new File(file));
+ } catch (MalformedURLException ex) {
+ throw new IllegalArgumentException("Invalid file path: " + file, ex);
+ } catch (IOException ex) {
+ throw new UncheckedIOException("Failed to read file: " + file, ex);
+ }
+ }
}
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
index e07c4b8d..0dc55104 100644
--- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java
@@ -25,6 +25,7 @@
import com.cmclinnovations.stack.services.config.ServiceConfig;
import com.github.dockerjava.api.command.InspectImageCmd;
import com.github.dockerjava.api.command.InspectImageResponse;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
import com.github.dockerjava.api.model.ContainerSpec;
import com.github.dockerjava.api.model.ServiceSpec;
import com.github.dockerjava.api.model.TaskSpec;
@@ -240,4 +241,7 @@ public String getDNSIPAddress() {
return dockerClient.getDNSIPAddress();
}
+ protected List getVolumes() {
+ return dockerClient.getVolumes();
+ }
}
diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
new file mode 100644
index 00000000..4a3649dc
--- /dev/null
+++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java
@@ -0,0 +1,174 @@
+package com.cmclinnovations.stack.services;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringJoiner;
+
+import org.apache.commons.lang3.exception.UncheckedException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.cmclinnovations.stack.clients.core.StackClient;
+import com.cmclinnovations.stack.clients.utils.JsonHelper;
+import com.cmclinnovations.stack.services.config.ServiceConfig;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.github.dockerjava.api.command.InspectVolumeResponse;
+import com.github.dockerjava.api.model.ContainerSpec;
+import com.github.dockerjava.api.model.Mount;
+import com.github.dockerjava.api.model.MountType;
+
+public class KopiaService extends ContainerService {
+
+ public static final String TYPE = "kopia";
+
+ private static final String VOLUME_DIR = "/data/";
+ private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password";
+ private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config";
+ private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key";
+ private static final String KNOWN_HOSTS_PATH = "/.ssh/known_hosts";
+ private static final String ROOT_KNOWN_HOSTS_PATH = "/root" + KNOWN_HOSTS_PATH;
+ private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh";
+
+ private static final String STORAGE_KEY = "storage";
+ private static final String CREATE_REPO_ACTION = "create";
+ private static final String CONNECT_REPO_ACTION = "connect";
+
+ private final String storageType;
+ private final String passwordFlag;
+ private final JsonNode storageConfig;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class);
+
+ public KopiaService(String stackName, ServiceConfig config) {
+ super(stackName, config);
+ JsonNode repoConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY);
+ this.storageConfig = repoConfig.get("config");
+ this.storageType = repoConfig.get("type").asText();
+ this.passwordFlag = "--password=\"$(cat " + KOPIA_PASSWORD_PATH + ")\"";
+ }
+
+ @Override
+ public void doPreStartUpConfiguration() {
+ // Mount all volumes on the stack (except kopia) for backups
+ List volumes = super.getVolumes();
+ List mounts = new ArrayList<>();
+
+ for (InspectVolumeResponse vol : volumes) {
+ if (!vol.getName().endsWith(TYPE)) {
+ // Remove stack name prefix (STACK_) from volume name
+ String volName = vol.getName().substring(StackClient.getStackName().length() + 1);
+ String destinationPath = VOLUME_DIR + volName;
+ mounts.add(new Mount()
+ .withType(MountType.VOLUME) // Use Docker volume for named volumes
+ .withSource(volName)
+ .withTarget(destinationPath)
+ .withReadOnly(false));
+ }
+ }
+
+ ContainerSpec containerSpec = super.getContainerSpec();
+ containerSpec.withMounts(mounts);
+ }
+
+ @Override
+ public void doFirstTimePostStartUpConfiguration() {
+ this.genScheduledBackups();
+ }
+
+ private void genScheduledBackups() {
+ LOGGER.info("Generating scheduled backup script...");
+ String connectRepoCommand = this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION);
+ String createRepoCommand = this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION);
+ String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR,
+ this.passwordFlag);
+ String snapshotCommand = "kopia snapshot create --all " + this.passwordFlag;
+
+ // Add a warning that this file is auto-generated
+ StringJoiner scriptContents = new StringJoiner("\n")
+ .add("#!/bin/bash")
+ .add("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY")
+ .add("");
+
+ if (this.storageType.equals("sftp") && this.requiresExternalSSH()) {
+ scriptContents.add("eval `keychain --eval --agents ssh " + SFTP_SSH_KEY_PATH)
+ .add("");
+ }
+
+ scriptContents.add("if ! " + connectRepoCommand + "; then")
+ .add("\t" + createRepoCommand)
+ .add("\t" + connectRepoCommand)
+ .add("\t" + initSnapshotCommand)
+ .add("else")
+ .add("\t" + snapshotCommand)
+ .add("fi");
+
+ super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8));
+ // Requires executable permission for root user for the crone job
+ super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH)
+ .withUser("root")
+ .exec();
+ }
+
+ private String genCreateOrConnectRepositoryCommand(String action) {
+ String storagePath = this.storageConfig.get("path").asText();
+ StringJoiner command = new StringJoiner(" ")
+ .add(TYPE).add("repository").add(action).add(this.storageType);
+ switch (this.storageType) {
+ case "sftp":
+ this.genSSHKeyFile(this.storageConfig.get("keyfile").asText());
+
+ String host = this.storageConfig.get("host").asText();
+ String user = this.storageConfig.get("username").asText();
+ if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) {
+ super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH);
+ }
+ command.add("--path=" + storagePath)
+ .add("--host=" + host)
+ .add("--username=" + user)
+ .add("--keyfile=" + SFTP_SSH_KEY_PATH)
+ .add("--known-hosts=" + ROOT_KNOWN_HOSTS_PATH);
+ if (this.requiresExternalSSH()) {
+ command.add("--external");
+ }
+ break;
+ case "filesystem":
+ command.add("--path=" + storagePath);
+ break;
+ default:
+ LOGGER.warn(
+ "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage",
+ this.storageType);
+ break;
+ }
+
+ command.add(this.passwordFlag);
+ return command.toString();
+ }
+
+ /**
+ * Checks if the repository requires external SSH key access due to passphrase
+ * protection.
+ */
+ private boolean requiresExternalSSH() {
+ return this.storageConfig.has("externalSSH") && this.storageConfig.get("externalSSH").asBoolean();
+ }
+
+ /**
+ * Generates an SSH key file in the Kopia container if it doesn't already exist.
+ *
+ * @param keyFilePath Path to the SSH key file
+ */
+ private void genSSHKeyFile(String keyFilePath) {
+ if (!super.fileExists(keyFilePath)) {
+ try {
+ super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath)));
+ } catch (IOException e) {
+ throw new UncheckedException(e);
+ }
+ }
+ }
+}
diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
new file mode 100644
index 00000000..03766a86
--- /dev/null
+++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json
@@ -0,0 +1,16 @@
+{
+ "type": "kopia",
+ "ServiceSpec": {
+ "Name": "kopia",
+ "TaskTemplate": {
+ "ContainerSpec": {
+ "Image": "ghcr.io/theworldavatar/kopia:0.1.0-backup-service-SNAPSHOT",
+ "Secrets": [
+ {
+ "SecretName": "kopia_password"
+ }
+ ]
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/stack-data-uploader/docker-compose.yml b/stack-data-uploader/docker-compose.yml
index a4e832e5..172a5ceb 100644
--- a/stack-data-uploader/docker-compose.yml
+++ b/stack-data-uploader/docker-compose.yml
@@ -1,6 +1,6 @@
services:
stack-data-uploader:
- image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.58.0
+ image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
secrets:
- blazegraph_password
- postgis_password
diff --git a/stack-data-uploader/pom.xml b/stack-data-uploader/pom.xml
index ab077088..ef79066e 100644
--- a/stack-data-uploader/pom.xml
+++ b/stack-data-uploader/pom.xml
@@ -7,7 +7,7 @@
com.cmclinnovations
stack-data-uploader
- 1.58.0
+ 1.59.0-backup-service-SNAPSHOT
Stack Data Uploader
https://theworldavatar.io
@@ -38,7 +38,7 @@
com.cmclinnovations
stack-clients
- 1.58.0
+ 1.59.0-backup-service-SNAPSHOT
diff --git a/stack-manager/README.md b/stack-manager/README.md
index 4e63b3a6..99891928 100644
--- a/stack-manager/README.md
+++ b/stack-manager/README.md
@@ -414,6 +414,69 @@ graph TB
```
+## Volume backup
+
+The stack includes an automated backup solution using [Kopia](https://kopia.io/). It is configured to perform daily snapshots of all volumes attached to the stack to a target destination.
+
+> [!IMPORTANT]
+> Kopia can only access and back up volumes within the same stack
+
+### Prerequisites
+
+1) `stack.json`: Include the `kopia` service
+2) `kopia_password`: A [Docker secret](#secrets) containing the master password for the Kopia repository
+3) `./inputs/data/kopia/repository.config`: Contains the connection details for the backup destination (known as a Kopia repository); Note that this directory need not be mounted to the kopia service
+
+### Repository configuration
+
+Kopia requires the creation of a [repository](https://kopia.io/docs/repositories/) that specifies the destination of backups. Note that this automated implementation only supports a filesystem or sftp repository connections using the custom JSON-based `repository.config`. The skeleton template is as follows:
+
+```json
+{
+ "storage": {
+ "type": "sftp OR filesystem",
+ "config": {
+ ... // config options
+ }
+ }
+}
+```
+
+#### 1. Filesystem
+
+```json
+{
+ "storage": {
+ "type": "filesystem",
+ "config": {
+ "path": "/backup", // local or network mounted path to storage location
+ }
+ }
+}
+```
+
+#### 2. SFTP
+
+For the `SFTP` option, users must include a `ssh_key` file in the `./inputs/data/kopia/` directory containing the `SSH` key to access the repository.
+
+```json
+{
+ "storage": {
+ "type": "sftp",
+ "config": {
+ "path": "/backup", // path to destination
+ "host": "XXX.XXX.X.XX", // domain or ip address
+ "port": 22,
+ "username": "user",
+ "keyfile": "/inputs/data/kopia/ssh_key", // do NOT change
+ "externalSSH": false // optional and will default to false
+ }
+ }
+}
+```
+
+If the `SSH` key is encrypted by a passphrase, please set the `externalSSH` flag to true and manually add the private passphrase in a terminal via the `keychain /tmp/ssh_key` command.
+
## Example - including a visualisation
This example explains how to spin up a TWA-VF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created.
diff --git a/stack-manager/docker-compose.yml b/stack-manager/docker-compose.yml
index 9981f1c4..e3dae325 100644
--- a/stack-manager/docker-compose.yml
+++ b/stack-manager/docker-compose.yml
@@ -1,6 +1,6 @@
services:
stack-manager:
- image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.58.0
+ image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT
environment:
EXTERNAL_PORT: "${EXTERNAL_PORT-3838}"
STACK_BASE_DIR: "${STACK_BASE_DIR}"
diff --git a/stack-manager/pom.xml b/stack-manager/pom.xml
index 7e6712a6..daf7dbcb 100644
--- a/stack-manager/pom.xml
+++ b/stack-manager/pom.xml
@@ -7,7 +7,7 @@
com.cmclinnovations
stack-manager
- 1.58.0
+ 1.59.0-backup-service-SNAPSHOT
Stack Manager
https://theworldavatar.io
@@ -38,7 +38,7 @@
com.cmclinnovations
stack-clients
- 1.58.0
+ 1.59.0-backup-service-SNAPSHOT