diff --git a/stack-clients/docker-compose.yml b/stack-clients/docker-compose.yml index 5097ef0c..206bdb9b 100644 --- a/stack-clients/docker-compose.yml +++ b/stack-clients/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-client: - image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.58.0 + image: ghcr.io/theworldavatar/stack-client${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT secrets: - blazegraph_password - postgis_password diff --git a/stack-clients/pom.xml b/stack-clients/pom.xml index 725c7281..7b86396d 100644 --- a/stack-clients/pom.xml +++ b/stack-clients/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-clients - 1.58.0 + 1.59.0-backup-service-SNAPSHOT Stack Clients https://theworldavatar.io diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java index 85b7bca1..d2db1160 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/docker/DockerClient.java @@ -42,9 +42,11 @@ import com.github.dockerjava.api.command.InspectContainerCmd; import com.github.dockerjava.api.command.InspectExecCmd; import com.github.dockerjava.api.command.InspectExecResponse; +import com.github.dockerjava.api.command.InspectVolumeResponse; import com.github.dockerjava.api.command.ListConfigsCmd; import com.github.dockerjava.api.command.ListContainersCmd; import com.github.dockerjava.api.command.ListSecretsCmd; +import com.github.dockerjava.api.command.ListVolumesCmd; import com.github.dockerjava.api.command.RemoveConfigCmd; import com.github.dockerjava.api.command.RemoveSecretCmd; import com.github.dockerjava.api.exception.NotFoundException; @@ -639,6 +641,17 @@ public Optional getConfig(List configs, String configName) { } + public List getVolumes() { + try (ListVolumesCmd listVolumesCmd = internalClient.listVolumesCmd()) { + return listVolumesCmd + .exec() + .getVolumes() + .stream() + .filter(v -> v.getName() != null && v.getName().startsWith(StackClient.getStackName())) + .collect(Collectors.toList()); + } + } + public List getConfigs() { try (ListConfigsCmd listConfigsCmd = internalClient.listConfigsCmd()) { return listConfigsCmd diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java index 1a887cd4..f6ccdc25 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/clients/utils/JsonHelper.java @@ -1,11 +1,15 @@ package com.cmclinnovations.stack.clients.utils; +import java.io.File; import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.MalformedURLException; import java.nio.file.Files; import java.nio.file.Path; import javax.annotation.Nonnull; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.json.JsonMapper; @@ -31,4 +35,19 @@ public static final String handleFileValues(String value) { } return value; } + + /** + * Parses a JSON file into a JSON object. + * + * @param file Path to the file. + */ + public static final JsonNode readFile(String file) { + try { + return getMapper().readTree(new File(file)); + } catch (MalformedURLException ex) { + throw new IllegalArgumentException("Invalid file path: " + file, ex); + } catch (IOException ex) { + throw new UncheckedIOException("Failed to read file: " + file, ex); + } + } } diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java index e07c4b8d..0dc55104 100644 --- a/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/ContainerService.java @@ -25,6 +25,7 @@ import com.cmclinnovations.stack.services.config.ServiceConfig; import com.github.dockerjava.api.command.InspectImageCmd; import com.github.dockerjava.api.command.InspectImageResponse; +import com.github.dockerjava.api.command.InspectVolumeResponse; import com.github.dockerjava.api.model.ContainerSpec; import com.github.dockerjava.api.model.ServiceSpec; import com.github.dockerjava.api.model.TaskSpec; @@ -240,4 +241,7 @@ public String getDNSIPAddress() { return dockerClient.getDNSIPAddress(); } + protected List getVolumes() { + return dockerClient.getVolumes(); + } } diff --git a/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java new file mode 100644 index 00000000..4a3649dc --- /dev/null +++ b/stack-clients/src/main/java/com/cmclinnovations/stack/services/KopiaService.java @@ -0,0 +1,174 @@ +package com.cmclinnovations.stack.services; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.StringJoiner; + +import org.apache.commons.lang3.exception.UncheckedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.cmclinnovations.stack.clients.core.StackClient; +import com.cmclinnovations.stack.clients.utils.JsonHelper; +import com.cmclinnovations.stack.services.config.ServiceConfig; +import com.fasterxml.jackson.databind.JsonNode; +import com.github.dockerjava.api.command.InspectVolumeResponse; +import com.github.dockerjava.api.model.ContainerSpec; +import com.github.dockerjava.api.model.Mount; +import com.github.dockerjava.api.model.MountType; + +public class KopiaService extends ContainerService { + + public static final String TYPE = "kopia"; + + private static final String VOLUME_DIR = "/data/"; + private static final String KOPIA_PASSWORD_PATH = "/run/secrets/kopia_password"; + private static final String REPOSITORY_CONFIG = "/inputs/data/kopia/repository.config"; + private static final String SFTP_SSH_KEY_PATH = "/tmp/ssh_key"; + private static final String KNOWN_HOSTS_PATH = "/.ssh/known_hosts"; + private static final String ROOT_KNOWN_HOSTS_PATH = "/root" + KNOWN_HOSTS_PATH; + private static final String SCHEDULED_SCRIPT_PATH = "/usr/local/bin/kopia-backup.sh"; + + private static final String STORAGE_KEY = "storage"; + private static final String CREATE_REPO_ACTION = "create"; + private static final String CONNECT_REPO_ACTION = "connect"; + + private final String storageType; + private final String passwordFlag; + private final JsonNode storageConfig; + + private static final Logger LOGGER = LoggerFactory.getLogger(KopiaService.class); + + public KopiaService(String stackName, ServiceConfig config) { + super(stackName, config); + JsonNode repoConfig = JsonHelper.readFile(REPOSITORY_CONFIG).get(STORAGE_KEY); + this.storageConfig = repoConfig.get("config"); + this.storageType = repoConfig.get("type").asText(); + this.passwordFlag = "--password=\"$(cat " + KOPIA_PASSWORD_PATH + ")\""; + } + + @Override + public void doPreStartUpConfiguration() { + // Mount all volumes on the stack (except kopia) for backups + List volumes = super.getVolumes(); + List mounts = new ArrayList<>(); + + for (InspectVolumeResponse vol : volumes) { + if (!vol.getName().endsWith(TYPE)) { + // Remove stack name prefix (STACK_) from volume name + String volName = vol.getName().substring(StackClient.getStackName().length() + 1); + String destinationPath = VOLUME_DIR + volName; + mounts.add(new Mount() + .withType(MountType.VOLUME) // Use Docker volume for named volumes + .withSource(volName) + .withTarget(destinationPath) + .withReadOnly(false)); + } + } + + ContainerSpec containerSpec = super.getContainerSpec(); + containerSpec.withMounts(mounts); + } + + @Override + public void doFirstTimePostStartUpConfiguration() { + this.genScheduledBackups(); + } + + private void genScheduledBackups() { + LOGGER.info("Generating scheduled backup script..."); + String connectRepoCommand = this.genCreateOrConnectRepositoryCommand(CONNECT_REPO_ACTION); + String createRepoCommand = this.genCreateOrConnectRepositoryCommand(CREATE_REPO_ACTION); + String initSnapshotCommand = String.join(" ", TYPE, "snapshot", CREATE_REPO_ACTION, VOLUME_DIR, + this.passwordFlag); + String snapshotCommand = "kopia snapshot create --all " + this.passwordFlag; + + // Add a warning that this file is auto-generated + StringJoiner scriptContents = new StringJoiner("\n") + .add("#!/bin/bash") + .add("# AUTO-GENERATED BY JAVA - DO NOT EDIT MANUALLY") + .add(""); + + if (this.storageType.equals("sftp") && this.requiresExternalSSH()) { + scriptContents.add("eval `keychain --eval --agents ssh " + SFTP_SSH_KEY_PATH) + .add(""); + } + + scriptContents.add("if ! " + connectRepoCommand + "; then") + .add("\t" + createRepoCommand) + .add("\t" + connectRepoCommand) + .add("\t" + initSnapshotCommand) + .add("else") + .add("\t" + snapshotCommand) + .add("fi"); + + super.sendFileContent(SCHEDULED_SCRIPT_PATH, scriptContents.toString().getBytes(StandardCharsets.UTF_8)); + // Requires executable permission for root user for the crone job + super.createComplexCommand("chmod", "+x", SCHEDULED_SCRIPT_PATH) + .withUser("root") + .exec(); + } + + private String genCreateOrConnectRepositoryCommand(String action) { + String storagePath = this.storageConfig.get("path").asText(); + StringJoiner command = new StringJoiner(" ") + .add(TYPE).add("repository").add(action).add(this.storageType); + switch (this.storageType) { + case "sftp": + this.genSSHKeyFile(this.storageConfig.get("keyfile").asText()); + + String host = this.storageConfig.get("host").asText(); + String user = this.storageConfig.get("username").asText(); + if (action.equals(CREATE_REPO_ACTION) && !super.fileExists(ROOT_KNOWN_HOSTS_PATH)) { + super.executeCommand("sh", "-c", "ssh-keyscan -H " + host + " >> ~" + KNOWN_HOSTS_PATH); + } + command.add("--path=" + storagePath) + .add("--host=" + host) + .add("--username=" + user) + .add("--keyfile=" + SFTP_SSH_KEY_PATH) + .add("--known-hosts=" + ROOT_KNOWN_HOSTS_PATH); + if (this.requiresExternalSSH()) { + command.add("--external"); + } + break; + case "filesystem": + command.add("--path=" + storagePath); + break; + default: + LOGGER.warn( + "Unsupported storage type '{}' for automatic stack setup. Please manually configure the storage", + this.storageType); + break; + } + + command.add(this.passwordFlag); + return command.toString(); + } + + /** + * Checks if the repository requires external SSH key access due to passphrase + * protection. + */ + private boolean requiresExternalSSH() { + return this.storageConfig.has("externalSSH") && this.storageConfig.get("externalSSH").asBoolean(); + } + + /** + * Generates an SSH key file in the Kopia container if it doesn't already exist. + * + * @param keyFilePath Path to the SSH key file + */ + private void genSSHKeyFile(String keyFilePath) { + if (!super.fileExists(keyFilePath)) { + try { + super.sendFileContent(SFTP_SSH_KEY_PATH, Files.readAllBytes(Paths.get(keyFilePath))); + } catch (IOException e) { + throw new UncheckedException(e); + } + } + } +} diff --git a/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json new file mode 100644 index 00000000..03766a86 --- /dev/null +++ b/stack-clients/src/main/resources/com/cmclinnovations/stack/services/built-ins/kopia.json @@ -0,0 +1,16 @@ +{ + "type": "kopia", + "ServiceSpec": { + "Name": "kopia", + "TaskTemplate": { + "ContainerSpec": { + "Image": "ghcr.io/theworldavatar/kopia:0.1.0-backup-service-SNAPSHOT", + "Secrets": [ + { + "SecretName": "kopia_password" + } + ] + } + } + } +} \ No newline at end of file diff --git a/stack-data-uploader/docker-compose.yml b/stack-data-uploader/docker-compose.yml index a4e832e5..172a5ceb 100644 --- a/stack-data-uploader/docker-compose.yml +++ b/stack-data-uploader/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-data-uploader: - image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.58.0 + image: ghcr.io/theworldavatar/stack-data-uploader${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT secrets: - blazegraph_password - postgis_password diff --git a/stack-data-uploader/pom.xml b/stack-data-uploader/pom.xml index ab077088..ef79066e 100644 --- a/stack-data-uploader/pom.xml +++ b/stack-data-uploader/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-data-uploader - 1.58.0 + 1.59.0-backup-service-SNAPSHOT Stack Data Uploader https://theworldavatar.io @@ -38,7 +38,7 @@ com.cmclinnovations stack-clients - 1.58.0 + 1.59.0-backup-service-SNAPSHOT diff --git a/stack-manager/README.md b/stack-manager/README.md index 4e63b3a6..99891928 100644 --- a/stack-manager/README.md +++ b/stack-manager/README.md @@ -414,6 +414,69 @@ graph TB ``` +## Volume backup + +The stack includes an automated backup solution using [Kopia](https://kopia.io/). It is configured to perform daily snapshots of all volumes attached to the stack to a target destination. + +> [!IMPORTANT] +> Kopia can only access and back up volumes within the same stack + +### Prerequisites + +1) `stack.json`: Include the `kopia` service +2) `kopia_password`: A [Docker secret](#secrets) containing the master password for the Kopia repository +3) `./inputs/data/kopia/repository.config`: Contains the connection details for the backup destination (known as a Kopia repository); Note that this directory need not be mounted to the kopia service + +### Repository configuration + +Kopia requires the creation of a [repository](https://kopia.io/docs/repositories/) that specifies the destination of backups. Note that this automated implementation only supports a filesystem or sftp repository connections using the custom JSON-based `repository.config`. The skeleton template is as follows: + +```json +{ + "storage": { + "type": "sftp OR filesystem", + "config": { + ... // config options + } + } +} +``` + +#### 1. Filesystem + +```json +{ + "storage": { + "type": "filesystem", + "config": { + "path": "/backup", // local or network mounted path to storage location + } + } +} +``` + +#### 2. SFTP + +For the `SFTP` option, users must include a `ssh_key` file in the `./inputs/data/kopia/` directory containing the `SSH` key to access the repository. + +```json +{ + "storage": { + "type": "sftp", + "config": { + "path": "/backup", // path to destination + "host": "XXX.XXX.X.XX", // domain or ip address + "port": 22, + "username": "user", + "keyfile": "/inputs/data/kopia/ssh_key", // do NOT change + "externalSSH": false // optional and will default to false + } + } +} +``` + +If the `SSH` key is encrypted by a passphrase, please set the `externalSSH` flag to true and manually add the private passphrase in a terminal via the `keychain /tmp/ssh_key` command. + ## Example - including a visualisation This example explains how to spin up a TWA-VF based visualisation container within a stack. The visualisation container requires a volume called `vis-files` to be populated and secrets `mapbox_username`, and `mapbox_api_key` to be created. diff --git a/stack-manager/docker-compose.yml b/stack-manager/docker-compose.yml index 9981f1c4..e3dae325 100644 --- a/stack-manager/docker-compose.yml +++ b/stack-manager/docker-compose.yml @@ -1,6 +1,6 @@ services: stack-manager: - image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.58.0 + image: ghcr.io/theworldavatar/stack-manager${IMAGE_SUFFIX}:1.59.0-backup-service-SNAPSHOT environment: EXTERNAL_PORT: "${EXTERNAL_PORT-3838}" STACK_BASE_DIR: "${STACK_BASE_DIR}" diff --git a/stack-manager/pom.xml b/stack-manager/pom.xml index 7e6712a6..daf7dbcb 100644 --- a/stack-manager/pom.xml +++ b/stack-manager/pom.xml @@ -7,7 +7,7 @@ com.cmclinnovations stack-manager - 1.58.0 + 1.59.0-backup-service-SNAPSHOT Stack Manager https://theworldavatar.io @@ -38,7 +38,7 @@ com.cmclinnovations stack-clients - 1.58.0 + 1.59.0-backup-service-SNAPSHOT