From a100773be03194a6c99d800a079f7863e42584c6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 14:52:03 +0200 Subject: [PATCH 01/10] blake3: add to requirements and borg benchmark cpu --- pyproject.toml | 1 + src/borg/archiver/benchmark_cmd.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de1f852d51..c4ab56bdea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "xxhash>=2.0.0", "jsonargparse>=4.47.0", "PyYAML>=6.0.2", # we need to register our types with yaml, jsonargparse uses yaml for config files + "blake3>=1.0.0", ] [project.optional-dependencies] diff --git a/src/borg/archiver/benchmark_cmd.py b/src/borg/archiver/benchmark_cmd.py index 6cf5a4f1e1..e436a5ec45 100644 --- a/src/borg/archiver/benchmark_cmd.py +++ b/src/borg/archiver/benchmark_cmd.py @@ -232,16 +232,20 @@ def chunkit(ch): print(f"{spec:<24} {format_file_size(size):<10} {dt:.3f}s") from ..crypto.low_level import hmac_sha256, blake2b_256 + import blake3 if not args.json: print("Cryptographic hashes / MACs ====================================") else: result["hashes"] = [] size = 1000000000 - for spec, func in [ + hashes_tests = [ ("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)), ("blake2b-256", lambda: blake2b_256(key_256, random_10M)), - ]: + ("blake3", lambda: blake3.blake3(random_10M, key=key_256).digest()), + ("blake3-mt", lambda: blake3.blake3(random_10M, key=key_256, max_threads=blake3.blake3.AUTO).digest()), + ] + for spec, func in hashes_tests: dt = timeit(func, number=number_default) if args.json: result["hashes"].append({"algo": spec, "size": size, "time": dt}) From c32e4f848b7c61e3ab567c9cee78af7f2e0fba91 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 15:02:55 +0200 Subject: [PATCH 02/10] CI: install rust on omniOS --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8317c0a638..671f132381 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -537,7 +537,7 @@ jobs: ;; omnios) - sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils + sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils rust sudo ln -sf /usr/bin/python3.13 /usr/bin/python3 sudo ln -sf /usr/bin/python3.13-config /usr/bin/python3-config sudo python3 -m ensurepip From 8831bc9ea8a6d7532eac9271d6e3557991097a28 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 15:32:48 +0200 Subject: [PATCH 03/10] CI: install rust and python-maturin for windows/msys2 --- scripts/msys2-install-deps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/msys2-install-deps b/scripts/msys2-install-deps index 6e6836dd28..e831dc6793 100644 --- a/scripts/msys2-install-deps +++ b/scripts/msys2-install-deps @@ -1,6 +1,6 @@ #!/bin/bash -pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko} +pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin} if [ "$1" = "development" ]; then pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist} From 4ca60de44c5bd46d2d285548d9d7bdd5f999a049 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 15:06:22 +0200 Subject: [PATCH 04/10] CI: fix building with rust on Haiku OS --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 671f132381..6aea5619e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -566,7 +566,7 @@ jobs: # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade - python3 -m pip install --upgrade pip wheel + python3 -m pip install --upgrade pip setuptools wheel python3 -m venv .venv . .venv/bin/activate From f6ea8397f7753b8fb7978704f61b342e9f98d54b Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 19:29:46 +0200 Subject: [PATCH 05/10] CI: haiku: debugging maturin / cargo --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6aea5619e8..e25e4284cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -559,11 +559,13 @@ jobs: pkgman refresh pkgman install -y git pkgconfig lz4 pkgman install -y openssl3 - pkgman install -y rust_bin + pkgman install -y rust_bin setuptools_rust pkgman install -y python3.10 pkgman install -y cffi pkgman install -y lz4_devel openssl3_devel libffi_devel + cargo --version + # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade python3 -m pip install --upgrade pip setuptools wheel @@ -574,6 +576,8 @@ jobs: export BORG_LIBLZ4_PREFIX=/system/develop export BORG_OPENSSL_PREFIX=/system/develop pip install -r requirements.d/development.lock.txt + pip install -v maturin + pip install -v blake3 pip install -e . # troubles with either tox or pytest xdist, so we run pytest manually: From 95f25751ae4a6ec6892f64c7ede2ae5faa637eb9 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Thu, 14 May 2026 23:46:21 +0200 Subject: [PATCH 06/10] crypto: integrate blake3, blake2b is legacy, fixes #8867 BLAKE3 is generally faster and provides a more modern construction for keyed hashing (using its internal keyed mode instead of the construction used for BLAKE2b). Key types changed: - authenticated-blake2 -> authenticated-blake3 - {keyfile,repokey}-blake2-aes-ocb -> {keyfile,repokey}-blake3-aes-ocb - {keyfile,repokey}-blake2-chacha20-poly1305 -> {keyfile,repokey}-blake3-chacha20-poly1305 This also fixes the slightly unusual way how we used blake2b, it is only supported for importing borg 1.x repos. New repos either use HMAC-SHA256 or BLAKE3. --- docs/usage/transfer.rst | 66 ++++++-- src/borg/archiver/key_cmds.py | 20 +-- src/borg/archiver/transfer_cmd.py | 11 +- src/borg/constants.py | 9 +- src/borg/crypto/key.py | 151 +++++++++---------- src/borg/legacy/crypto/key.py | 47 +++++- src/borg/testsuite/archiver/key_cmds_test.py | 16 +- src/borg/testsuite/crypto/key_test.py | 27 +++- 8 files changed, 211 insertions(+), 136 deletions(-) diff --git a/docs/usage/transfer.rst b/docs/usage/transfer.rst index fe186e69be..8632cf9d01 100644 --- a/docs/usage/transfer.rst +++ b/docs/usage/transfer.rst @@ -2,33 +2,69 @@ Examples ~~~~~~~~ + +To keep the following examples short and readable, we export the repository +locations and passphrases first: + +:: + + export BORG_REPO=ssh://borg2@borgbackup/./tests/b20 + export BORG_PASSPHRASE='your-borg2-repo-passphrase' + export BORG_OTHER_REPO=ssh://borg2@borgbackup/./tests/b1x + export BORG_OTHER_PASSPHRASE='your-borg1-repo-passphrase' + :: - # 0. Have Borg 2.0 installed on the client AND server; have a b12 repository copy for testing. + # borg 1.x repo -> borg 2.0 repo (hmac-sha256 -> hmac-sha256, keeping same chunk ID algorithm) + + # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing. # 1. Create a new "related" repository: - # Here, the existing Borg 1.2 repository used repokey-blake2 (and AES-CTR mode), - # thus we use repokey-blake2-aes-ocb for the new Borg 2.0 repository. - # Staying with the same chunk ID algorithm (BLAKE2) and with the same - # key material (via --other-repo ) will make deduplication work + # Here, the existing Borg 1.x repository used repokey (and AES-CTR mode), + # thus we use repokey-aes-ocb for the new Borg 2.0 repository. + # Staying with the same chunk ID algorithm (hmac-sha256) and with the same + # key material (via BORG_OTHER_REPO) will make deduplication work # between old archives (copied with borg transfer) and future ones. # The AEAD cipher does not matter (everything must be re-encrypted and - # re-authenticated anyway); you could also choose repokey-blake2-chacha20-poly1305. - # In case your old Borg repository did not use BLAKE2, just remove the "-blake2". - $ borg --repo ssh://borg2@borgbackup/./tests/b20 repo-create \ - --other-repo ssh://borg2@borgbackup/./tests/b12 -e repokey-blake2-aes-ocb + # re-authenticated anyway); you could also choose repokey-chacha20-poly1305. + $ borg repo-create -e repokey-aes-ocb + + # 2. Check what and how much it would transfer: + $ borg transfer --from-borg1 --dry-run + + # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!): + $ borg transfer --from-borg1 + + # 4. Check whether we have everything (same as step 2): + $ borg transfer --from-borg1 --dry-run + +:: + + # borg 1.x repo -> borg 2.0 repo (blake2 -> blake3, changing chunk ID algorithm) + + # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing. + + # 1. Create a new "related" repository: + # Here, the existing Borg 1.x repository used repokey-blake2 (and AES-CTR mode), + # thus we use repokey-blake3-aes-ocb for the new Borg 2.0 repository. + # We need to change from blake2 to blake3, because blake2 is not supported + # for borg2 repos (blake3 is much faster). Because we change how chunk IDs are + # computed, we need to re-chunk everything while doing the transfer. + # The chunker parameters you provide here should be the same as you will + # use for all future Borg 2.0 archives. + # The AEAD cipher does not matter (everything must be re-encrypted and + # re-authenticated anyway); you could also choose repokey-blake3-chacha20-poly1305. + $ borg repo-create -e repokey-blake3-aes-ocb + $ export CHUNKER_PARAMS="buzhash64,19,23,21,4095" # 2. Check what and how much it would transfer: - $ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \ - --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run + $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!): - $ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \ - --other-repo ssh://borg2@borgbackup/./tests/b12 + $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS # 4. Check whether we have everything (same as step 2): - $ borg --repo ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \ - --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run + $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run Keyfile considerations when upgrading from borg 1.x ++++++++++++++++++++++++++++++++++++++++++++++++++++ diff --git a/src/borg/archiver/key_cmds.py b/src/borg/archiver/key_cmds.py index 5bdfc0b5dc..23f7c6cee0 100644 --- a/src/borg/archiver/key_cmds.py +++ b/src/borg/archiver/key_cmds.py @@ -1,8 +1,8 @@ import os from ..constants import * # NOQA -from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake2AESOCBRepoKey, Blake2CHPORepoKey -from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake2AESOCBKeyfileKey, Blake2CHPOKeyfileKey +from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake3AESOCBRepoKey, Blake3CHPORepoKey +from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake3AESOCBKeyfileKey, Blake3CHPOKeyfileKey from ..crypto.keymanager import KeyManager from ..helpers import PathSpec, CommandError from ..helpers.argparsing import ArgumentParser @@ -40,10 +40,10 @@ def do_key_change_location(self, args, repository, manifest, cache): key_new = AESOCBKeyfileKey(repository) elif isinstance(key, CHPORepoKey): key_new = CHPOKeyfileKey(repository) - elif isinstance(key, Blake2AESOCBRepoKey): - key_new = Blake2AESOCBKeyfileKey(repository) - elif isinstance(key, Blake2CHPORepoKey): - key_new = Blake2CHPOKeyfileKey(repository) + elif isinstance(key, Blake3AESOCBRepoKey): + key_new = Blake3AESOCBKeyfileKey(repository) + elif isinstance(key, Blake3CHPORepoKey): + key_new = Blake3CHPOKeyfileKey(repository) else: print("Change not needed or not supported.") return @@ -52,10 +52,10 @@ def do_key_change_location(self, args, repository, manifest, cache): key_new = AESOCBRepoKey(repository) elif isinstance(key, CHPOKeyfileKey): key_new = CHPORepoKey(repository) - elif isinstance(key, Blake2AESOCBKeyfileKey): - key_new = Blake2AESOCBRepoKey(repository) - elif isinstance(key, Blake2CHPOKeyfileKey): - key_new = Blake2CHPORepoKey(repository) + elif isinstance(key, Blake3AESOCBKeyfileKey): + key_new = Blake3AESOCBRepoKey(repository) + elif isinstance(key, Blake3CHPOKeyfileKey): + key_new = Blake3CHPORepoKey(repository) else: print("Change not needed or not supported.") return diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index e37ddf57b2..b5d010de03 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -134,12 +134,11 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non """archives transfer from other repository, optionally upgrade data format""" key = manifest.key other_key = other_manifest.key - if not uses_same_id_hash(other_key, key): - raise Error( - "You must keep the same ID hash ([HMAC-]SHA256 or BLAKE2b) or deduplication will break. " - "Use a related repository!" - ) - if not uses_same_chunker_secret(other_key, key): + using_same_id_hash = uses_same_id_hash(other_key, key) + rechunking = args.chunker_params is not None + if not using_same_id_hash and not rechunking: + raise Error("You must either keep the same ID hash or use --chunker-params.") + if not rechunking and not uses_same_chunker_secret(other_key, key): raise Error( "You must use the same chunker secret or deduplication will break. " "Use a related repository!" ) diff --git a/src/borg/constants.py b/src/borg/constants.py index f128cc2f88..3383f5d773 100644 --- a/src/borg/constants.py +++ b/src/borg/constants.py @@ -191,10 +191,11 @@ class KeyType: AESOCBREPO = 0x11 CHPOKEYFILE = 0x20 CHPOREPO = 0x21 - BLAKE2AESOCBKEYFILE = 0x30 - BLAKE2AESOCBREPO = 0x31 - BLAKE2CHPOKEYFILE = 0x40 - BLAKE2CHPOREPO = 0x41 + BLAKE3AESOCBKEYFILE = 0x30 + BLAKE3AESOCBREPO = 0x31 + BLAKE3CHPOKEYFILE = 0x40 + BLAKE3CHPOREPO = 0x41 + BLAKE3AUTHENTICATED = 0x50 CACHE_TAG_NAME = "CACHEDIR.TAG" diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 30044bf920..bfdb100599 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -11,6 +11,7 @@ logger = create_logger() +from blake3 import blake3 import argon2.low_level from ..constants import * # NOQA @@ -28,11 +29,11 @@ from ..repoobj import RepoObj -from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256 +from .low_level import AES, bytes_to_int, num_cipher_blocks, hmac_sha256 from .low_level import AES256_OCB, CHACHA20_POLY1305 from . import low_level -# workaround for lost passphrase or key in "authenticated" or "authenticated-blake2" mode +# workaround for lost passphrase or key in "authenticated*" modes AUTHENTICATED_NO_KEY = "authenticated_no_key" in workarounds @@ -123,19 +124,19 @@ def uses_same_id_hash(other_key, key): new_sha256_ids = (PlaintextKey,) old_hmac_sha256_ids = (RepoKey, KeyfileKey, AuthenticatedKey) new_hmac_sha256_ids = (AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey, AuthenticatedKey) - old_blake2_ids = (Blake2RepoKey, Blake2KeyfileKey, Blake2AuthenticatedKey) - new_blake2_ids = ( - Blake2AESOCBRepoKey, - Blake2AESOCBKeyfileKey, - Blake2CHPORepoKey, - Blake2CHPOKeyfileKey, - Blake2AuthenticatedKey, + # note: we do not support blake2b for new repos, see #8867 + new_blake3_ids = ( + Blake3AESOCBRepoKey, + Blake3AESOCBKeyfileKey, + Blake3CHPORepoKey, + Blake3CHPOKeyfileKey, + Blake3AuthenticatedKey, ) same_ids = ( isinstance(other_key, old_hmac_sha256_ids + new_hmac_sha256_ids) and isinstance(key, new_hmac_sha256_ids) - or isinstance(other_key, old_blake2_ids + new_blake2_ids) - and isinstance(key, new_blake2_ids) + or isinstance(other_key, new_blake3_ids) + and isinstance(key, new_blake3_ids) or isinstance(other_key, old_sha256_ids + new_sha256_ids) and isinstance(key, new_sha256_ids) ) @@ -276,38 +277,6 @@ def decrypt(self, id, data): return memoryview(data)[1:] -def random_blake2b_256_key(): - # This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b. - # Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b - # has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a - # "local wide pipe" design, because the compression function transforms (block, state) => state, - # and len(block) >= len(state), hence wide.) - # In other words, a key longer than 64 bytes would have simply no advantage, since the function - # has no way of propagating more than 64 bytes of entropy internally. - # It's padded to a full block so that the key is never buffered internally by blake2b_update, ie. - # it remains in a single memory location that can be tracked and could be erased securely, if we - # wanted to. - return os.urandom(64) + bytes(64) - - -class ID_BLAKE2b_256: - """ - Key mix-in class for using BLAKE2b-256 for the id key. - - The id_key length must be 32 bytes. - """ - - def id_hash(self, data): - return blake2b_256(self.id_key, data) - - def init_from_random_data(self): - super().init_from_random_data() - enc_key = os.urandom(32) - enc_hmac_key = random_blake2b_256_key() - self.crypt_key = enc_key + enc_hmac_key - self.id_key = random_blake2b_256_key() - - class ID_HMAC_SHA_256: """ Key mix-in class for using HMAC-SHA-256 for the id key. @@ -558,15 +527,18 @@ def create(cls, repository, args, *, other_key=None): if isinstance(key, AESKeyBase): # user must use an AEADKeyBase subclass (AEAD modes with session keys) raise Error("Copying key material to an AES-CTR based mode is insecure and unsupported.") - if not uses_same_id_hash(other_key, key): - raise Error("You must keep the same ID hash (HMAC-SHA256 or BLAKE2b) or deduplication will break.") if other_key.copy_crypt_key: # give the user the option to use the same authenticated encryption (AE) key crypt_key = other_key.crypt_key else: # borg transfer re-encrypts all data anyway, thus we can default to a new, random AE key crypt_key = os.urandom(64) - key.init_from_given_data(crypt_key=crypt_key, id_key=other_key.id_key, chunk_seed=other_key.chunk_seed) + if len(other_key.id_key) == 128: # blake2b id key from borg 1.x is not supported anymore + id_key = os.urandom(32) # hmac-sha256 and blake3 use 32 bytes + else: + id_key = other_key.id_key + chunk_seed = other_key.chunk_seed + key.init_from_given_data(crypt_key=crypt_key, id_key=id_key, chunk_seed=chunk_seed) else: key.init_from_random_data() passphrase = Passphrase.new(allow_empty=True) @@ -729,12 +701,6 @@ def remove(self, target): raise TypeError("Unsupported borg key storage type") -# legacy imports placed after FlexiKey/AESKeyBase/KeyBase so those names are already -# in the partial module when legacy/crypto/key.py imports them back during circular load -from ..legacy.crypto.key import KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey # noqa: E402 -from ..legacy.crypto.key import LEGACY_KEY_TYPES # noqa: E402 - - class AuthenticatedKeyBase(AESKeyBase, FlexiKey): STORAGE = KeyBlobStorage.REPO @@ -774,6 +740,14 @@ def decrypt(self, id, data): return memoryview(data)[1:] +# legacy imports placed after FlexiKey/AESKeyBase/KeyBase/AuthenticatedKeyBase so those names are already +# in the partial module when legacy/crypto/key.py imports them back during circular load +from ..legacy.crypto.key import KeyfileKey, RepoKey +from ..legacy.crypto.key import Blake2KeyfileKey, Blake2RepoKey, Blake2AuthenticatedKey # noqa: F401 +from ..legacy.crypto.key import LEGACY_KEY_TYPES # noqa: E402 +from ..legacy.crypto.key import ID_BLAKE2b_256 # noqa: F401 + + class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase): TYPE = KeyType.AUTHENTICATED TYPES_ACCEPTABLE = {TYPE} @@ -781,14 +755,25 @@ class AuthenticatedKey(ID_HMAC_SHA_256, AuthenticatedKeyBase): ARG_NAME = "authenticated" -class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): - TYPE = KeyType.BLAKE2AUTHENTICATED - TYPES_ACCEPTABLE = {TYPE} - NAME = "authenticated BLAKE2b" - ARG_NAME = "authenticated-blake2" +# ------------ new crypto ------------ -# ------------ new crypto ------------ +class ID_BLAKE3_256: + """ + Key mix-in class for using BLAKE3 for the id key. + + The id_key length must be 32 bytes. + """ + + def id_hash(self, data): + return blake3(data, key=self.id_key).digest(length=32) + + +class Blake3AuthenticatedKey(ID_BLAKE3_256, AuthenticatedKeyBase): + TYPE = KeyType.BLAKE3AUTHENTICATED + TYPES_ACCEPTABLE = {TYPE} + NAME = "authenticated BLAKE3" + ARG_NAME = "authenticated-blake3" class AEADKeyBase(KeyBase): @@ -936,38 +921,38 @@ class CHPORepoKey(ID_HMAC_SHA_256, AEADKeyBase, FlexiKey): CIPHERSUITE = CHACHA20_POLY1305 -class Blake2AESOCBKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO} - TYPE = KeyType.BLAKE2AESOCBKEYFILE - NAME = "key file BLAKE2b AES-OCB" - ARG_NAME = "keyfile-blake2-aes-ocb" +class Blake3AESOCBKeyfileKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE3AESOCBKEYFILE, KeyType.BLAKE3AESOCBREPO} + TYPE = KeyType.BLAKE3AESOCBKEYFILE + NAME = "key file BLAKE3 AES-OCB" + ARG_NAME = "keyfile-blake3-aes-ocb" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = AES256_OCB -class Blake2AESOCBRepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2AESOCBKEYFILE, KeyType.BLAKE2AESOCBREPO} - TYPE = KeyType.BLAKE2AESOCBREPO - NAME = "repokey BLAKE2b AES-OCB" - ARG_NAME = "repokey-blake2-aes-ocb" +class Blake3AESOCBRepoKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE3AESOCBKEYFILE, KeyType.BLAKE3AESOCBREPO} + TYPE = KeyType.BLAKE3AESOCBREPO + NAME = "repokey BLAKE3 AES-OCB" + ARG_NAME = "repokey-blake3-aes-ocb" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = AES256_OCB -class Blake2CHPOKeyfileKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO} - TYPE = KeyType.BLAKE2CHPOKEYFILE - NAME = "key file BLAKE2b ChaCha20-Poly1305" - ARG_NAME = "keyfile-blake2-chacha20-poly1305" +class Blake3CHPOKeyfileKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE3CHPOKEYFILE, KeyType.BLAKE3CHPOREPO} + TYPE = KeyType.BLAKE3CHPOKEYFILE + NAME = "key file BLAKE3 ChaCha20-Poly1305" + ARG_NAME = "keyfile-blake3-chacha20-poly1305" STORAGE = KeyBlobStorage.KEYFILE CIPHERSUITE = CHACHA20_POLY1305 -class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): - TYPES_ACCEPTABLE = {KeyType.BLAKE2CHPOKEYFILE, KeyType.BLAKE2CHPOREPO} - TYPE = KeyType.BLAKE2CHPOREPO - NAME = "repokey BLAKE2b ChaCha20-Poly1305" - ARG_NAME = "repokey-blake2-chacha20-poly1305" +class Blake3CHPORepoKey(ID_BLAKE3_256, AEADKeyBase, FlexiKey): + TYPES_ACCEPTABLE = {KeyType.BLAKE3CHPOKEYFILE, KeyType.BLAKE3CHPOREPO} + TYPE = KeyType.BLAKE3CHPOREPO + NAME = "repokey BLAKE3 ChaCha20-Poly1305" + ARG_NAME = "repokey-blake3-chacha20-poly1305" STORAGE = KeyBlobStorage.REPO CIPHERSUITE = CHACHA20_POLY1305 @@ -977,14 +962,14 @@ class Blake2CHPORepoKey(ID_BLAKE2b_256, AEADKeyBase, FlexiKey): # not encrypted modes PlaintextKey, AuthenticatedKey, - Blake2AuthenticatedKey, # new crypto + Blake3AuthenticatedKey, AESOCBKeyfileKey, AESOCBRepoKey, CHPOKeyfileKey, CHPORepoKey, - Blake2AESOCBKeyfileKey, - Blake2AESOCBRepoKey, - Blake2CHPOKeyfileKey, - Blake2CHPORepoKey, + Blake3AESOCBKeyfileKey, + Blake3AESOCBRepoKey, + Blake3CHPOKeyfileKey, + Blake3CHPORepoKey, ) diff --git a/src/borg/legacy/crypto/key.py b/src/borg/legacy/crypto/key.py index 4454015ee6..f285943bd2 100644 --- a/src/borg/legacy/crypto/key.py +++ b/src/borg/legacy/crypto/key.py @@ -1,6 +1,47 @@ +import os + from ...constants import * # NOQA -from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b -from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, AESKeyBase, FlexiKey +from ...crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, blake2b_256 +from ...crypto.key import ID_HMAC_SHA_256, AESKeyBase, FlexiKey, AuthenticatedKeyBase + + +def random_blake2b_256_key(): + # This might look a bit curious, but is the same construction used in the keyed mode of BLAKE2b. + # Why limit the key to 64 bytes and pad it with 64 nulls nonetheless? The answer is that BLAKE2b + # has a 128 byte block size, but only 64 bytes of internal state (this is also referred to as a + # "local wide pipe" design, because the compression function transforms (block, state) => state, + # and len(block) >= len(state), hence wide.) + # In other words, a key longer than 64 bytes would have simply no advantage, since the function + # has no way of propagating more than 64 bytes of entropy internally. + # It's padded to a full block so that the key is never buffered internally by blake2b_update, ie. + # it remains in a single memory location that can be tracked and could be erased securely, if we + # wanted to. + return os.urandom(64) + bytes(64) + + +class ID_BLAKE2b_256: + """ + Key mix-in class for using BLAKE2b-256 for the id key. + + The id_key length must be 32 bytes. + """ + + def id_hash(self, data): + return blake2b_256(self.id_key, data) + + def init_from_random_data(self): + super().init_from_random_data() + enc_key = os.urandom(32) + enc_hmac_key = random_blake2b_256_key() + self.crypt_key = enc_key + enc_hmac_key + self.id_key = random_blake2b_256_key() + + +class Blake2AuthenticatedKey(ID_BLAKE2b_256, AuthenticatedKeyBase): # type: ignore[misc] + TYPE = KeyType.BLAKE2AUTHENTICATED + TYPES_ACCEPTABLE = {TYPE} + NAME = "authenticated BLAKE2b" + ARG_NAME = "authenticated-blake2" class KeyfileKey(ID_HMAC_SHA_256, AESKeyBase, FlexiKey): # type: ignore[misc] @@ -39,4 +80,4 @@ class Blake2RepoKey(ID_BLAKE2b_256, AESKeyBase, FlexiKey): # type: ignore[misc] CIPHERSUITE = AES256_CTR_BLAKE2b -LEGACY_KEY_TYPES = (KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey) +LEGACY_KEY_TYPES = (KeyfileKey, RepoKey, Blake2KeyfileKey, Blake2RepoKey, Blake2AuthenticatedKey) diff --git a/src/borg/testsuite/archiver/key_cmds_test.py b/src/borg/testsuite/archiver/key_cmds_test.py index 69a1ad15a6..5a41b51e7d 100644 --- a/src/borg/testsuite/archiver/key_cmds_test.py +++ b/src/borg/testsuite/archiver/key_cmds_test.py @@ -36,14 +36,14 @@ def test_change_location_to_keyfile(archivers, request): assert "(key file" in log -def test_change_location_to_b2keyfile(archivers, request): +def test_change_location_to_b3keyfile(archivers, request): archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", "--encryption=repokey-blake2-aes-ocb") + cmd(archiver, "repo-create", "--encryption=repokey-blake3-aes-ocb") log = cmd(archiver, "repo-info") - assert "(repokey BLAKE2b" in log + assert "(repokey BLAKE3" in log cmd(archiver, "key", "change-location", "keyfile") log = cmd(archiver, "repo-info") - assert "(key file BLAKE2b" in log + assert "(key file BLAKE3" in log def test_change_location_to_repokey(archivers, request): @@ -56,14 +56,14 @@ def test_change_location_to_repokey(archivers, request): assert "(repokey" in log -def test_change_location_to_b2repokey(archivers, request): +def test_change_location_to_b3repokey(archivers, request): archiver = request.getfixturevalue(archivers) - cmd(archiver, "repo-create", "--encryption=keyfile-blake2-aes-ocb") + cmd(archiver, "repo-create", "--encryption=keyfile-blake3-aes-ocb") log = cmd(archiver, "repo-info") - assert "(key file BLAKE2b" in log + assert "(key file BLAKE3" in log cmd(archiver, "key", "change-location", "repokey") log = cmd(archiver, "repo-info") - assert "(repokey BLAKE2b" in log + assert "(repokey BLAKE3" in log def test_key_export_keyfile(archivers, request): diff --git a/src/borg/testsuite/crypto/key_test.py b/src/borg/testsuite/crypto/key_test.py index ca2884e299..693b14f97d 100644 --- a/src/borg/testsuite/crypto/key_test.py +++ b/src/borg/testsuite/crypto/key_test.py @@ -8,8 +8,9 @@ from ...crypto.key import RepoKey, KeyfileKey, Blake2RepoKey, Blake2KeyfileKey from ...crypto.key import AEADKeyBase from ...crypto.key import AESOCBRepoKey, AESOCBKeyfileKey, CHPORepoKey, CHPOKeyfileKey -from ...crypto.key import Blake2AESOCBRepoKey, Blake2AESOCBKeyfileKey, Blake2CHPORepoKey, Blake2CHPOKeyfileKey -from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256 +from ...crypto.key import Blake3AESOCBRepoKey, Blake3AESOCBKeyfileKey, Blake3CHPORepoKey, Blake3CHPOKeyfileKey +from ...crypto.key import Blake3AuthenticatedKey +from ...crypto.key import ID_HMAC_SHA_256, ID_BLAKE2b_256, ID_BLAKE3_256 from ...crypto.key import UnsupportedManifestError, UnsupportedKeyFormatError from ...crypto.key import identify_key from ...crypto.low_level import IntegrityError as IntegrityErrorBase @@ -76,21 +77,22 @@ def keys_dir(self, request, monkeypatch, tmpdir): # not encrypted PlaintextKey, AuthenticatedKey, - Blake2AuthenticatedKey, + Blake3AuthenticatedKey, # legacy crypto KeyfileKey, Blake2KeyfileKey, RepoKey, Blake2RepoKey, + Blake2AuthenticatedKey, # new crypto AESOCBKeyfileKey, AESOCBRepoKey, - Blake2AESOCBKeyfileKey, - Blake2AESOCBRepoKey, + Blake3AESOCBKeyfileKey, + Blake3AESOCBRepoKey, CHPOKeyfileKey, CHPORepoKey, - Blake2CHPOKeyfileKey, - Blake2CHPORepoKey, + Blake3CHPOKeyfileKey, + Blake3CHPORepoKey, ) ) def key(self, request, monkeypatch): @@ -254,6 +256,17 @@ def test_blake2_authenticated_encrypt(self, monkeypatch): # 0x06 is the key TYPE. assert authenticated == b"\x06" + plaintext + def test_blake3_authenticated_encrypt(self, monkeypatch): + monkeypatch.setenv("BORG_PASSPHRASE", "test") + key = Blake3AuthenticatedKey.create(self.MockRepository(), self.MockArgs()) + assert Blake3AuthenticatedKey.id_hash is ID_BLAKE3_256.id_hash + assert len(key.id_key) == 32 + plaintext = b"123456789" + id = key.id_hash(plaintext) + authenticated = key.encrypt(id, plaintext) + # 0x50 is the key TYPE. + assert authenticated == b"\x50" + plaintext + class TestTAM: @pytest.fixture From 56205cfc90c53d753f87afce9cd7059394fc957c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Fri, 15 May 2026 19:10:17 +0200 Subject: [PATCH 07/10] fix typos / grammar in transfer docs --- docs/usage/transfer.rst | 34 +++++++++++++++---------------- src/borg/archiver/transfer_cmd.py | 10 ++++----- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/usage/transfer.rst b/docs/usage/transfer.rst index 8632cf9d01..5588a57799 100644 --- a/docs/usage/transfer.rst +++ b/docs/usage/transfer.rst @@ -15,9 +15,9 @@ locations and passphrases first: :: - # borg 1.x repo -> borg 2.0 repo (hmac-sha256 -> hmac-sha256, keeping same chunk ID algorithm) + # Borg 1.x repository -> Borg 2.0 repository (hmac-sha256 -> hmac-sha256, keeping the same chunk ID algorithm) - # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing. + # 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing. # 1. Create a new "related" repository: # Here, the existing Borg 1.x repository used repokey (and AES-CTR mode), @@ -40,9 +40,9 @@ locations and passphrases first: :: - # borg 1.x repo -> borg 2.0 repo (blake2 -> blake3, changing chunk ID algorithm) + # Borg 1.x repository -> Borg 2.0 repository (blake2 -> blake3, changing the chunk ID algorithm) - # 0. Have Borg 2.0 installed on the client AND server; have a b1x repository copy for testing. + # 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing. # 1. Create a new "related" repository: # Here, the existing Borg 1.x repository used repokey-blake2 (and AES-CTR mode), @@ -66,24 +66,24 @@ locations and passphrases first: # 4. Check whether we have everything (same as step 2): $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run -Keyfile considerations when upgrading from borg 1.x +Keyfile considerations when upgrading from Borg 1.x ++++++++++++++++++++++++++++++++++++++++++++++++++++ -If you are using a ``keyfile`` encryption mode (not ``repokey``), borg 2 -may not automatically find your borg 1.x key file, because the default +If you are using a ``keyfile`` encryption mode (not ``repokey``), Borg 2 +may not automatically find your Borg 1.x key file, because the default key file directory has changed on some platforms due to the switch to the `platformdirs `_ library. -On **Linux**, there is typically no change -- both borg 1.x and borg 2 +On **Linux**, there is typically no change -- both Borg 1.x and Borg 2 use ``~/.config/borg/keys/``. -On **macOS**, borg 1.x stored key files in ``~/.config/borg/keys/``, -but borg 2 defaults to ``~/Library/Application Support/borg/keys/``. +On **macOS**, Borg 1.x stored key files in ``~/.config/borg/keys/``, +but Borg 2 defaults to ``~/Library/Application Support/borg/keys/``. -On **Windows**, borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``), -while borg 2 defaults to ``C:\Users\\AppData\Roaming\borg\keys\``. +On **Windows**, Borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``), +while Borg 2 defaults to ``C:\Users\\AppData\Roaming\borg\keys\``. -If borg 2 cannot find your key file, you have several options: +If Borg 2 cannot find your key file, you have several options: 1. **Copy the key file** from the old location to the new one. 2. **Set BORG_KEYS_DIR** to point to the old key file directory:: @@ -94,13 +94,13 @@ If borg 2 cannot find your key file, you have several options: export BORG_KEY_FILE=~/.config/borg/keys/your_key_file -4. **Set BORG_BASE_DIR** to force borg 2 to use the same base directory - as borg 1.x:: +4. **Set BORG_BASE_DIR** to force Borg 2 to use the same base directory + as Borg 1.x:: export BORG_BASE_DIR=$HOME - This makes borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``, - etc., matching borg 1.x behaviour on all platforms. + This makes Borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``, + etc., matching Borg 1.x behavior on all platforms. See :ref:`env_vars` for more details on directory environment variables. diff --git a/src/borg/archiver/transfer_cmd.py b/src/borg/archiver/transfer_cmd.py index b5d010de03..829c9b031f 100644 --- a/src/borg/archiver/transfer_cmd.py +++ b/src/borg/archiver/transfer_cmd.py @@ -183,7 +183,7 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non raise Error(f"No such upgrader: {upgrader}") if UpgraderCls is not upgrade_mod.UpgraderFrom12To20 and other_manifest.repository.version == 1: - raise Error("To transfer from a borg 1.x repo, you need to use: --upgrader=From12To20") + raise Error("To transfer from a Borg 1.x repo, you need to use: --upgrader=From12To20") upgrader = UpgraderCls(cache=cache, args=args) @@ -306,13 +306,13 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): borg --repo=DST_REPO transfer --other-repo=SRC_REPO # do it! borg --repo=DST_REPO transfer --other-repo=SRC_REPO --dry-run # check! anything left? - Data migration / upgrade from borg 1.x + Data migration / upgrade from Borg 1.x ++++++++++++++++++++++++++++++++++++++ - To migrate your borg 1.x archives into a related, new borg2 repository, usage is quite similar + To migrate your Borg 1.x archives into a related, new Borg 2 repository, usage is quite similar to the above, but you need the ``--from-borg1`` option:: - borg --repo=DST_REPO repocreate --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1 + borg --repo=DST_REPO repo-create --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1 # to continue using lz4 compression as you did in SRC_REPO: borg --repo=DST_REPO transfer --other-repo=SRC_REPO --from-borg1 \\ @@ -331,7 +331,7 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser): subparser = ArgumentParser( parents=[common_parser], description=self.do_transfer.__doc__, epilog=transfer_epilog ) - subparsers.add_subcommand("transfer", subparser, help="transfer of archives from another repository") + subparsers.add_subcommand("transfer", subparser, help="Transfer of archives from another repository") subparser.add_argument( "-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository, just check" ) From 85ef80af9e7caf63f7eed7aea343a2dd74703e10 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 May 2026 14:01:45 +0200 Subject: [PATCH 08/10] CI: more Haiku debugging --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e25e4284cd..6a5f1ba535 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -559,12 +559,15 @@ jobs: pkgman refresh pkgman install -y git pkgconfig lz4 pkgman install -y openssl3 - pkgman install -y rust_bin setuptools_rust pkgman install -y python3.10 + pkgman install -y rust_bin + pkgman install -y setuptools_rust setuptools_rust_python310 # not sure whether this is needed pkgman install -y cffi pkgman install -y lz4_devel openssl3_devel libffi_devel - cargo --version + which cargo || true + cargo --version || true + cat /var/log/syslog # there is no pkgman package for tox, so we install it into a venv python3 -m ensurepip --upgrade From f451d868d56e4a6589d6a1b4ff0e65d9192a75e3 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 May 2026 14:18:43 +0200 Subject: [PATCH 09/10] CI: more Haiku debugging (2) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6a5f1ba535..614b65921e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -561,7 +561,7 @@ jobs: pkgman install -y openssl3 pkgman install -y python3.10 pkgman install -y rust_bin - pkgman install -y setuptools_rust setuptools_rust_python310 # not sure whether this is needed + # pkgman install -y setuptools_rust setuptools_rust_python310 # not sure whether this is needed pkgman install -y cffi pkgman install -y lz4_devel openssl3_devel libffi_devel From 39bc8c80f3b754f193b83cab0a6c50baa1c0106c Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sat, 16 May 2026 22:06:42 +0200 Subject: [PATCH 10/10] CI: more Haiku debugging (3) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 614b65921e..ebc077a5f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -558,7 +558,7 @@ jobs: haiku) pkgman refresh pkgman install -y git pkgconfig lz4 - pkgman install -y openssl3 + pkgman install openssl3==3.5.6 pkgman install -y python3.10 pkgman install -y rust_bin # pkgman install -y setuptools_rust setuptools_rust_python310 # not sure whether this is needed