add blake3, blake2b is legacy

.github/workflows/ci.yml

@@ -537,7 +537,7 @@ jobs:
                 ;;
 
               omnios)
-                sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils
+                sudo pkg install gcc14 git pkg-config python-313 gnu-make gnu-coreutils rust
                 sudo ln -sf /usr/bin/python3.13 /usr/bin/python3
                 sudo ln -sf /usr/bin/python3.13-config /usr/bin/python3-config
                 sudo python3 -m ensurepip
@@ -559,21 +559,28 @@ jobs:
                 pkgman refresh
                 pkgman install -y git pkgconfig lz4
                 pkgman install -y openssl3
-                pkgman install -y rust_bin
                 pkgman install -y python3.10
+                pkgman install -y rust_bin
+                # pkgman install -y setuptools_rust setuptools_rust_python310  # not sure whether this is needed
                 pkgman install -y cffi
                 pkgman install -y lz4_devel openssl3_devel libffi_devel
 
+                which cargo || true
+                cargo --version || true
+                cat /var/log/syslog
+
                 # there is no pkgman package for tox, so we install it into a venv
                 python3 -m ensurepip --upgrade
-                python3 -m pip install --upgrade pip wheel
+                python3 -m pip install --upgrade pip setuptools wheel
                 python3 -m venv .venv
                 . .venv/bin/activate
 
                 export PKG_CONFIG_PATH="/system/develop/lib/pkgconfig:/system/lib/pkgconfig:${PKG_CONFIG_PATH:-}"
                 export BORG_LIBLZ4_PREFIX=/system/develop
                 export BORG_OPENSSL_PREFIX=/system/develop
                 pip install -r requirements.d/development.lock.txt
+                pip install -v maturin
+                pip install -v blake3
                 pip install -e .
 
                 # troubles with either tox or pytest xdist, so we run pytest manually:

docs/usage/transfer.rst

@@ -2,52 +2,88 @@
 
 Examples
 ~~~~~~~~
+
+To keep the following examples short and readable, we export the repository
+locations and passphrases first:
+
+::
+
+    export BORG_REPO=ssh://borg2@borgbackup/./tests/b20
+    export BORG_PASSPHRASE='your-borg2-repo-passphrase'
+    export BORG_OTHER_REPO=ssh://borg2@borgbackup/./tests/b1x
+    export BORG_OTHER_PASSPHRASE='your-borg1-repo-passphrase'
+
 ::
 
-    # 0. Have Borg 2.0 installed on the client AND server; have a b12 repository copy for testing.
+    # Borg 1.x repository -> Borg 2.0 repository (hmac-sha256 -> hmac-sha256, keeping the same chunk ID algorithm)
+
+    # 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing.
 
     # 1. Create a new "related" repository:
-    # Here, the existing Borg 1.2 repository used repokey-blake2 (and AES-CTR mode),
-    # thus we use repokey-blake2-aes-ocb for the new Borg 2.0 repository.
-    # Staying with the same chunk ID algorithm (BLAKE2) and with the same
-    # key material (via --other-repo <oldrepo>) will make deduplication work
+    # Here, the existing Borg 1.x repository used repokey (and AES-CTR mode),
+    # thus we use repokey-aes-ocb for the new Borg 2.0 repository.
+    # Staying with the same chunk ID algorithm (hmac-sha256) and with the same
+    # key material (via BORG_OTHER_REPO) will make deduplication work
     # between old archives (copied with borg transfer) and future ones.
     # The AEAD cipher does not matter (everything must be re-encrypted and
-    # re-authenticated anyway); you could also choose repokey-blake2-chacha20-poly1305.
-    # In case your old Borg repository did not use BLAKE2, just remove the "-blake2".
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 repo-create \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 -e repokey-blake2-aes-ocb
+    # re-authenticated anyway); you could also choose repokey-chacha20-poly1305.
+    $ borg repo-create -e repokey-aes-ocb
+
+    # 2. Check what and how much it would transfer:
+    $ borg transfer --from-borg1 --dry-run
+
+    # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
+    $ borg transfer --from-borg1
+
+    # 4. Check whether we have everything (same as step 2):
+    $ borg transfer --from-borg1 --dry-run
+
+::
+
+    # Borg 1.x repository -> Borg 2.0 repository (blake2 -> blake3, changing the chunk ID algorithm)
+
+    # 0. Have Borg 2.0 installed on the client AND server; have a Borg 1.x repository copy for testing.
+
+    # 1. Create a new "related" repository:
+    # Here, the existing Borg 1.x repository used repokey-blake2 (and AES-CTR mode),
+    # thus we use repokey-blake3-aes-ocb for the new Borg 2.0 repository.
+    # We need to change from blake2 to blake3, because blake2 is not supported
+    # for borg2 repos (blake3 is much faster). Because we change how chunk IDs are
+    # computed, we need to re-chunk everything while doing the transfer.
+    # The chunker parameters you provide here should be the same as you will
+    # use for all future Borg 2.0 archives.
+    # The AEAD cipher does not matter (everything must be re-encrypted and
+    # re-authenticated anyway); you could also choose repokey-blake3-chacha20-poly1305.
+    $ borg repo-create -e repokey-blake3-aes-ocb
+    $ export CHUNKER_PARAMS="buzhash64,19,23,21,4095"
 
     # 2. Check what and how much it would transfer:
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
 
     # 3. Transfer (copy) archives from the old repository into the new repository (takes time and space!):
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS
 
     # 4. Check whether we have everything (same as step 2):
-    $ borg --repo       ssh://borg2@borgbackup/./tests/b20 transfer --upgrader=From12To20 \
-           --other-repo ssh://borg2@borgbackup/./tests/b12 --dry-run
+    $ borg transfer --from-borg1 --chunker-params=$CHUNKER_PARAMS --dry-run
 
-Keyfile considerations when upgrading from borg 1.x
+Keyfile considerations when upgrading from Borg 1.x
 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 
-If you are using a ``keyfile`` encryption mode (not ``repokey``), borg 2
-may not automatically find your borg 1.x key file, because the default
+If you are using a ``keyfile`` encryption mode (not ``repokey``), Borg 2
+may not automatically find your Borg 1.x key file, because the default
 key file directory has changed on some platforms due to the switch to
 the `platformdirs <https://pypi.org/project/platformdirs/>`_ library.
 
-On **Linux**, there is typically no change -- both borg 1.x and borg 2
+On **Linux**, there is typically no change -- both Borg 1.x and Borg 2
 use ``~/.config/borg/keys/``.
 
-On **macOS**, borg 1.x stored key files in ``~/.config/borg/keys/``,
-but borg 2 defaults to ``~/Library/Application Support/borg/keys/``.
+On **macOS**, Borg 1.x stored key files in ``~/.config/borg/keys/``,
+but Borg 2 defaults to ``~/Library/Application Support/borg/keys/``.
 
-On **Windows**, borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``),
-while borg 2 defaults to ``C:\Users\<user>\AppData\Roaming\borg\keys\``.
+On **Windows**, Borg 1.x used XDG-style paths (e.g. ``~/.config/borg/keys/``),
+while Borg 2 defaults to ``C:\Users\<user>\AppData\Roaming\borg\keys\``.
 
-If borg 2 cannot find your key file, you have several options:
+If Borg 2 cannot find your key file, you have several options:
 
 1. **Copy the key file** from the old location to the new one.
 2. **Set BORG_KEYS_DIR** to point to the old key file directory::
@@ -58,13 +94,13 @@ If borg 2 cannot find your key file, you have several options:
 
        export BORG_KEY_FILE=~/.config/borg/keys/your_key_file
 
-4. **Set BORG_BASE_DIR** to force borg 2 to use the same base directory
-   as borg 1.x::
+4. **Set BORG_BASE_DIR** to force Borg 2 to use the same base directory
+   as Borg 1.x::
 
        export BORG_BASE_DIR=$HOME
 
-   This makes borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``,
-   etc., matching borg 1.x behaviour on all platforms.
+   This makes Borg 2 use ``$HOME/.config/borg``, ``$HOME/.cache/borg``,
+   etc., matching Borg 1.x behavior on all platforms.
 
 See :ref:`env_vars` for more details on directory environment variables.
 

pyproject.toml

@@ -42,6 +42,7 @@ dependencies = [
   "xxhash>=2.0.0",
   "jsonargparse>=4.47.0",
   "PyYAML>=6.0.2",  # we need to register our types with yaml, jsonargparse uses yaml for config files
+  "blake3>=1.0.0",
 ]
 
 [project.optional-dependencies]

scripts/msys2-install-deps

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko}
+pacman -S --needed --noconfirm git mingw-w64-ucrt-x86_64-{toolchain,pkgconf,lz4,xxhash,openssl,rclone,python-msgpack,python-argon2_cffi,python-platformdirs,python,cython,python-setuptools,python-wheel,python-build,python-pkgconfig,python-packaging,python-pip,python-paramiko,rust,python-maturin}
 
 if [ "$1" = "development" ]; then
 	pacman -S --needed --noconfirm mingw-w64-ucrt-x86_64-python-{pytest,pytest-benchmark,pytest-cov,pytest-xdist}

src/borg/archiver/benchmark_cmd.py

@@ -232,16 +232,20 @@ def chunkit(ch):
                 print(f"{spec:<24} {format_file_size(size):<10} {dt:.3f}s")
 
         from ..crypto.low_level import hmac_sha256, blake2b_256
+        import blake3
 
         if not args.json:
             print("Cryptographic hashes / MACs ====================================")
         else:
             result["hashes"] = []
         size = 1000000000
-        for spec, func in [
+        hashes_tests = [
             ("hmac-sha256", lambda: hmac_sha256(key_256, random_10M)),
             ("blake2b-256", lambda: blake2b_256(key_256, random_10M)),
-        ]:
+            ("blake3", lambda: blake3.blake3(random_10M, key=key_256).digest()),
+            ("blake3-mt", lambda: blake3.blake3(random_10M, key=key_256, max_threads=blake3.blake3.AUTO).digest()),
+        ]
+        for spec, func in hashes_tests:
             dt = timeit(func, number=number_default)
             if args.json:
                 result["hashes"].append({"algo": spec, "size": size, "time": dt})

src/borg/archiver/key_cmds.py

@@ -1,8 +1,8 @@
 import os
 
 from ..constants import *  # NOQA
-from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake2AESOCBRepoKey, Blake2CHPORepoKey
-from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake2AESOCBKeyfileKey, Blake2CHPOKeyfileKey
+from ..crypto.key import AESOCBRepoKey, CHPORepoKey, Blake3AESOCBRepoKey, Blake3CHPORepoKey
+from ..crypto.key import AESOCBKeyfileKey, CHPOKeyfileKey, Blake3AESOCBKeyfileKey, Blake3CHPOKeyfileKey
 from ..crypto.keymanager import KeyManager
 from ..helpers import PathSpec, CommandError
 from ..helpers.argparsing import ArgumentParser
@@ -40,10 +40,10 @@ def do_key_change_location(self, args, repository, manifest, cache):
                 key_new = AESOCBKeyfileKey(repository)
             elif isinstance(key, CHPORepoKey):
                 key_new = CHPOKeyfileKey(repository)
-            elif isinstance(key, Blake2AESOCBRepoKey):
-                key_new = Blake2AESOCBKeyfileKey(repository)
-            elif isinstance(key, Blake2CHPORepoKey):
-                key_new = Blake2CHPOKeyfileKey(repository)
+            elif isinstance(key, Blake3AESOCBRepoKey):
+                key_new = Blake3AESOCBKeyfileKey(repository)
+            elif isinstance(key, Blake3CHPORepoKey):
+                key_new = Blake3CHPOKeyfileKey(repository)
             else:
                 print("Change not needed or not supported.")
                 return
@@ -52,10 +52,10 @@ def do_key_change_location(self, args, repository, manifest, cache):
                 key_new = AESOCBRepoKey(repository)
             elif isinstance(key, CHPOKeyfileKey):
                 key_new = CHPORepoKey(repository)
-            elif isinstance(key, Blake2AESOCBKeyfileKey):
-                key_new = Blake2AESOCBRepoKey(repository)
-            elif isinstance(key, Blake2CHPOKeyfileKey):
-                key_new = Blake2CHPORepoKey(repository)
+            elif isinstance(key, Blake3AESOCBKeyfileKey):
+                key_new = Blake3AESOCBRepoKey(repository)
+            elif isinstance(key, Blake3CHPOKeyfileKey):
+                key_new = Blake3CHPORepoKey(repository)
             else:
                 print("Change not needed or not supported.")
                 return

src/borg/archiver/transfer_cmd.py

@@ -134,12 +134,11 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non
         """archives transfer from other repository, optionally upgrade data format"""
         key = manifest.key
         other_key = other_manifest.key
-        if not uses_same_id_hash(other_key, key):
-            raise Error(
-                "You must keep the same ID hash ([HMAC-]SHA256 or BLAKE2b) or deduplication will break. "
-                "Use a related repository!"
-            )
-        if not uses_same_chunker_secret(other_key, key):
+        using_same_id_hash = uses_same_id_hash(other_key, key)
+        rechunking = args.chunker_params is not None
+        if not using_same_id_hash and not rechunking:
+            raise Error("You must either keep the same ID hash or use --chunker-params.")
+        if not rechunking and not uses_same_chunker_secret(other_key, key):
             raise Error(
                 "You must use the same chunker secret or deduplication will break. " "Use a related repository!"
             )
@@ -184,7 +183,7 @@ def do_transfer(self, args, *, repository, manifest, cache, other_repository=Non
             raise Error(f"No such upgrader: {upgrader}")
 
         if UpgraderCls is not upgrade_mod.UpgraderFrom12To20 and other_manifest.repository.version == 1:
-            raise Error("To transfer from a borg 1.x repo, you need to use: --upgrader=From12To20")
+            raise Error("To transfer from a Borg 1.x repo, you need to use: --upgrader=From12To20")
 
         upgrader = UpgraderCls(cache=cache, args=args)
 
@@ -307,13 +306,13 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser):
             borg --repo=DST_REPO transfer --other-repo=SRC_REPO            # do it!
             borg --repo=DST_REPO transfer --other-repo=SRC_REPO --dry-run  # check! anything left?
 
-        Data migration / upgrade from borg 1.x
+        Data migration / upgrade from Borg 1.x
         ++++++++++++++++++++++++++++++++++++++
 
-        To migrate your borg 1.x archives into a related, new borg2 repository, usage is quite similar
+        To migrate your Borg 1.x archives into a related, new Borg 2 repository, usage is quite similar
         to the above, but you need the ``--from-borg1`` option::
 
-            borg --repo=DST_REPO repocreate --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1
+            borg --repo=DST_REPO repo-create --encryption=DST_ENC --other-repo=SRC_REPO --from-borg1
 
             # to continue using lz4 compression as you did in SRC_REPO:
             borg --repo=DST_REPO transfer --other-repo=SRC_REPO --from-borg1 \\
@@ -332,7 +331,7 @@ def build_parser_transfer(self, subparsers, common_parser, mid_common_parser):
         subparser = ArgumentParser(
             parents=[common_parser], description=self.do_transfer.__doc__, epilog=transfer_epilog
         )
-        subparsers.add_subcommand("transfer", subparser, help="transfer of archives from another repository")
+        subparsers.add_subcommand("transfer", subparser, help="Transfer of archives from another repository")
         subparser.add_argument(
             "-n", "--dry-run", dest="dry_run", action="store_true", help="do not change repository, just check"
         )

src/borg/constants.py

@@ -191,10 +191,11 @@ class KeyType:
     AESOCBREPO = 0x11
     CHPOKEYFILE = 0x20
     CHPOREPO = 0x21
-    BLAKE2AESOCBKEYFILE = 0x30
-    BLAKE2AESOCBREPO = 0x31
-    BLAKE2CHPOKEYFILE = 0x40
-    BLAKE2CHPOREPO = 0x41
+    BLAKE3AESOCBKEYFILE = 0x30
+    BLAKE3AESOCBREPO = 0x31
+    BLAKE3CHPOKEYFILE = 0x40
+    BLAKE3CHPOREPO = 0x41
+    BLAKE3AUTHENTICATED = 0x50
 
 
 CACHE_TAG_NAME = "CACHEDIR.TAG"