diff --git a/docs/faq.rst b/docs/faq.rst index 5d8d7fc984..291cef6d35 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -99,6 +99,18 @@ Also, you must not run borg against multiple instances of the same repo See also: :ref:`faq_corrupt_repo` +Prepare for borg2 "Related repositories" and borg transfer +---------------------------------------------------------- + +A related repository is a repository that shares the same deduplication +secrets (``id_key`` and ``chunk_seed``) as another repository, but uses +its own independent encryption keys. + +This will allow archives to be transferred between related repositories (e.g. +using ``borg transfer`` in Borg 2.0) without breaking deduplication. + +For more information and detailed instructions, see :ref:`borg_key_export-related-secrets`. + "this is either an attack or unsafe" warning -------------------------------------------- diff --git a/docs/man/borg-init.1 b/docs/man/borg-init.1 index 5532923989..e6f1d87b1a 100644 --- a/docs/man/borg-init.1 +++ b/docs/man/borg-init.1 @@ -29,7 +29,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-init" "1" "2026-03-18" "" "borg backup tool" +.TH "borg-init" "1" "2026-05-15" "" "borg backup tool" .SH Name borg-init \- Initialize an empty repository .SH SYNOPSIS @@ -276,11 +276,11 @@ BLAKE2b\-256 hash from the other BLAKE2b modes. This mode is only compatible with Borg 1.1 and later. .sp \fBnone\fP mode uses no encryption and no authentication. It uses SHA256 -as chunk ID hash. This mode is not recommended. You should instead -consider using an authenticated or authenticated/encrypted mode. This -mode has possible denial\-of\-service issues when running \fBborg create\fP -on contents controlled by an attacker. See above for alternatives. -This mode is compatible with all Borg versions. +as chunk ID hash. This mode is not recommended +as it is vulnerable to DoS attacks by an attacker (for example, +crafting content that causes hash index collisions). Do not use it if +untrusted clients use the repository. See \fIinternals_hashindex\fP for +details. This mode is compatible with all Borg versions. .SH OPTIONS .sp See \fIborg\-common(1)\fP for common options of Borg commands. @@ -304,6 +304,9 @@ Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota. .TP .B \-\-make\-parent\-dirs create the parent directories of the repository directory, if they are missing. +.TP +.BI \-\-import\-related\-secrets \ PATH +import related secrets from PATH .UNINDENT .SH EXAMPLES .INDENT 0.0 diff --git a/docs/man/borg-key-export-related-secrets.1 b/docs/man/borg-key-export-related-secrets.1 new file mode 100644 index 0000000000..2002d7b1ce --- /dev/null +++ b/docs/man/borg-key-export-related-secrets.1 @@ -0,0 +1,109 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-key-export-related-secrets" "1" "2026-05-15" "" "borg backup tool" +.SH Name +borg-key-export-related-secrets \- Export secrets for creating related repositories +.SH SYNOPSIS +.sp +borg [common options] key export\-related\-secrets [options] [REPOSITORY] [PATH] +.SH DESCRIPTION +.sp +This command exports the deduplication secrets (\fBid_key\fP and \fBchunk_seed\fP) +of a repository. These secrets can be used to initialize a \fBrelated repository\fP\&. +.sp +Related repositories share the same deduplication metadata but have their own +independent encryption keys. This is useful for: +.INDENT 0.0 +.IP 1. 3 +Creating independent backup targets that still benefit from being +\(dqcompatible\(dq for future archive transfers. +.IP 2. 3 +Preparing for a migration to Borg 2.0, where archives can be transferred +between related repositories using \fBborg transfer\fP\&. +.UNINDENT +.sp +The exported secrets are stored in a JSON file. This file contains sensitive +information and should be deleted immediately after usage. +.sp +Examples: +.INDENT 0.0 +.INDENT 3.5 +.sp +.EX +# Export secrets from an existing repository +$ borg key export\-related\-secrets /path/to/repo1 secrets.json + +# Initialize a new related repository using these secrets +$ borg init \-\-import\-related\-secrets=secrets.json \-\-encryption=repokey /path/to/repo2 +$ rm secrets.json +.EE +.UNINDENT +.UNINDENT +.sp +\fBImportant:\fP +.INDENT 0.0 +.INDENT 3.5 +When initializing a related repository using \fBborg init \-\-import\-related\-secrets\fP, +the new repository must use the same ID hash algorithm (either both HMAC\-SHA256 +or both BLAKE2) as the original repository. +.INDENT 0.0 +.IP \(bu 2 +HMAC\-SHA256: \fBrepokey\fP, \fBkeyfile\fP, \fBauthenticated\fP +.IP \(bu 2 +BLAKE2: \fBrepokey\-blake2\fP, \fBkeyfile\-blake2\fP, \fBauthenticated\-blake2\fP +.UNINDENT +.UNINDENT +.UNINDENT +.sp +\fBWarning:\fP +.INDENT 0.0 +.INDENT 3.5 +Please note that future Borg 2.0 versions might remove support for BLAKE2 +in new repositories (see #8867). +.UNINDENT +.UNINDENT +.SH OPTIONS +.sp +See \fIborg\-common(1)\fP for common options of Borg commands. +.SS arguments +.sp +REPOSITORY +.INDENT 0.0 +.TP +.B PATH +where to store the secrets +.UNINDENT +.SH SEE ALSO +.sp +\fIborg\-common(1)\fP +.SH Author +The Borg Collective +.\" End of generated man page. diff --git a/docs/usage/init.rst.inc b/docs/usage/init.rst.inc index 54edc5c413..13275c780c 100644 --- a/docs/usage/init.rst.inc +++ b/docs/usage/init.rst.inc @@ -27,6 +27,8 @@ borg init +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | | ``--make-parent-dirs`` | create the parent directories of the repository directory, if they are missing. | +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | | ``--import-related-secrets PATH`` | import related secrets from PATH | + +-------------------------------------------------------+------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | .. class:: borg-common-opt-ref | | | | :ref:`common_options` | @@ -51,6 +53,7 @@ borg init --append-only create an append-only mode repository. Note that this only affects the low level structure of the repository, and running `delete` or `prune` will still be allowed. See :ref:`append_only_mode` in Additional Notes for more details. --storage-quota QUOTA Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota. --make-parent-dirs create the parent directories of the repository directory, if they are missing. + --import-related-secrets PATH import related secrets from PATH :ref:`common_options` @@ -265,8 +268,8 @@ BLAKE2b-256 hash from the other BLAKE2b modes. This mode is only compatible with Borg 1.1 and later. ``none`` mode uses no encryption and no authentication. It uses SHA256 -as chunk ID hash. This mode is not recommended. You should instead -consider using an authenticated or authenticated/encrypted mode. This -mode has possible denial-of-service issues when running ``borg create`` -on contents controlled by an attacker. See above for alternatives. -This mode is compatible with all Borg versions. \ No newline at end of file +as chunk ID hash. This mode is not recommended +as it is vulnerable to DoS attacks by an attacker (for example, +crafting content that causes hash index collisions). Do not use it if +untrusted clients use the repository. See :ref:`internals_hashindex` for +details. This mode is compatible with all Borg versions. \ No newline at end of file diff --git a/docs/usage/key.rst b/docs/usage/key.rst index 10ccdfe5b7..333d1a7955 100644 --- a/docs/usage/key.rst +++ b/docs/usage/key.rst @@ -44,3 +44,7 @@ Fully automated using environment variables: .. include:: key_export.rst.inc .. include:: key_import.rst.inc + +This command can be used to create a related repository: + +.. include:: key_export-related-secrets.rst.inc diff --git a/docs/usage/key_export-related-secrets.rst b/docs/usage/key_export-related-secrets.rst new file mode 100644 index 0000000000..54f01b4121 --- /dev/null +++ b/docs/usage/key_export-related-secrets.rst @@ -0,0 +1 @@ +.. include:: key_export-related-secrets.rst.inc diff --git a/docs/usage/key_export-related-secrets.rst.inc b/docs/usage/key_export-related-secrets.rst.inc new file mode 100644 index 0000000000..1fbd8c5896 --- /dev/null +++ b/docs/usage/key_export-related-secrets.rst.inc @@ -0,0 +1,82 @@ +.. IMPORTANT: this file is auto-generated from borg's built-in help, do not edit! + +.. _borg_key_export-related-secrets: + +borg key export-related-secrets +------------------------------- +.. code-block:: none + + borg [common options] key export-related-secrets [options] [REPOSITORY] [PATH] + +.. only:: html + + .. class:: borg-options-table + + +-------------------------------------------------------+----------------+----------------------------+ + | **positional arguments** | + +-------------------------------------------------------+----------------+----------------------------+ + | | ``REPOSITORY`` | | + +-------------------------------------------------------+----------------+----------------------------+ + | | ``PATH`` | where to store the secrets | + +-------------------------------------------------------+----------------+----------------------------+ + | .. class:: borg-common-opt-ref | + | | + | :ref:`common_options` | + +-------------------------------------------------------+----------------+----------------------------+ + + .. raw:: html + + + +.. only:: latex + + REPOSITORY + + PATH + where to store the secrets + + + :ref:`common_options` + | + +Description +~~~~~~~~~~~ + +This command exports the deduplication secrets (``id_key`` and ``chunk_seed``) +of a repository. These secrets can be used to initialize a **related repository**. + +Related repositories share the same deduplication metadata but have their own +independent encryption keys. This is useful for: + +1. Creating independent backup targets that still benefit from being + "compatible" for future archive transfers. +2. Preparing for a migration to Borg 2.0, where archives can be transferred + between related repositories using ``borg transfer``. + +The exported secrets are stored in a JSON file. This file contains sensitive +information and should be deleted immediately after usage. + +Examples:: + + # Export secrets from an existing repository + $ borg key export-related-secrets /path/to/repo1 secrets.json + + # Initialize a new related repository using these secrets + $ borg init --import-related-secrets=secrets.json --encryption=repokey /path/to/repo2 + $ rm secrets.json + +.. IMPORTANT:: + When initializing a related repository using ``borg init --import-related-secrets``, + the new repository must use the same ID hash algorithm (either both HMAC-SHA256 + or both BLAKE2) as the original repository. + + - HMAC-SHA256: ``repokey``, ``keyfile``, ``authenticated`` + - BLAKE2: ``repokey-blake2``, ``keyfile-blake2``, ``authenticated-blake2`` + +.. WARNING:: + Please note that future Borg 2.0 versions might remove support for BLAKE2 + in new repositories (see :issue:`8867`). \ No newline at end of file diff --git a/src/borg/archiver.py b/src/borg/archiver.py index 44602bb760..d6fd9d34cd 100644 --- a/src/borg/archiver.py +++ b/src/borg/archiver.py @@ -333,8 +333,21 @@ def do_init(self, args, repository): """Initialize an empty repository""" path = args.location.canonical_path() logger.info('Initializing repository at "%s"' % path) + related_secrets = None + if args.import_related_secrets: + with dash_open(args.import_related_secrets, 'r') as fd: + try: + related_secrets = json.load(fd) + except ValueError: + raise CommandError(f"Invalid JSON in related secrets file: {args.import_related_secrets}") + if related_secrets.get('version') != 1: + raise CommandError(f"Unsupported related secrets version: {related_secrets.get('version')}") + try: + related_secrets['id_key'] = hex_to_bin(related_secrets['id_key']) + except (KeyError, ValueError): + raise CommandError(f"Invalid id_key in related secrets file: {args.import_related_secrets}") try: - key = key_creator(repository, args) + key = key_creator(repository, args, related_secrets=related_secrets) except (EOFError, KeyboardInterrupt): repository.destroy() raise CancelledByUser() @@ -413,6 +426,19 @@ def do_change_passphrase(self, args, repository, manifest, key): # print key location to make backing it up easier logger.info('Key location: %s', key.find_key()) + @with_repository(manifest=True, compatibility=(Manifest.Operation.READ,)) + def do_key_export_related_secrets(self, args, repository, manifest, key): + """Export secrets for creating related repositories""" + secrets = { + 'version': 1, + 'id_key': bin_to_hex(key.id_key), + 'chunk_seed': key.chunk_seed, + 'key_name': key.NAME, + } + with dash_open(args.path, 'w') as fd: + json.dump(secrets, fd, indent=4) + fd.write('\n') + @with_repository(lock=False, exclusive=False, manifest=False, cache=False) def do_key_export(self, args, repository): """Export the repository key for backup""" @@ -4784,6 +4810,8 @@ def diff_sort_spec_validator(s): help='Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.') subparser.add_argument('--make-parent-dirs', dest='make_parent_dirs', action='store_true', help='create the parent directories of the repository directory, if they are missing.') + subparser.add_argument('--import-related-secrets', metavar='PATH', dest='import_related_secrets', + type=PathSpec, help='import related secrets from PATH') # borg key subparser = subparsers.add_parser('key', parents=[mid_common_parser], add_help=False, @@ -4851,6 +4879,54 @@ def diff_sort_spec_validator(s): subparser.add_argument('--qr-html', dest='qr', action='store_true', help='Create an html file suitable for printing and later type-in or qr scan') + export_related_secrets_epilog = process_epilog(""" + This command exports the deduplication secrets (``id_key`` and ``chunk_seed``) + of a repository. These secrets can be used to initialize a **related repository**. + + Related repositories share the same deduplication metadata but have their own + independent encryption keys. This is useful for: + + 1. Creating independent backup targets that still benefit from being + "compatible" for future archive transfers. + 2. Preparing for a migration to Borg 2.0, where archives can be transferred + between related repositories using ``borg transfer``. + + The exported secrets are stored in a JSON file. This file contains sensitive + information and should be deleted immediately after usage. + + Examples:: + + # Export secrets from an existing repository + $ borg key export-related-secrets /path/to/repo1 secrets.json + + # Initialize a new related repository using these secrets + $ borg init --import-related-secrets=secrets.json --encryption=repokey /path/to/repo2 + $ rm secrets.json + + .. IMPORTANT:: + When initializing a related repository using ``borg init --import-related-secrets``, + the new repository must use the same ID hash algorithm (either both HMAC-SHA256 + or both BLAKE2) as the original repository. + + - HMAC-SHA256: ``repokey``, ``keyfile``, ``authenticated`` + - BLAKE2: ``repokey-blake2``, ``keyfile-blake2``, ``authenticated-blake2`` + + .. WARNING:: + Please note that future Borg 2.0 versions might remove support for BLAKE2 + in new repositories (see :issue:`8867`). + """) + + subparser = key_parsers.add_parser('export-related-secrets', parents=[common_parser], add_help=False, + description=self.do_key_export_related_secrets.__doc__, + epilog=export_related_secrets_epilog, + formatter_class=argparse.RawDescriptionHelpFormatter, + help='export related secrets for related repositories') + subparser.set_defaults(func=self.do_key_export_related_secrets) + subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='', + type=location_validator(archive=False)) + subparser.add_argument('path', metavar='PATH', nargs='?', type=PathSpec, + help='where to store the secrets') + key_import_epilog = process_epilog(""" This command restores a key previously backed up with the export command. diff --git a/src/borg/crypto/key.py b/src/borg/crypto/key.py index 7e655e5325..b5b55b8532 100644 --- a/src/borg/crypto/key.py +++ b/src/borg/crypto/key.py @@ -134,15 +134,24 @@ class KeyBlobStorage: REPO = 'repository' -def key_creator(repository, args): +def key_creator(repository, args, related_secrets=None): for key in AVAILABLE_KEY_TYPES: if key.ARG_NAME == args.encryption: assert key.ARG_NAME is not None - return key.create(repository, args) + return key.create(repository, args, related_secrets=related_secrets) else: raise ValueError('Invalid encryption mode "%s"' % args.encryption) +def uses_same_id_hash(other_key_name, key): + """is the id hash the same?""" + # avoid breaking the deduplication by changing the id hash + hmac_sha256_names = ('repokey', 'key file', 'authenticated') + blake2_names = ('repokey BLAKE2b', 'key file BLAKE2b', 'authenticated BLAKE2b') + return (other_key_name in hmac_sha256_names and key.NAME in hmac_sha256_names or + other_key_name in blake2_names and key.NAME in blake2_names) + + def key_argument_names(): return [key.ARG_NAME for key in AVAILABLE_KEY_TYPES if key.ARG_NAME] @@ -355,7 +364,7 @@ def __init__(self, repository): self.tam_required = False @classmethod - def create(cls, repository, args): + def create(cls, repository, args, related_secrets=None): logger.info('Encryption NOT enabled.\nUse the "--encryption=repokey|keyfile" to enable encryption.') return cls(repository) @@ -622,11 +631,13 @@ class PassphraseKey(ID_HMAC_SHA_256, AESKeyBase): iterations = 100000 # must not be changed ever! @classmethod - def create(cls, repository, args): + def create(cls, repository, args, related_secrets=None): key = cls(repository) logger.warning('WARNING: "passphrase" mode is unsupported since borg 1.0.') passphrase = Passphrase.new(allow_empty=False) key.init(repository, passphrase) + if related_secrets: + raise Error('Importing related secrets is not supported for "passphrase" mode.') return key @classmethod @@ -762,11 +773,16 @@ def change_passphrase(self, passphrase=None): self.save(self.target, passphrase) @classmethod - def create(cls, repository, args): + def create(cls, repository, args, related_secrets=None): passphrase = Passphrase.new(allow_empty=True) key = cls(repository) key.repository_id = repository.id key.init_from_random_data() + if related_secrets: + if not uses_same_id_hash(related_secrets['key_name'], key): + raise Error('You must keep the same ID hash (HMAC-SHA256 or BLAKE2b) or deduplication will break.') + key.id_key = related_secrets['id_key'] + key.chunk_seed = related_secrets['chunk_seed'] key.init_ciphers() target = key.get_new_target(args) key.save(target, passphrase, create=True) diff --git a/src/borg/testsuite/archiver.py b/src/borg/testsuite/archiver.py index 4f4f390283..5358768427 100644 --- a/src/borg/testsuite/archiver.py +++ b/src/borg/testsuite/archiver.py @@ -3209,7 +3209,7 @@ def test_debug_put_get_delete_obj(self): assert "is invalid" in output def test_init_interrupt(self): - def raise_eof(*args): + def raise_eof(*args, **kwargs): raise EOFError with patch.object(KeyfileKeyBase, 'create', raise_eof): @@ -4217,6 +4217,76 @@ def original_size(archive_name): self.cmd('recreate', '--chunker-params=10,12,11,63', self.repository_location + '::archive') assert original_size('archive') == sum(sizes) + def test_related_repos_deduplication(self): + CHUNKER_PARAMS = 'buzhash,10,18,14,4095' # ~16kiB chunks + # 1. Create repo1 + self.cmd('init', '--encryption=repokey', self.repository_location) + self.create_regular_file('file1', contents=os.urandom(128 * 1024)) + self.cmd('create', f'--chunker-params={CHUNKER_PARAMS}', self.repository_location + '::archive1', 'input') + + # 2. Export related secrets + secrets_path = os.path.join(self.tmpdir, 'secrets.json') + self.cmd('key', 'export-related-secrets', self.repository_location, secrets_path) + + with open(secrets_path, 'r') as f: + secrets = json.load(f) + assert secrets['version'] == 1 + assert 'id_key' in secrets + assert 'chunk_seed' in secrets + assert 'key_name' in secrets + + # 3. Create repo2 using imported secrets + repo2_path = os.path.join(self.tmpdir, 'repo2') + repo2_location = self.prefix + repo2_path + self.cmd('init', '--encryption=repokey', '--import-related-secrets', secrets_path, repo2_location) + + # 4. Create backup in repo2 with same data + self.cmd('create', f'--chunker-params={CHUNKER_PARAMS}', repo2_location + '::archive2', 'input') + + # 5. Verify chunk IDs are identical + def get_chunk_ids(repo_path, archive_name): + with Repository(repo_path) as repository: + manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + archive = Archive(repository, key, manifest, archive_name) + ids = [] + for item in archive.iter_items(): + chunks = getattr(item, 'chunks', None) + if chunks: + ids.extend(c.id for c in chunks) + return ids + + ids1 = get_chunk_ids(self.repository_path, 'archive1') + ids2 = get_chunk_ids(repo2_path, 'archive2') + + assert ids1 == ids2 + assert len(ids1) > 3 + + # 6. Verify encryption keys are different, but id_key and chunk_seed are same + def get_keys(repo_path): + with Repository(repo_path) as repository: + manifest, key = Manifest.load(repository, Manifest.NO_OPERATION_CHECK) + return key.enc_key, key.enc_hmac_key, key.id_key, key.chunk_seed + + enc_key1, hmac_key1, id_key1, chunk_seed1 = get_keys(self.repository_path) + enc_key2, hmac_key2, id_key2, chunk_seed2 = get_keys(repo2_path) + + assert enc_key1 != enc_key2 + assert hmac_key1 != hmac_key2 + assert id_key1 == id_key2 + assert chunk_seed1 == chunk_seed2 + + def test_related_repos_incompatible_key_name(self): + # Create repo1 with default (HMAC-SHA256) + self.cmd('init', '--encryption=repokey', self.repository_location) + secrets_path = os.path.join(self.tmpdir, 'secrets.json') + self.cmd('key', 'export-related-secrets', self.repository_location, secrets_path) + + # Try to create repo2 with BLAKE2b (incompatible) + repo2_path = os.path.join(self.tmpdir, 'repo2') + repo2_location = self.prefix + repo2_path + # This should fail + out = self.cmd('init', '--encryption=repokey-blake2', '--import-related-secrets', secrets_path, repo2_location, exit_code=2, fork=True) + assert 'deduplication will break' in out @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available') class ArchiverTestCaseBinary(ArchiverTestCase):