From d1fa670de07ec9d08fc1333b3addf746c7d304e2 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:14 +0200 Subject: [PATCH 01/36] object-file: move `mkdir_in_gitdir()` into "path.c" The `mkdir_in_gitdir()` function is similar to `safe_create_dir()`, but the former is hosted in "object-file.c" whereas the latter is hosted in "path.c". The latter code unit makes way more sense though as the logic has nothing to do with object files in particular. Move the file into "path.c". While at it, we: - Rename the function to `safe_create_dir_in_gitdir()` so that the function names are similar to one another. - Remove the dependency on `the_repository` by making the callers pass the repository instead. Adjust callers accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/rebase.c | 3 ++- object-file.c | 33 ++------------------------------- object-file.h | 2 -- path.c | 29 +++++++++++++++++++++++++++++ path.h | 11 +++++++++++ rerere.c | 5 +++-- 6 files changed, 47 insertions(+), 36 deletions(-) diff --git a/builtin/rebase.c b/builtin/rebase.c index 1cd802dd92a53d..cd9371536f87f3 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -267,7 +267,8 @@ static int init_basic_state(struct replay_opts *opts, const char *head_name, { FILE *interactive; - if (!is_directory(merge_dir()) && mkdir_in_gitdir(merge_dir())) + if (!is_directory(merge_dir()) && + safe_create_dir_in_gitdir(the_repository, merge_dir())) return error_errno(_("could not create temporary %s"), merge_dir()); refs_delete_reflog(get_main_ref_store(the_repository), "REBASE_HEAD"); diff --git a/object-file.c b/object-file.c index 772c311f188a22..00451876bd05cf 100644 --- a/object-file.c +++ b/object-file.c @@ -90,36 +90,6 @@ static int get_conv_flags(unsigned flags) return 0; } - -int mkdir_in_gitdir(const char *path) -{ - if (mkdir(path, 0777)) { - int saved_errno = errno; - struct stat st; - struct strbuf sb = STRBUF_INIT; - - if (errno != EEXIST) - return -1; - /* - * Are we looking at a path in a symlinked worktree - * whose original repository does not yet have it? - * e.g. .git/rr-cache pointing at its original - * repository in which the user hasn't performed any - * conflict resolution yet? - */ - if (lstat(path, &st) || !S_ISLNK(st.st_mode) || - strbuf_readlink(&sb, path, st.st_size) || - !is_absolute_path(sb.buf) || - mkdir(sb.buf, 0777)) { - strbuf_release(&sb); - errno = saved_errno; - return -1; - } - strbuf_release(&sb); - } - return adjust_shared_perm(the_repository, path); -} - static enum scld_error safe_create_leading_directories_1(char *path, int share) { char *next_component = path + offset_1st_component(path); @@ -2196,7 +2166,8 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, struct strbuf dir = STRBUF_INIT; strbuf_add(&dir, filename.buf, dirlen); - if (mkdir_in_gitdir(dir.buf) && errno != EEXIST) { + if (safe_create_dir_in_gitdir(the_repository, dir.buf) && + errno != EEXIST) { err = error_errno(_("unable to create directory %s"), dir.buf); strbuf_release(&dir); goto cleanup; diff --git a/object-file.h b/object-file.h index 81b30d269c82e4..4649a3f37d4da9 100644 --- a/object-file.h +++ b/object-file.h @@ -54,8 +54,6 @@ enum scld_error safe_create_leading_directories(char *path); enum scld_error safe_create_leading_directories_const(const char *path); enum scld_error safe_create_leading_directories_no_share(char *path); -int mkdir_in_gitdir(const char *path); - int git_open_cloexec(const char *name, int flags); #define git_open(name) git_open_cloexec(name, O_RDONLY) diff --git a/path.c b/path.c index 910756c8b3249c..c688f874580293 100644 --- a/path.c +++ b/path.c @@ -902,6 +902,35 @@ void safe_create_dir(struct repository *repo, const char *dir, int share) die(_("Could not make %s writable by group"), dir); } +int safe_create_dir_in_gitdir(struct repository *repo, const char *path) +{ + if (mkdir(path, 0777)) { + int saved_errno = errno; + struct stat st; + struct strbuf sb = STRBUF_INIT; + + if (errno != EEXIST) + return -1; + /* + * Are we looking at a path in a symlinked worktree + * whose original repository does not yet have it? + * e.g. .git/rr-cache pointing at its original + * repository in which the user hasn't performed any + * conflict resolution yet? + */ + if (lstat(path, &st) || !S_ISLNK(st.st_mode) || + strbuf_readlink(&sb, path, st.st_size) || + !is_absolute_path(sb.buf) || + mkdir(sb.buf, 0777)) { + strbuf_release(&sb); + errno = saved_errno; + return -1; + } + strbuf_release(&sb); + } + return adjust_shared_perm(repo, path); +} + static int have_same_root(const char *path1, const char *path2) { int is_abs1, is_abs2; diff --git a/path.h b/path.h index 65fe968a13a191..a427516d818429 100644 --- a/path.h +++ b/path.h @@ -221,6 +221,17 @@ char *xdg_cache_home(const char *filename); */ void safe_create_dir(struct repository *repo, const char *dir, int share); +/* + * Similar to `safe_create_dir()`, but with two differences: + * + * - It knows to resolve gitlink files for symlinked worktrees. + * + * - It always adjusts shared permissions. + * + * Returns a negative erorr code on error, 0 on success. + */ +int safe_create_dir_in_gitdir(struct repository *repo, const char *path); + # ifdef USE_THE_REPOSITORY_VARIABLE # include "strbuf.h" # include "repository.h" diff --git a/rerere.c b/rerere.c index 740e8ad1a0b40a..0832cc54840a95 100644 --- a/rerere.c +++ b/rerere.c @@ -860,7 +860,7 @@ static int do_plain_rerere(struct repository *r, string_list_insert(rr, path)->util = id; /* Ensure that the directory exists. */ - mkdir_in_gitdir(rerere_path(&buf, id, NULL)); + safe_create_dir_in_gitdir(the_repository, rerere_path(&buf, id, NULL)); } for (i = 0; i < rr->nr; i++) @@ -895,7 +895,8 @@ static int is_rerere_enabled(void) if (rerere_enabled < 0) return rr_cache_exists; - if (!rr_cache_exists && mkdir_in_gitdir(git_path_rr_cache())) + if (!rr_cache_exists && + safe_create_dir_in_gitdir(the_repository, git_path_rr_cache())) die(_("could not create directory '%s'"), git_path_rr_cache()); return 1; } From 1a99fe8010642a71063536510c578c1543d763b4 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:15 +0200 Subject: [PATCH 02/36] object-file: move `safe_create_leading_directories()` into "path.c" The `safe_create_leading_directories()` function and its relatives are located in "object-file.c", which is not a good fit as they provide generic functionality not related to objects at all. Move them into "path.c", which already hosts `safe_create_dir()` and its relative `safe_create_dir_in_gitdir()`. "path.c" is free of `the_repository`, but the moved functions depend on `the_repository` to read the "core.sharedRepository" config. Adapt the function signature to accept a repository as argument to fix the issue and adjust callers accordingly. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/bugreport.c | 4 +- builtin/clone.c | 4 +- builtin/credential-cache--daemon.c | 4 +- builtin/diagnose.c | 4 +- builtin/difftool.c | 29 ++++++----- builtin/fast-import.c | 2 +- builtin/fsck.c | 2 +- builtin/gc.c | 7 ++- builtin/init-db.c | 3 +- builtin/log.c | 4 +- builtin/mv.c | 3 +- builtin/sparse-checkout.c | 5 +- builtin/submodule--helper.c | 4 +- builtin/worktree.c | 8 +-- commit-graph.c | 2 +- dir.c | 5 +- merge-recursive.c | 4 +- midx-write.c | 2 +- notes-merge.c | 4 +- object-file.c | 81 +----------------------------- object-file.h | 33 ------------ path.c | 80 +++++++++++++++++++++++++++++ path.h | 34 +++++++++++++ refs/files-backend.c | 4 +- sequencer.c | 4 +- server-info.c | 2 +- submodule.c | 2 +- 27 files changed, 173 insertions(+), 167 deletions(-) diff --git a/builtin/bugreport.c b/builtin/bugreport.c index 66d64bfd5aec25..f78c3f2aed6e55 100644 --- a/builtin/bugreport.c +++ b/builtin/bugreport.c @@ -4,13 +4,13 @@ #include "editor.h" #include "gettext.h" #include "parse-options.h" +#include "path.h" #include "strbuf.h" #include "help.h" #include "compat/compiler.h" #include "hook.h" #include "hook-list.h" #include "diagnose.h" -#include "object-file.h" #include "setup.h" #include "version.h" @@ -141,7 +141,7 @@ int cmd_bugreport(int argc, } strbuf_addstr(&report_path, ".txt"); - switch (safe_create_leading_directories(report_path.buf)) { + switch (safe_create_leading_directories(the_repository, report_path.buf)) { case SCLD_OK: case SCLD_EXISTS: break; diff --git a/builtin/clone.c b/builtin/clone.c index 2993acb630e143..31f2198c1b3c57 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -1090,7 +1090,7 @@ int cmd_clone(int argc, sigchain_push_common(remove_junk_on_signal); if (!option_bare) { - if (safe_create_leading_directories_const(work_tree) < 0) + if (safe_create_leading_directories_const(the_repository, work_tree) < 0) die_errno(_("could not create leading directories of '%s'"), work_tree); if (dest_exists) @@ -1111,7 +1111,7 @@ int cmd_clone(int argc, junk_git_dir_flags |= REMOVE_DIR_KEEP_TOPLEVEL; junk_git_dir = git_dir; } - if (safe_create_leading_directories_const(git_dir) < 0) + if (safe_create_leading_directories_const(the_repository, git_dir) < 0) die(_("could not create leading directories of '%s'"), git_dir); if (0 <= option_verbosity) { diff --git a/builtin/credential-cache--daemon.c b/builtin/credential-cache--daemon.c index e707618e743942..5065ff4660bec9 100644 --- a/builtin/credential-cache--daemon.c +++ b/builtin/credential-cache--daemon.c @@ -2,8 +2,8 @@ #include "builtin.h" #include "abspath.h" #include "gettext.h" -#include "object-file.h" #include "parse-options.h" +#include "path.h" #ifndef NO_UNIX_SOCKETS @@ -271,7 +271,7 @@ static void init_socket_directory(const char *path) * condition in which somebody can chdir to it, sleep, then try to open * our protected socket. */ - if (safe_create_leading_directories_const(dir) < 0) + if (safe_create_leading_directories_const(the_repository, dir) < 0) die_errno("unable to create directories for '%s'", dir); if (mkdir(dir, 0700) < 0) die_errno("unable to mkdir '%s'", dir); diff --git a/builtin/diagnose.c b/builtin/diagnose.c index 33c39bd5981f22..ec86d66389e3da 100644 --- a/builtin/diagnose.c +++ b/builtin/diagnose.c @@ -3,8 +3,8 @@ #include "builtin.h" #include "abspath.h" #include "gettext.h" -#include "object-file.h" #include "parse-options.h" +#include "path.h" #include "diagnose.h" static const char * const diagnose_usage[] = { @@ -50,7 +50,7 @@ int cmd_diagnose(int argc, strbuf_addftime(&zip_path, option_suffix, localtime_r(&now, &tm), 0, 0); strbuf_addstr(&zip_path, ".zip"); - switch (safe_create_leading_directories(zip_path.buf)) { + switch (safe_create_leading_directories(the_repository, zip_path.buf)) { case SCLD_OK: case SCLD_EXISTS: break; diff --git a/builtin/difftool.c b/builtin/difftool.c index 41cd00066cc58a..8292aedaaf06f8 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -22,6 +22,7 @@ #include "gettext.h" #include "hex.h" #include "parse-options.h" +#include "path.h" #include "read-cache-ll.h" #include "repository.h" #include "sparse-index.h" @@ -271,9 +272,9 @@ static void changed_files(struct repository *repo, strbuf_release(&buf); } -static int ensure_leading_directories(char *path) +static int ensure_leading_directories(struct repository *repo, char *path) { - switch (safe_create_leading_directories(path)) { + switch (safe_create_leading_directories(repo, path)) { case SCLD_OK: case SCLD_EXISTS: return 0; @@ -341,11 +342,12 @@ static int checkout_path(unsigned mode, struct object_id *oid, return ret; } -static void write_file_in_directory(struct strbuf *dir, size_t dir_len, - const char *path, const char *content) +static void write_file_in_directory(struct repository *repo, + struct strbuf *dir, size_t dir_len, + const char *path, const char *content) { add_path(dir, dir_len, path); - ensure_leading_directories(dir->buf); + ensure_leading_directories(repo, dir->buf); unlink(dir->buf); write_file(dir->buf, "%s", content); } @@ -356,14 +358,15 @@ static void write_file_in_directory(struct strbuf *dir, size_t dir_len, * as text files, resulting in behavior that is analogous to what "git diff" * displays for symlink and submodule diffs. */ -static void write_standin_files(struct pair_entry *entry, - struct strbuf *ldir, size_t ldir_len, - struct strbuf *rdir, size_t rdir_len) +static void write_standin_files(struct repository *repo, + struct pair_entry *entry, + struct strbuf *ldir, size_t ldir_len, + struct strbuf *rdir, size_t rdir_len) { if (*entry->left) - write_file_in_directory(ldir, ldir_len, entry->path, entry->left); + write_file_in_directory(repo, ldir, ldir_len, entry->path, entry->left); if (*entry->right) - write_file_in_directory(rdir, rdir_len, entry->path, entry->right); + write_file_in_directory(repo, rdir, rdir_len, entry->path, entry->right); } static int run_dir_diff(struct repository *repo, @@ -533,7 +536,7 @@ static int run_dir_diff(struct repository *repo, ADD_CACHE_JUST_APPEND); add_path(&rdir, rdir_len, dst_path); - if (ensure_leading_directories(rdir.buf)) { + if (ensure_leading_directories(repo, rdir.buf)) { ret = error("could not create " "directory for '%s'", dst_path); @@ -576,7 +579,7 @@ static int run_dir_diff(struct repository *repo, */ hashmap_for_each_entry(&submodules, &iter, entry, entry /* member name */) { - write_standin_files(entry, &ldir, ldir_len, &rdir, rdir_len); + write_standin_files(repo, entry, &ldir, ldir_len, &rdir, rdir_len); } /* @@ -587,7 +590,7 @@ static int run_dir_diff(struct repository *repo, hashmap_for_each_entry(&symlinks2, &iter, entry, entry /* member name */) { - write_standin_files(entry, &ldir, ldir_len, &rdir, rdir_len); + write_standin_files(repo, entry, &ldir, ldir_len, &rdir, rdir_len); } strbuf_setlen(&ldir, ldir_len); diff --git a/builtin/fast-import.c b/builtin/fast-import.c index 63880b595ccad9..e4523cc6f1bd39 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -1720,7 +1720,7 @@ static void dump_marks(void) if (!export_marks_file || (import_marks_file && !import_marks_file_done)) return; - if (safe_create_leading_directories_const(export_marks_file)) { + if (safe_create_leading_directories_const(the_repository, export_marks_file)) { failure |= error_errno("unable to create leading directories of %s", export_marks_file); return; diff --git a/builtin/fsck.c b/builtin/fsck.c index 9c8a6d6a8dfa5f..92312b5959174c 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -332,7 +332,7 @@ static void check_unreachable_object(struct object *obj) describe_object(&obj->oid)); FILE *f; - if (safe_create_leading_directories_const(filename)) { + if (safe_create_leading_directories_const(the_repository, filename)) { error(_("could not create lost-found")); free(filename); return; diff --git a/builtin/gc.c b/builtin/gc.c index 99431fd46744cd..dae1e545514825 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -28,7 +28,6 @@ #include "commit.h" #include "commit-graph.h" #include "packfile.h" -#include "object-file.h" #include "object-store-ll.h" #include "pack.h" #include "pack-objects.h" @@ -2099,7 +2098,7 @@ static int launchctl_schedule_plist(const char *exec_path, enum schedule_priorit } strbuf_addstr(&plist, "\n\n\n"); - if (safe_create_leading_directories(filename)) + if (safe_create_leading_directories(the_repository, filename)) die(_("failed to create directories for '%s'"), filename); if ((long)lock_file_timeout_ms < 0 && @@ -2565,7 +2564,7 @@ static int systemd_timer_write_timer_file(enum schedule_priority schedule, filename = xdg_config_home_systemd(local_timer_name); - if (safe_create_leading_directories(filename)) { + if (safe_create_leading_directories(the_repository, filename)) { error(_("failed to create directories for '%s'"), filename); goto error; } @@ -2638,7 +2637,7 @@ static int systemd_timer_write_service_template(const char *exec_path) char *local_service_name = xstrfmt(SYSTEMD_UNIT_FORMAT, "", "service"); filename = xdg_config_home_systemd(local_service_name); - if (safe_create_leading_directories(filename)) { + if (safe_create_leading_directories(the_repository, filename)) { error(_("failed to create directories for '%s'"), filename); goto error; } diff --git a/builtin/init-db.c b/builtin/init-db.c index 196dccdd77acb8..91c2563e3410ff 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -8,7 +8,6 @@ #include "abspath.h" #include "environment.h" #include "gettext.h" -#include "object-file.h" #include "parse-options.h" #include "path.h" #include "refs.h" @@ -134,7 +133,7 @@ int cmd_init_db(int argc, */ saved = repo_settings_get_shared_repository(the_repository); repo_settings_set_shared_repository(the_repository, 0); - switch (safe_create_leading_directories_const(argv[0])) { + switch (safe_create_leading_directories_const(the_repository, argv[0])) { case SCLD_OK: case SCLD_PERMS: break; diff --git a/builtin/log.c b/builtin/log.c index 0d4c579dad761f..516c9ec8400b75 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -14,7 +14,6 @@ #include "gettext.h" #include "hex.h" #include "refs.h" -#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" #include "pager.h" @@ -29,6 +28,7 @@ #include "tag.h" #include "reflog-walk.h" #include "patch-ids.h" +#include "path.h" #include "shortlog.h" #include "remote.h" #include "string-list.h" @@ -2311,7 +2311,7 @@ int cmd_format_patch(int argc, */ saved = repo_settings_get_shared_repository(the_repository); repo_settings_set_shared_repository(the_repository, 0); - switch (safe_create_leading_directories_const(output_directory)) { + switch (safe_create_leading_directories_const(the_repository, output_directory)) { case SCLD_OK: case SCLD_EXISTS: break; diff --git a/builtin/mv.c b/builtin/mv.c index 55a7d471dca012..99fe7a0c561ec2 100644 --- a/builtin/mv.c +++ b/builtin/mv.c @@ -15,6 +15,7 @@ #include "gettext.h" #include "name-hash.h" #include "object-file.h" +#include "path.h" #include "pathspec.h" #include "lockfile.h" #include "dir.h" @@ -555,7 +556,7 @@ int cmd_mv(int argc, */ char *dst_dup = xstrdup(dst); string_list_append(&dirty_paths, dst); - safe_create_leading_directories(dst_dup); + safe_create_leading_directories(the_repository, dst_dup); FREE_AND_NULL(dst_dup); rename(src, dst); } diff --git a/builtin/sparse-checkout.c b/builtin/sparse-checkout.c index 14dcace5f8ff7c..1bf01591b27523 100644 --- a/builtin/sparse-checkout.c +++ b/builtin/sparse-checkout.c @@ -9,6 +9,7 @@ #include "object-file.h" #include "object-name.h" #include "parse-options.h" +#include "path.h" #include "pathspec.h" #include "strbuf.h" #include "string-list.h" @@ -335,7 +336,7 @@ static int write_patterns_and_update(struct pattern_list *pl) sparse_filename = get_sparse_checkout_filename(); - if (safe_create_leading_directories(sparse_filename)) + if (safe_create_leading_directories(the_repository, sparse_filename)) die(_("failed to create directory for sparse-checkout file")); hold_lock_file_for_update(&lk, sparse_filename, LOCK_DIE_ON_ERROR); @@ -491,7 +492,7 @@ static int sparse_checkout_init(int argc, const char **argv, const char *prefix, FILE *fp; /* assume we are in a fresh repo, but update the sparse-checkout file */ - if (safe_create_leading_directories(sparse_filename)) + if (safe_create_leading_directories(the_repository, sparse_filename)) die(_("unable to create leading directories of %s"), sparse_filename); fp = xfopen(sparse_filename, "w"); diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 570226ea16653a..cc001d0b4cd884 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -1739,7 +1739,7 @@ static int clone_submodule(const struct module_clone_data *clone_data, !is_empty_dir(clone_data_path)) die(_("directory not empty: '%s'"), clone_data_path); - if (safe_create_leading_directories_const(sm_gitdir) < 0) + if (safe_create_leading_directories_const(the_repository, sm_gitdir) < 0) die(_("could not create directory '%s'"), sm_gitdir); prepare_possible_alternates(clone_data->name, reference); @@ -1800,7 +1800,7 @@ static int clone_submodule(const struct module_clone_data *clone_data, if (clone_data->require_init && !stat(clone_data_path, &st) && !is_empty_dir(clone_data_path)) die(_("directory not empty: '%s'"), clone_data_path); - if (safe_create_leading_directories_const(clone_data_path) < 0) + if (safe_create_leading_directories_const(the_repository, clone_data_path) < 0) die(_("could not create directory '%s'"), clone_data_path); path = xstrfmt("%s/index", sm_gitdir); unlink_or_warn(path); diff --git a/builtin/worktree.c b/builtin/worktree.c index 87ccd47794cbf7..88a36ea9f8674e 100644 --- a/builtin/worktree.c +++ b/builtin/worktree.c @@ -348,7 +348,7 @@ static void copy_sparse_checkout(const char *worktree_git_dir) char *to_file = xstrfmt("%s/info/sparse-checkout", worktree_git_dir); if (file_exists(from_file)) { - if (safe_create_leading_directories(to_file) || + if (safe_create_leading_directories(the_repository, to_file) || copy_file(to_file, from_file, 0666)) error(_("failed to copy '%s' to '%s'; sparse-checkout may not work correctly"), from_file, to_file); @@ -367,7 +367,7 @@ static void copy_filtered_worktree_config(const char *worktree_git_dir) struct config_set cs = { { 0 } }; int bare; - if (safe_create_leading_directories(to_file) || + if (safe_create_leading_directories(the_repository, to_file) || copy_file(to_file, from_file, 0666)) { error(_("failed to copy worktree config from '%s' to '%s'"), from_file, to_file); @@ -466,7 +466,7 @@ static int add_worktree(const char *path, const char *refname, name = sb_name.buf; repo_git_path_replace(the_repository, &sb_repo, "worktrees/%s", name); len = sb_repo.len; - if (safe_create_leading_directories_const(sb_repo.buf)) + if (safe_create_leading_directories_const(the_repository, sb_repo.buf)) die_errno(_("could not create leading directories of '%s'"), sb_repo.buf); @@ -498,7 +498,7 @@ static int add_worktree(const char *path, const char *refname, write_file(sb.buf, _("initializing")); strbuf_addf(&sb_git, "%s/.git", path); - if (safe_create_leading_directories_const(sb_git.buf)) + if (safe_create_leading_directories_const(the_repository, sb_git.buf)) die_errno(_("could not create leading directories of '%s'"), sb_git.buf); junk_work_tree = xstrdup(path); diff --git a/commit-graph.c b/commit-graph.c index 8286d5dda241ff..3b5bae00af930b 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -2065,7 +2065,7 @@ static int write_commit_graph_file(struct write_commit_graph_context *ctx) ctx->graph_name = get_commit_graph_filename(ctx->odb); } - if (safe_create_leading_directories(ctx->graph_name)) { + if (safe_create_leading_directories(the_repository, ctx->graph_name)) { error(_("unable to create leading directories of %s"), ctx->graph_name); return -1; diff --git a/dir.c b/dir.c index 28b0e03feb498a..49008739b9bcae 100644 --- a/dir.c +++ b/dir.c @@ -17,7 +17,6 @@ #include "environment.h" #include "gettext.h" #include "name-hash.h" -#include "object-file.h" #include "object-store-ll.h" #include "path.h" #include "refs.h" @@ -4063,12 +4062,12 @@ void connect_work_tree_and_git_dir(const char *work_tree_, /* Prepare .git file */ strbuf_addf(&gitfile_sb, "%s/.git", work_tree_); - if (safe_create_leading_directories_const(gitfile_sb.buf)) + if (safe_create_leading_directories_const(the_repository, gitfile_sb.buf)) die(_("could not create directories for %s"), gitfile_sb.buf); /* Prepare config file */ strbuf_addf(&cfg_sb, "%s/config", git_dir_); - if (safe_create_leading_directories_const(cfg_sb.buf)) + if (safe_create_leading_directories_const(the_repository, cfg_sb.buf)) die(_("could not create directories for %s"), cfg_sb.buf); git_dir = real_pathdup(git_dir_, 1); diff --git a/merge-recursive.c b/merge-recursive.c index 9aedffc546b230..f71490517e1ff8 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -910,7 +910,7 @@ static int make_room_for_path(struct merge_options *opt, const char *path) } /* Make sure leading directories are created */ - status = safe_create_leading_directories_const(path); + status = safe_create_leading_directories_const(the_repository, path); if (status) { if (status == SCLD_EXISTS) /* something else exists */ @@ -1003,7 +1003,7 @@ static int update_file_flags(struct merge_options *opt, close(fd); } else if (S_ISLNK(contents->mode)) { char *lnk = xmemdupz(buf, size); - safe_create_leading_directories_const(path); + safe_create_leading_directories_const(the_repository, path); unlink(path); if (symlink(lnk, path)) ret = err(opt, _("failed to symlink '%s': %s"), diff --git a/midx-write.c b/midx-write.c index a628ac24dcb428..fbba55f9d92194 100644 --- a/midx-write.c +++ b/midx-write.c @@ -1086,7 +1086,7 @@ static int write_midx_internal(struct repository *r, const char *object_dir, object_dir); else get_midx_filename(r->hash_algo, &midx_name, object_dir); - if (safe_create_leading_directories(midx_name.buf)) + if (safe_create_leading_directories(r, midx_name.buf)) die_errno(_("unable to create leading directories of %s"), midx_name.buf); diff --git a/notes-merge.c b/notes-merge.c index 5008faef450ca3..fce45043655edb 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -296,7 +296,7 @@ static void check_notes_merge_worktree(struct notes_merge_options *o) "(%s exists)."), repo_git_path_replace(the_repository, &buf, "NOTES_MERGE_*")); } - if (safe_create_leading_directories_const(repo_git_path_replace(the_repository, &buf, + if (safe_create_leading_directories_const(the_repository, repo_git_path_replace(the_repository, &buf, NOTES_MERGE_WORKTREE "/.test"))) die_errno("unable to create directory %s", repo_git_path_replace(the_repository, &buf, NOTES_MERGE_WORKTREE)); @@ -314,7 +314,7 @@ static void write_buf_to_worktree(const struct object_id *obj, { int fd; char *path = repo_git_path(the_repository, NOTES_MERGE_WORKTREE "/%s", oid_to_hex(obj)); - if (safe_create_leading_directories_const(path)) + if (safe_create_leading_directories_const(the_repository, path)) die_errno("unable to create directory for '%s'", path); fd = xopen(path, O_WRONLY | O_EXCL | O_CREAT, 0666); diff --git a/object-file.c b/object-file.c index 00451876bd05cf..6228e1c40f8fc4 100644 --- a/object-file.c +++ b/object-file.c @@ -90,83 +90,6 @@ static int get_conv_flags(unsigned flags) return 0; } -static enum scld_error safe_create_leading_directories_1(char *path, int share) -{ - char *next_component = path + offset_1st_component(path); - enum scld_error ret = SCLD_OK; - - while (ret == SCLD_OK && next_component) { - struct stat st; - char *slash = next_component, slash_character; - - while (*slash && !is_dir_sep(*slash)) - slash++; - - if (!*slash) - break; - - next_component = slash + 1; - while (is_dir_sep(*next_component)) - next_component++; - if (!*next_component) - break; - - slash_character = *slash; - *slash = '\0'; - if (!stat(path, &st)) { - /* path exists */ - if (!S_ISDIR(st.st_mode)) { - errno = ENOTDIR; - ret = SCLD_EXISTS; - } - } else if (mkdir(path, 0777)) { - if (errno == EEXIST && - !stat(path, &st) && S_ISDIR(st.st_mode)) - ; /* somebody created it since we checked */ - else if (errno == ENOENT) - /* - * Either mkdir() failed because - * somebody just pruned the containing - * directory, or stat() failed because - * the file that was in our way was - * just removed. Either way, inform - * the caller that it might be worth - * trying again: - */ - ret = SCLD_VANISHED; - else - ret = SCLD_FAILED; - } else if (share && adjust_shared_perm(the_repository, path)) { - ret = SCLD_PERMS; - } - *slash = slash_character; - } - return ret; -} - -enum scld_error safe_create_leading_directories(char *path) -{ - return safe_create_leading_directories_1(path, 1); -} - -enum scld_error safe_create_leading_directories_no_share(char *path) -{ - return safe_create_leading_directories_1(path, 0); -} - -enum scld_error safe_create_leading_directories_const(const char *path) -{ - int save_errno; - /* path points to cache entries, so xstrdup before messing with it */ - char *buf = xstrdup(path); - enum scld_error result = safe_create_leading_directories(buf); - - save_errno = errno; - free(buf); - errno = save_errno; - return result; -} - int odb_mkstemp(struct strbuf *temp_filename, const char *pattern) { int fd; @@ -183,7 +106,7 @@ int odb_mkstemp(struct strbuf *temp_filename, const char *pattern) /* slow path */ /* some mkstemp implementations erase temp_filename on failure */ repo_git_path_replace(the_repository, temp_filename, "objects/%s", pattern); - safe_create_leading_directories(temp_filename->buf); + safe_create_leading_directories(the_repository, temp_filename->buf); return xmkstemp_mode(temp_filename->buf, mode); } @@ -196,7 +119,7 @@ int odb_pack_keep(const char *name) return fd; /* slow path */ - safe_create_leading_directories_const(name); + safe_create_leading_directories_const(the_repository, name); return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); } diff --git a/object-file.h b/object-file.h index 4649a3f37d4da9..922f2bba8c9156 100644 --- a/object-file.h +++ b/object-file.h @@ -21,39 +21,6 @@ extern int fetch_if_missing; int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); -/* - * Create the directory containing the named path, using care to be - * somewhat safe against races. Return one of the scld_error values to - * indicate success/failure. On error, set errno to describe the - * problem. - * - * SCLD_VANISHED indicates that one of the ancestor directories of the - * path existed at one point during the function call and then - * suddenly vanished, probably because another process pruned the - * directory while we were working. To be robust against this kind of - * race, callers might want to try invoking the function again when it - * returns SCLD_VANISHED. - * - * safe_create_leading_directories() temporarily changes path while it - * is working but restores it before returning. - * safe_create_leading_directories_const() doesn't modify path, even - * temporarily. Both these variants adjust the permissions of the - * created directories to honor core.sharedRepository, so they are best - * suited for files inside the git dir. For working tree files, use - * safe_create_leading_directories_no_share() instead, as it ignores - * the core.sharedRepository setting. - */ -enum scld_error { - SCLD_OK = 0, - SCLD_FAILED = -1, - SCLD_PERMS = -2, - SCLD_EXISTS = -3, - SCLD_VANISHED = -4 -}; -enum scld_error safe_create_leading_directories(char *path); -enum scld_error safe_create_leading_directories_const(const char *path); -enum scld_error safe_create_leading_directories_no_share(char *path); - int git_open_cloexec(const char *name, int flags); #define git_open(name) git_open_cloexec(name, O_RDONLY) diff --git a/path.c b/path.c index c688f874580293..62d67166dff945 100644 --- a/path.c +++ b/path.c @@ -931,6 +931,86 @@ int safe_create_dir_in_gitdir(struct repository *repo, const char *path) return adjust_shared_perm(repo, path); } +static enum scld_error safe_create_leading_directories_1(struct repository *repo, + char *path) +{ + char *next_component = path + offset_1st_component(path); + enum scld_error ret = SCLD_OK; + + while (ret == SCLD_OK && next_component) { + struct stat st; + char *slash = next_component, slash_character; + + while (*slash && !is_dir_sep(*slash)) + slash++; + + if (!*slash) + break; + + next_component = slash + 1; + while (is_dir_sep(*next_component)) + next_component++; + if (!*next_component) + break; + + slash_character = *slash; + *slash = '\0'; + if (!stat(path, &st)) { + /* path exists */ + if (!S_ISDIR(st.st_mode)) { + errno = ENOTDIR; + ret = SCLD_EXISTS; + } + } else if (mkdir(path, 0777)) { + if (errno == EEXIST && + !stat(path, &st) && S_ISDIR(st.st_mode)) + ; /* somebody created it since we checked */ + else if (errno == ENOENT) + /* + * Either mkdir() failed because + * somebody just pruned the containing + * directory, or stat() failed because + * the file that was in our way was + * just removed. Either way, inform + * the caller that it might be worth + * trying again: + */ + ret = SCLD_VANISHED; + else + ret = SCLD_FAILED; + } else if (repo && adjust_shared_perm(repo, path)) { + ret = SCLD_PERMS; + } + *slash = slash_character; + } + return ret; +} + +enum scld_error safe_create_leading_directories(struct repository *repo, + char *path) +{ + return safe_create_leading_directories_1(repo, path); +} + +enum scld_error safe_create_leading_directories_no_share(char *path) +{ + return safe_create_leading_directories_1(NULL, path); +} + +enum scld_error safe_create_leading_directories_const(struct repository *repo, + const char *path) +{ + int save_errno; + /* path points to cache entries, so xstrdup before messing with it */ + char *buf = xstrdup(path); + enum scld_error result = safe_create_leading_directories(repo, buf); + + save_errno = errno; + free(buf); + errno = save_errno; + return result; +} + static int have_same_root(const char *path1, const char *path2) { int is_abs1, is_abs2; diff --git a/path.h b/path.h index a427516d818429..fd1a194b060135 100644 --- a/path.h +++ b/path.h @@ -232,6 +232,40 @@ void safe_create_dir(struct repository *repo, const char *dir, int share); */ int safe_create_dir_in_gitdir(struct repository *repo, const char *path); +/* + * Create the directory containing the named path, using care to be + * somewhat safe against races. Return one of the scld_error values to + * indicate success/failure. On error, set errno to describe the + * problem. + * + * SCLD_VANISHED indicates that one of the ancestor directories of the + * path existed at one point during the function call and then + * suddenly vanished, probably because another process pruned the + * directory while we were working. To be robust against this kind of + * race, callers might want to try invoking the function again when it + * returns SCLD_VANISHED. + * + * safe_create_leading_directories() temporarily changes path while it + * is working but restores it before returning. + * safe_create_leading_directories_const() doesn't modify path, even + * temporarily. Both these variants adjust the permissions of the + * created directories to honor core.sharedRepository, so they are best + * suited for files inside the git dir. For working tree files, use + * safe_create_leading_directories_no_share() instead, as it ignores + * the core.sharedRepository setting. + */ +enum scld_error { + SCLD_OK = 0, + SCLD_FAILED = -1, + SCLD_PERMS = -2, + SCLD_EXISTS = -3, + SCLD_VANISHED = -4 +}; +enum scld_error safe_create_leading_directories(struct repository *repo, char *path); +enum scld_error safe_create_leading_directories_const(struct repository *repo, + const char *path); +enum scld_error safe_create_leading_directories_no_share(char *path); + # ifdef USE_THE_REPOSITORY_VARIABLE # include "strbuf.h" # include "repository.h" diff --git a/refs/files-backend.c b/refs/files-backend.c index 91d3aca70a7fab..10c439a56f827e 100644 --- a/refs/files-backend.c +++ b/refs/files-backend.c @@ -705,7 +705,7 @@ static int lock_raw_ref(struct files_ref_store *refs, files_ref_path(refs, &ref_file, refname); retry: - switch (safe_create_leading_directories(ref_file.buf)) { + switch (safe_create_leading_directories(the_repository, ref_file.buf)) { case SCLD_OK: break; /* success */ case SCLD_EXISTS: @@ -1109,7 +1109,7 @@ static int raceproof_create_file(const char *path, create_file_fn fn, void *cb) strbuf_addstr(&path_copy, path); do { - scld_result = safe_create_leading_directories(path_copy.buf); + scld_result = safe_create_leading_directories(the_repository, path_copy.buf); if (scld_result == SCLD_OK) goto retry_fn; } while (scld_result == SCLD_VANISHED && create_directories_remaining-- > 0); diff --git a/sequencer.c b/sequencer.c index c112d2e1c43e26..9fda9be9266b26 100644 --- a/sequencer.c +++ b/sequencer.c @@ -4411,7 +4411,7 @@ static int write_update_refs_state(struct string_list *refs_to_oids) goto cleanup; } - if (safe_create_leading_directories(path)) { + if (safe_create_leading_directories(the_repository, path)) { result = error(_("unable to create leading directories of %s"), path); goto cleanup; @@ -4677,7 +4677,7 @@ static void create_autostash_internal(struct repository *r, strbuf_add_unique_abbrev(&buf, &oid, DEFAULT_ABBREV); if (path) { - if (safe_create_leading_directories_const(path)) + if (safe_create_leading_directories_const(the_repository, path)) die(_("Could not create directory for '%s'"), path); write_file(path, "%s", oid_to_hex(&oid)); diff --git a/server-info.c b/server-info.c index 1ca0e00d51e6c0..f0646ac92a9eed 100644 --- a/server-info.c +++ b/server-info.c @@ -88,7 +88,7 @@ static int update_info_file(struct repository *r, char *path, .old_sb = STRBUF_INIT }; - safe_create_leading_directories(path); + safe_create_leading_directories(r, path); f = mks_tempfile_m(tmp, 0666); if (!f) goto out; diff --git a/submodule.c b/submodule.c index 0821507ecaa493..218c8c17603ff2 100644 --- a/submodule.c +++ b/submodule.c @@ -2384,7 +2384,7 @@ static void relocate_single_git_dir_into_superproject(const char *path, if (validate_submodule_git_dir(new_gitdir.buf, sub->name) < 0) die(_("refusing to move '%s' into an existing git dir"), real_old_git_dir); - if (safe_create_leading_directories_const(new_gitdir.buf) < 0) + if (safe_create_leading_directories_const(the_repository, new_gitdir.buf) < 0) die(_("could not create directory '%s'"), new_gitdir.buf); real_new_git_dir = real_pathdup(new_gitdir.buf, 1); From 97dc141fd676e7079c2fd51e3bea2681a5b9f824 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:16 +0200 Subject: [PATCH 03/36] object-file: move `git_open_cloexec()` to "compat/open.c" The `git_open_cloexec()` wrapper function provides the ability to open a file with `O_CLOEXEC` in a platform-agnostic way. This function is provided by "object-file.c" even though it is not specific to the object subsystem at all. Move the file into "compat/open.c". This file already exists before this commit, but has only been compiled conditionally depending on whether or not open(3p) may return EINTR. With this change we now unconditionally compile the object, but wrap `git_open_with_retry()` in an ifdef. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Makefile | 2 +- commit-graph.c | 1 - compat/open.c | 29 +++++++++++++++++++++++++++++ git-compat-util.h | 3 +++ meson.build | 1 + midx.c | 1 - object-file.c | 27 --------------------------- object-file.h | 3 --- pack-bitmap.c | 1 - pack-mtimes.c | 1 - pack-revindex.c | 1 - 11 files changed, 34 insertions(+), 36 deletions(-) diff --git a/Makefile b/Makefile index c41fc41ef0e4b4..bb5407b4703798 100644 --- a/Makefile +++ b/Makefile @@ -994,6 +994,7 @@ LIB_OBJS += common-exit.o LIB_OBJS += common-init.o LIB_OBJS += compat/nonblock.o LIB_OBJS += compat/obstack.o +LIB_OBJS += compat/open.o LIB_OBJS += compat/terminal.o LIB_OBJS += compiler-tricks/not-constant.o LIB_OBJS += config.o @@ -1812,7 +1813,6 @@ ifdef FREAD_READS_DIRECTORIES endif ifdef OPEN_RETURNS_EINTR COMPAT_CFLAGS += -DOPEN_RETURNS_EINTR - COMPAT_OBJS += compat/open.o endif ifdef NO_SYMLINK_HEAD BASIC_CFLAGS += -DNO_SYMLINK_HEAD diff --git a/commit-graph.c b/commit-graph.c index 3b5bae00af930b..9621c454972957 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -13,7 +13,6 @@ #include "refs.h" #include "hash-lookup.h" #include "commit-graph.h" -#include "object-file.h" #include "object-store-ll.h" #include "oid-array.h" #include "path.h" diff --git a/compat/open.c b/compat/open.c index eb3754a23b8f62..37ae2b1aeb9249 100644 --- a/compat/open.c +++ b/compat/open.c @@ -1,5 +1,6 @@ #include "git-compat-util.h" +#ifdef OPEN_RETURNS_EINTR #undef open int git_open_with_retry(const char *path, int flags, ...) { @@ -23,3 +24,31 @@ int git_open_with_retry(const char *path, int flags, ...) return ret; } +#endif + +int git_open_cloexec(const char *name, int flags) +{ + int fd; + static int o_cloexec = O_CLOEXEC; + + fd = open(name, flags | o_cloexec); + if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { + /* Try again w/o O_CLOEXEC: the kernel might not support it */ + o_cloexec &= ~O_CLOEXEC; + fd = open(name, flags | o_cloexec); + } + +#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC) + { + static int fd_cloexec = FD_CLOEXEC; + + if (!o_cloexec && 0 <= fd && fd_cloexec) { + /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */ + int flags = fcntl(fd, F_GETFD); + if (fcntl(fd, F_SETFD, flags | fd_cloexec)) + fd_cloexec = 0; + } + } +#endif + return fd; +} diff --git a/git-compat-util.h b/git-compat-util.h index cf733b38acdea0..9273a8ee087439 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -1000,6 +1000,9 @@ int git_vsnprintf(char *str, size_t maxsize, int git_open_with_retry(const char *path, int flag, ...); #endif +int git_open_cloexec(const char *name, int flags); +#define git_open(name) git_open_cloexec(name, O_RDONLY) + #ifdef __GLIBC_PREREQ #if __GLIBC_PREREQ(2, 1) #define HAVE_STRCHRNUL diff --git a/meson.build b/meson.build index 145d2f7ff9e7ed..a55e800b85b903 100644 --- a/meson.build +++ b/meson.build @@ -263,6 +263,7 @@ libgit_sources = [ 'common-init.c', 'compat/nonblock.c', 'compat/obstack.c', + 'compat/open.c', 'compat/terminal.c', 'compiler-tricks/not-constant.c', 'config.c', diff --git a/midx.c b/midx.c index 807fdf72f7b81c..3d0015f782818c 100644 --- a/midx.c +++ b/midx.c @@ -5,7 +5,6 @@ #include "dir.h" #include "hex.h" #include "packfile.h" -#include "object-file.h" #include "hash-lookup.h" #include "midx.h" #include "progress.h" diff --git a/object-file.c b/object-file.c index 6228e1c40f8fc4..c3e20417f3f71e 100644 --- a/object-file.c +++ b/object-file.c @@ -833,33 +833,6 @@ int stream_object_signature(struct repository *r, const struct object_id *oid) return !oideq(oid, &real_oid) ? -1 : 0; } -int git_open_cloexec(const char *name, int flags) -{ - int fd; - static int o_cloexec = O_CLOEXEC; - - fd = open(name, flags | o_cloexec); - if ((o_cloexec & O_CLOEXEC) && fd < 0 && errno == EINVAL) { - /* Try again w/o O_CLOEXEC: the kernel might not support it */ - o_cloexec &= ~O_CLOEXEC; - fd = open(name, flags | o_cloexec); - } - -#if defined(F_GETFD) && defined(F_SETFD) && defined(FD_CLOEXEC) - { - static int fd_cloexec = FD_CLOEXEC; - - if (!o_cloexec && 0 <= fd && fd_cloexec) { - /* Opened w/o O_CLOEXEC? try with fcntl(2) to add it */ - int flags = fcntl(fd, F_GETFD); - if (fcntl(fd, F_SETFD, flags | fd_cloexec)) - fd_cloexec = 0; - } - } -#endif - return fd; -} - /* * Find "oid" as a loose object in the local repository or in an alternate. * Returns 0 on success, negative on failure. diff --git a/object-file.h b/object-file.h index 922f2bba8c9156..353d8a85c3347e 100644 --- a/object-file.h +++ b/object-file.h @@ -21,9 +21,6 @@ extern int fetch_if_missing; int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); -int git_open_cloexec(const char *name, int flags); -#define git_open(name) git_open_cloexec(name, O_RDONLY) - /** * unpack_loose_header() initializes the data stream needed to unpack * a loose object header. diff --git a/pack-bitmap.c b/pack-bitmap.c index 7fd78c634ef142..0dbd7c4ffe161a 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -17,7 +17,6 @@ #include "packfile.h" #include "repository.h" #include "trace2.h" -#include "object-file.h" #include "object-store-ll.h" #include "list-objects-filter-options.h" #include "midx.h" diff --git a/pack-mtimes.c b/pack-mtimes.c index cdf30b8d2b0e80..bcea28e521dfb3 100644 --- a/pack-mtimes.c +++ b/pack-mtimes.c @@ -1,7 +1,6 @@ #include "git-compat-util.h" #include "gettext.h" #include "pack-mtimes.h" -#include "object-file.h" #include "object-store-ll.h" #include "packfile.h" #include "strbuf.h" diff --git a/pack-revindex.c b/pack-revindex.c index 038e0c96b1ca5e..1ee7b49e206ad4 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -1,7 +1,6 @@ #include "git-compat-util.h" #include "gettext.h" #include "pack-revindex.h" -#include "object-file.h" #include "object-store-ll.h" #include "packfile.h" #include "strbuf.h" From 632b5e3ee274a2012a88bc32af2d9cf04c5bd363 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:17 +0200 Subject: [PATCH 04/36] object-file: move `xmmap()` into "wrapper.c" The `xmmap()` function is provided by "object-file.c" even though its functionality has nothing to do with the object file subsystem. Move it into "wrapper.c", whose header already declares those functions. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- object-file.c | 48 ------------------------------------------------ wrapper.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/object-file.c b/object-file.c index c3e20417f3f71e..a7868201d09a85 100644 --- a/object-file.c +++ b/object-file.c @@ -718,54 +718,6 @@ int has_loose_object(const struct object_id *oid) return check_and_freshen(oid, 0); } -static void mmap_limit_check(size_t length) -{ - static size_t limit = 0; - if (!limit) { - limit = git_env_ulong("GIT_MMAP_LIMIT", 0); - if (!limit) - limit = SIZE_MAX; - } - if (length > limit) - die(_("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX), - (uintmax_t)length, (uintmax_t)limit); -} - -void *xmmap_gently(void *start, size_t length, - int prot, int flags, int fd, off_t offset) -{ - void *ret; - - mmap_limit_check(length); - ret = mmap(start, length, prot, flags, fd, offset); - if (ret == MAP_FAILED && !length) - ret = NULL; - return ret; -} - -const char *mmap_os_err(void) -{ - static const char blank[] = ""; -#if defined(__linux__) - if (errno == ENOMEM) { - /* this continues an existing error message: */ - static const char enomem[] = -", check sys.vm.max_map_count and/or RLIMIT_DATA"; - return enomem; - } -#endif /* OS-specific bits */ - return blank; -} - -void *xmmap(void *start, size_t length, - int prot, int flags, int fd, off_t offset) -{ - void *ret = xmmap_gently(start, length, prot, flags, fd, offset); - if (ret == MAP_FAILED) - die_errno(_("mmap failed%s"), mmap_os_err()); - return ret; -} - static int format_object_header_literally(char *str, size_t size, const char *type, size_t objsize) { diff --git a/wrapper.c b/wrapper.c index 8b985931490d62..3c79778055eb9d 100644 --- a/wrapper.c +++ b/wrapper.c @@ -829,3 +829,51 @@ uint32_t git_rand(unsigned flags) return result; } + +static void mmap_limit_check(size_t length) +{ + static size_t limit = 0; + if (!limit) { + limit = git_env_ulong("GIT_MMAP_LIMIT", 0); + if (!limit) + limit = SIZE_MAX; + } + if (length > limit) + die(_("attempting to mmap %"PRIuMAX" over limit %"PRIuMAX), + (uintmax_t)length, (uintmax_t)limit); +} + +void *xmmap_gently(void *start, size_t length, + int prot, int flags, int fd, off_t offset) +{ + void *ret; + + mmap_limit_check(length); + ret = mmap(start, length, prot, flags, fd, offset); + if (ret == MAP_FAILED && !length) + ret = NULL; + return ret; +} + +const char *mmap_os_err(void) +{ + static const char blank[] = ""; +#if defined(__linux__) + if (errno == ENOMEM) { + /* this continues an existing error message: */ + static const char enomem[] = +", check sys.vm.max_map_count and/or RLIMIT_DATA"; + return enomem; + } +#endif /* OS-specific bits */ + return blank; +} + +void *xmmap(void *start, size_t length, + int prot, int flags, int fd, off_t offset) +{ + void *ret = xmmap_gently(start, length, prot, flags, fd, offset); + if (ret == MAP_FAILED) + die_errno(_("mmap failed%s"), mmap_os_err()); + return ret; +} From d9f517d051d1008178cb6c809b5f906d0905508f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:18 +0200 Subject: [PATCH 05/36] object-file: split out functions relating to object store subsystem While we have the "object-store.h" header, most of the functionality for object stores is actually hosted in "object-file.c". This makes it hard to find relevant functions and causes us to mix up concerns. Split out functions relating to the object store subsystem into a new "object-store.c" file. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Makefile | 1 + builtin/checkout.c | 1 + builtin/merge-file.c | 1 + builtin/mktree.c | 1 + builtin/notes.c | 1 + builtin/receive-pack.c | 1 + builtin/tag.c | 1 + builtin/unpack-file.c | 1 + builtin/unpack-objects.c | 1 + commit.c | 1 + http-push.c | 1 + match-trees.c | 3 +- merge-ort.c | 3 +- meson.build | 1 + notes-cache.c | 3 +- notes.c | 3 +- object-file.c | 990 +-------------------------------------- object-file.h | 68 ++- object-store-ll.h | 60 +-- object-store.c | 972 ++++++++++++++++++++++++++++++++++++++ 20 files changed, 1074 insertions(+), 1040 deletions(-) create mode 100644 object-store.c diff --git a/Makefile b/Makefile index bb5407b4703798..d8ad76fb79190a 100644 --- a/Makefile +++ b/Makefile @@ -1086,6 +1086,7 @@ LIB_OBJS += notes.o LIB_OBJS += object-file-convert.o LIB_OBJS += object-file.o LIB_OBJS += object-name.o +LIB_OBJS += object-store.o LIB_OBJS += object.o LIB_OBJS += oid-array.o LIB_OBJS += oidmap.o diff --git a/builtin/checkout.c b/builtin/checkout.c index 2e7486cf6587fe..3e68623838ac03 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -18,6 +18,7 @@ #include "lockfile.h" #include "mem-pool.h" #include "merge-recursive.h" +#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" #include "parse-options.h" diff --git a/builtin/merge-file.c b/builtin/merge-file.c index 7e315f374b2425..2b16b10d2cad1f 100644 --- a/builtin/merge-file.c +++ b/builtin/merge-file.c @@ -5,6 +5,7 @@ #include "abspath.h" #include "diff.h" #include "hex.h" +#include "object-file.h" #include "object-name.h" #include "object-store.h" #include "config.h" diff --git a/builtin/mktree.c b/builtin/mktree.c index 3c16faa40e3810..0644f951161fc8 100644 --- a/builtin/mktree.c +++ b/builtin/mktree.c @@ -11,6 +11,7 @@ #include "strbuf.h" #include "tree.h" #include "parse-options.h" +#include "object-file.h" #include "object-store-ll.h" static struct treeent { diff --git a/builtin/notes.c b/builtin/notes.c index ff61ec5f2da953..0dbc233752d311 100644 --- a/builtin/notes.c +++ b/builtin/notes.c @@ -14,6 +14,7 @@ #include "gettext.h" #include "hex.h" #include "notes.h" +#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" #include "path.h" diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index b3e2a9d0c6074a..ee51bd76f60be6 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -31,6 +31,7 @@ #include "tmp-objdir.h" #include "oidset.h" #include "packfile.h" +#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" #include "path.h" diff --git a/builtin/tag.c b/builtin/tag.c index 7c173535cb320f..536a01ff3ae2ff 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -17,6 +17,7 @@ #include "gettext.h" #include "hex.h" #include "refs.h" +#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" #include "path.h" diff --git a/builtin/unpack-file.c b/builtin/unpack-file.c index fb5fcbc40a86d6..b19e5cabd03dd0 100644 --- a/builtin/unpack-file.c +++ b/builtin/unpack-file.c @@ -2,6 +2,7 @@ #include "builtin.h" #include "config.h" #include "hex.h" +#include "object-file.h" #include "object-name.h" #include "object-store-ll.h" diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 3bbcaf2de9ba76..4078eab92529cc 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -8,6 +8,7 @@ #include "gettext.h" #include "git-zlib.h" #include "hex.h" +#include "object-file.h" #include "object-store-ll.h" #include "object.h" #include "delta.h" diff --git a/commit.c b/commit.c index 48aeefaad31709..fbf4f8e87fd9c3 100644 --- a/commit.c +++ b/commit.c @@ -29,6 +29,7 @@ #include "tree.h" #include "hook.h" #include "parse.h" +#include "object-file.h" #include "object-file-convert.h" static struct commit_extra_header *read_commit_extra_header_lines(const char *buf, size_t len, const char **); diff --git a/http-push.c b/http-push.c index 1b030d96f48002..806eb67cf1b3b2 100644 --- a/http-push.c +++ b/http-push.c @@ -19,6 +19,7 @@ #include "tree-walk.h" #include "url.h" #include "packfile.h" +#include "object-file.h" #include "object-store-ll.h" #include "commit-reach.h" diff --git a/match-trees.c b/match-trees.c index ef14ceb594c72a..72922d5d64e777 100644 --- a/match-trees.c +++ b/match-trees.c @@ -6,7 +6,8 @@ #include "strbuf.h" #include "tree.h" #include "tree-walk.h" -#include "object-store-ll.h" +#include "object-file.h" +#include "object-store.h" #include "repository.h" static int score_missing(unsigned mode) diff --git a/merge-ort.c b/merge-ort.c index 1554900d984a61..5de3e2fc159980 100644 --- a/merge-ort.c +++ b/merge-ort.c @@ -36,8 +36,9 @@ #include "merge-ll.h" #include "match-trees.h" #include "mem-pool.h" +#include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "path.h" #include "promisor-remote.h" diff --git a/meson.build b/meson.build index a55e800b85b903..f656f9780208e7 100644 --- a/meson.build +++ b/meson.build @@ -355,6 +355,7 @@ libgit_sources = [ 'object-file-convert.c', 'object-file.c', 'object-name.c', + 'object-store.c', 'object.c', 'oid-array.c', 'oidmap.c', diff --git a/notes-cache.c b/notes-cache.c index ecfdf6e43b575c..150241b15e0905 100644 --- a/notes-cache.c +++ b/notes-cache.c @@ -2,7 +2,8 @@ #include "git-compat-util.h" #include "notes-cache.h" -#include "object-store-ll.h" +#include "object-file.h" +#include "object-store.h" #include "pretty.h" #include "repository.h" #include "commit.h" diff --git a/notes.c b/notes.c index ce5a1006a8332f..d9645c4b5dc603 100644 --- a/notes.c +++ b/notes.c @@ -6,8 +6,9 @@ #include "environment.h" #include "hex.h" #include "notes.h" +#include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "utf8.h" #include "strbuf.h" #include "tree-walk.h" diff --git a/object-file.c b/object-file.c index a7868201d09a85..baa828822eae97 100644 --- a/object-file.c +++ b/object-file.c @@ -11,75 +11,26 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" -#include "abspath.h" -#include "config.h" +#include "bulk-checkin.h" #include "convert.h" #include "environment.h" +#include "fsck.h" #include "gettext.h" #include "hex.h" -#include "string-list.h" -#include "lockfile.h" -#include "pack.h" -#include "commit.h" -#include "run-command.h" -#include "refs.h" -#include "bulk-checkin.h" -#include "repository.h" -#include "replace-object.h" -#include "streaming.h" -#include "dir.h" -#include "list.h" -#include "quote.h" -#include "packfile.h" +#include "loose.h" +#include "object-file-convert.h" #include "object-file.h" #include "object-store.h" #include "oidtree.h" +#include "pack.h" +#include "packfile.h" #include "path.h" -#include "promisor-remote.h" #include "setup.h" -#include "submodule.h" -#include "fsck.h" -#include "loose.h" -#include "object-file-convert.h" +#include "streaming.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 -/* - * This is meant to hold a *small* number of objects that you would - * want repo_read_object_file() to be able to return, but yet you do not want - * to write them into the object store (e.g. a browse-only - * application). - */ -static struct cached_object_entry { - struct object_id oid; - struct cached_object { - enum object_type type; - const void *buf; - unsigned long size; - } value; -} *cached_objects; -static int cached_object_nr, cached_object_alloc; - -static const struct cached_object *find_cached_object(const struct object_id *oid) -{ - static const struct cached_object empty_tree = { - .type = OBJ_TREE, - .buf = "", - }; - int i; - const struct cached_object_entry *co = cached_objects; - - for (i = 0; i < cached_object_nr; i++, co++) { - if (oideq(&co->oid, oid)) - return &co->value; - } - if (oideq(oid, the_hash_algo->empty_tree)) - return &empty_tree; - return NULL; -} - - static int get_conv_flags(unsigned flags) { if (flags & HASH_RENORMALIZE) @@ -90,39 +41,6 @@ static int get_conv_flags(unsigned flags) return 0; } -int odb_mkstemp(struct strbuf *temp_filename, const char *pattern) -{ - int fd; - /* - * we let the umask do its job, don't try to be more - * restrictive except to remove write permission. - */ - int mode = 0444; - repo_git_path_replace(the_repository, temp_filename, "objects/%s", pattern); - fd = git_mkstemp_mode(temp_filename->buf, mode); - if (0 <= fd) - return fd; - - /* slow path */ - /* some mkstemp implementations erase temp_filename on failure */ - repo_git_path_replace(the_repository, temp_filename, "objects/%s", pattern); - safe_create_leading_directories(the_repository, temp_filename->buf); - return xmkstemp_mode(temp_filename->buf, mode); -} - -int odb_pack_keep(const char *name) -{ - int fd; - - fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); - if (0 <= fd) - return fd; - - /* slow path */ - safe_create_leading_directories_const(the_repository, name); - return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); -} - static void fill_loose_path(struct strbuf *buf, const struct object_id *oid) { int i; @@ -136,9 +54,9 @@ static void fill_loose_path(struct strbuf *buf, const struct object_id *oid) } } -static const char *odb_loose_path(struct object_directory *odb, - struct strbuf *buf, - const struct object_id *oid) +const char *odb_loose_path(struct object_directory *odb, + struct strbuf *buf, + const struct object_id *oid) { strbuf_reset(buf); strbuf_addstr(buf, odb->path); @@ -147,513 +65,6 @@ static const char *odb_loose_path(struct object_directory *odb, return buf->buf; } -const char *loose_object_path(struct repository *r, struct strbuf *buf, - const struct object_id *oid) -{ - return odb_loose_path(r->objects->odb, buf, oid); -} - -/* - * Return non-zero iff the path is usable as an alternate object database. - */ -static int alt_odb_usable(struct raw_object_store *o, - struct strbuf *path, - const char *normalized_objdir, khiter_t *pos) -{ - int r; - - /* Detect cases where alternate disappeared */ - if (!is_directory(path->buf)) { - error(_("object directory %s does not exist; " - "check .git/objects/info/alternates"), - path->buf); - return 0; - } - - /* - * Prevent the common mistake of listing the same - * thing twice, or object directory itself. - */ - if (!o->odb_by_path) { - khiter_t p; - - o->odb_by_path = kh_init_odb_path_map(); - assert(!o->odb->next); - p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r); - assert(r == 1); /* never used */ - kh_value(o->odb_by_path, p) = o->odb; - } - if (fspatheq(path->buf, normalized_objdir)) - return 0; - *pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r); - /* r: 0 = exists, 1 = never used, 2 = deleted */ - return r == 0 ? 0 : 1; -} - -/* - * Prepare alternate object database registry. - * - * The variable alt_odb_list points at the list of struct - * object_directory. The elements on this list come from - * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT - * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates, - * whose contents is similar to that environment variable but can be - * LF separated. Its base points at a statically allocated buffer that - * contains "/the/directory/corresponding/to/.git/objects/...", while - * its name points just after the slash at the end of ".git/objects/" - * in the example above, and has enough space to hold all hex characters - * of the object ID, an extra slash for the first level indirection, and - * the terminating NUL. - */ -static void read_info_alternates(struct repository *r, - const char *relative_base, - int depth); -static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry, - const char *relative_base, int depth, const char *normalized_objdir) -{ - struct object_directory *ent; - struct strbuf pathbuf = STRBUF_INIT; - struct strbuf tmp = STRBUF_INIT; - khiter_t pos; - int ret = -1; - - if (!is_absolute_path(entry->buf) && relative_base) { - strbuf_realpath(&pathbuf, relative_base, 1); - strbuf_addch(&pathbuf, '/'); - } - strbuf_addbuf(&pathbuf, entry); - - if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { - error(_("unable to normalize alternate object path: %s"), - pathbuf.buf); - goto error; - } - strbuf_swap(&pathbuf, &tmp); - - /* - * The trailing slash after the directory name is given by - * this function at the end. Remove duplicates. - */ - while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') - strbuf_setlen(&pathbuf, pathbuf.len - 1); - - if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) - goto error; - - CALLOC_ARRAY(ent, 1); - /* pathbuf.buf is already in r->objects->odb_by_path */ - ent->path = strbuf_detach(&pathbuf, NULL); - - /* add the alternate entry */ - *r->objects->odb_tail = ent; - r->objects->odb_tail = &(ent->next); - ent->next = NULL; - assert(r->objects->odb_by_path); - kh_value(r->objects->odb_by_path, pos) = ent; - - /* recursively add alternates */ - read_info_alternates(r, ent->path, depth + 1); - ret = 0; - error: - strbuf_release(&tmp); - strbuf_release(&pathbuf); - return ret; -} - -static const char *parse_alt_odb_entry(const char *string, - int sep, - struct strbuf *out) -{ - const char *end; - - strbuf_reset(out); - - if (*string == '#') { - /* comment; consume up to next separator */ - end = strchrnul(string, sep); - } else if (*string == '"' && !unquote_c_style(out, string, &end)) { - /* - * quoted path; unquote_c_style has copied the - * data for us and set "end". Broken quoting (e.g., - * an entry that doesn't end with a quote) falls - * back to the unquoted case below. - */ - } else { - /* normal, unquoted path */ - end = strchrnul(string, sep); - strbuf_add(out, string, end - string); - } - - if (*end) - end++; - return end; -} - -static void link_alt_odb_entries(struct repository *r, const char *alt, - int sep, const char *relative_base, int depth) -{ - struct strbuf objdirbuf = STRBUF_INIT; - struct strbuf entry = STRBUF_INIT; - - if (!alt || !*alt) - return; - - if (depth > 5) { - error(_("%s: ignoring alternate object stores, nesting too deep"), - relative_base); - return; - } - - strbuf_realpath(&objdirbuf, r->objects->odb->path, 1); - - while (*alt) { - alt = parse_alt_odb_entry(alt, sep, &entry); - if (!entry.len) - continue; - link_alt_odb_entry(r, &entry, - relative_base, depth, objdirbuf.buf); - } - strbuf_release(&entry); - strbuf_release(&objdirbuf); -} - -static void read_info_alternates(struct repository *r, - const char *relative_base, - int depth) -{ - char *path; - struct strbuf buf = STRBUF_INIT; - - path = xstrfmt("%s/info/alternates", relative_base); - if (strbuf_read_file(&buf, path, 1024) < 0) { - warn_on_fopen_errors(path); - free(path); - return; - } - - link_alt_odb_entries(r, buf.buf, '\n', relative_base, depth); - strbuf_release(&buf); - free(path); -} - -void add_to_alternates_file(const char *reference) -{ - struct lock_file lock = LOCK_INIT; - char *alts = repo_git_path(the_repository, "objects/info/alternates"); - FILE *in, *out; - int found = 0; - - hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR); - out = fdopen_lock_file(&lock, "w"); - if (!out) - die_errno(_("unable to fdopen alternates lockfile")); - - in = fopen(alts, "r"); - if (in) { - struct strbuf line = STRBUF_INIT; - - while (strbuf_getline(&line, in) != EOF) { - if (!strcmp(reference, line.buf)) { - found = 1; - break; - } - fprintf_or_die(out, "%s\n", line.buf); - } - - strbuf_release(&line); - fclose(in); - } - else if (errno != ENOENT) - die_errno(_("unable to read alternates file")); - - if (found) { - rollback_lock_file(&lock); - } else { - fprintf_or_die(out, "%s\n", reference); - if (commit_lock_file(&lock)) - die_errno(_("unable to move new alternates file into place")); - if (the_repository->objects->loaded_alternates) - link_alt_odb_entries(the_repository, reference, - '\n', NULL, 0); - } - free(alts); -} - -void add_to_alternates_memory(const char *reference) -{ - /* - * Make sure alternates are initialized, or else our entry may be - * overwritten when they are. - */ - prepare_alt_odb(the_repository); - - link_alt_odb_entries(the_repository, reference, - '\n', NULL, 0); -} - -struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy) -{ - struct object_directory *new_odb; - - /* - * Make sure alternates are initialized, or else our entry may be - * overwritten when they are. - */ - prepare_alt_odb(the_repository); - - /* - * Make a new primary odb and link the old primary ODB in as an - * alternate - */ - new_odb = xcalloc(1, sizeof(*new_odb)); - new_odb->path = xstrdup(dir); - - /* - * Disable ref updates while a temporary odb is active, since - * the objects in the database may roll back. - */ - new_odb->disable_ref_updates = 1; - new_odb->will_destroy = will_destroy; - new_odb->next = the_repository->objects->odb; - the_repository->objects->odb = new_odb; - return new_odb->next; -} - -void restore_primary_odb(struct object_directory *restore_odb, const char *old_path) -{ - struct object_directory *cur_odb = the_repository->objects->odb; - - if (strcmp(old_path, cur_odb->path)) - BUG("expected %s as primary object store; found %s", - old_path, cur_odb->path); - - if (cur_odb->next != restore_odb) - BUG("we expect the old primary object store to be the first alternate"); - - the_repository->objects->odb = restore_odb; - free_object_directory(cur_odb); -} - -/* - * Compute the exact path an alternate is at and returns it. In case of - * error NULL is returned and the human readable error is added to `err` - * `path` may be relative and should point to $GIT_DIR. - * `err` must not be null. - */ -char *compute_alternate_path(const char *path, struct strbuf *err) -{ - char *ref_git = NULL; - const char *repo; - int seen_error = 0; - - ref_git = real_pathdup(path, 0); - if (!ref_git) { - seen_error = 1; - strbuf_addf(err, _("path '%s' does not exist"), path); - goto out; - } - - repo = read_gitfile(ref_git); - if (!repo) - repo = read_gitfile(mkpath("%s/.git", ref_git)); - if (repo) { - free(ref_git); - ref_git = xstrdup(repo); - } - - if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) { - char *ref_git_git = mkpathdup("%s/.git", ref_git); - free(ref_git); - ref_git = ref_git_git; - } else if (!is_directory(mkpath("%s/objects", ref_git))) { - struct strbuf sb = STRBUF_INIT; - seen_error = 1; - if (get_common_dir(&sb, ref_git)) { - strbuf_addf(err, - _("reference repository '%s' as a linked " - "checkout is not supported yet."), - path); - goto out; - } - - strbuf_addf(err, _("reference repository '%s' is not a " - "local repository."), path); - goto out; - } - - if (!access(mkpath("%s/shallow", ref_git), F_OK)) { - strbuf_addf(err, _("reference repository '%s' is shallow"), - path); - seen_error = 1; - goto out; - } - - if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) { - strbuf_addf(err, - _("reference repository '%s' is grafted"), - path); - seen_error = 1; - goto out; - } - -out: - if (seen_error) { - FREE_AND_NULL(ref_git); - } - - return ref_git; -} - -struct object_directory *find_odb(struct repository *r, const char *obj_dir) -{ - struct object_directory *odb; - char *obj_dir_real = real_pathdup(obj_dir, 1); - struct strbuf odb_path_real = STRBUF_INIT; - - prepare_alt_odb(r); - for (odb = r->objects->odb; odb; odb = odb->next) { - strbuf_realpath(&odb_path_real, odb->path, 1); - if (!strcmp(obj_dir_real, odb_path_real.buf)) - break; - } - - free(obj_dir_real); - strbuf_release(&odb_path_real); - - if (!odb) - die(_("could not find object directory matching %s"), obj_dir); - return odb; -} - -static void fill_alternate_refs_command(struct child_process *cmd, - const char *repo_path) -{ - const char *value; - - if (!git_config_get_value("core.alternateRefsCommand", &value)) { - cmd->use_shell = 1; - - strvec_push(&cmd->args, value); - strvec_push(&cmd->args, repo_path); - } else { - cmd->git_cmd = 1; - - strvec_pushf(&cmd->args, "--git-dir=%s", repo_path); - strvec_push(&cmd->args, "for-each-ref"); - strvec_push(&cmd->args, "--format=%(objectname)"); - - if (!git_config_get_value("core.alternateRefsPrefixes", &value)) { - strvec_push(&cmd->args, "--"); - strvec_split(&cmd->args, value); - } - } - - strvec_pushv(&cmd->env, (const char **)local_repo_env); - cmd->out = -1; -} - -static void read_alternate_refs(const char *path, - alternate_ref_fn *cb, - void *data) -{ - struct child_process cmd = CHILD_PROCESS_INIT; - struct strbuf line = STRBUF_INIT; - FILE *fh; - - fill_alternate_refs_command(&cmd, path); - - if (start_command(&cmd)) - return; - - fh = xfdopen(cmd.out, "r"); - while (strbuf_getline_lf(&line, fh) != EOF) { - struct object_id oid; - const char *p; - - if (parse_oid_hex(line.buf, &oid, &p) || *p) { - warning(_("invalid line while parsing alternate refs: %s"), - line.buf); - break; - } - - cb(&oid, data); - } - - fclose(fh); - finish_command(&cmd); - strbuf_release(&line); -} - -struct alternate_refs_data { - alternate_ref_fn *fn; - void *data; -}; - -static int refs_from_alternate_cb(struct object_directory *e, - void *data) -{ - struct strbuf path = STRBUF_INIT; - size_t base_len; - struct alternate_refs_data *cb = data; - - if (!strbuf_realpath(&path, e->path, 0)) - goto out; - if (!strbuf_strip_suffix(&path, "/objects")) - goto out; - base_len = path.len; - - /* Is this a git repository with refs? */ - strbuf_addstr(&path, "/refs"); - if (!is_directory(path.buf)) - goto out; - strbuf_setlen(&path, base_len); - - read_alternate_refs(path.buf, cb->fn, cb->data); - -out: - strbuf_release(&path); - return 0; -} - -void for_each_alternate_ref(alternate_ref_fn fn, void *data) -{ - struct alternate_refs_data cb; - cb.fn = fn; - cb.data = data; - foreach_alt_odb(refs_from_alternate_cb, &cb); -} - -int foreach_alt_odb(alt_odb_fn fn, void *cb) -{ - struct object_directory *ent; - int r = 0; - - prepare_alt_odb(the_repository); - for (ent = the_repository->objects->odb->next; ent; ent = ent->next) { - r = fn(ent, cb); - if (r) - break; - } - return r; -} - -void prepare_alt_odb(struct repository *r) -{ - if (r->objects->loaded_alternates) - return; - - link_alt_odb_entries(r, r->objects->alternate_db, PATH_SEP, NULL, 0); - - read_info_alternates(r, r->objects->odb->path, 0); - r->objects->loaded_alternates = 1; -} - -int has_alt_odb(struct repository *r) -{ - prepare_alt_odb(r); - return !!r->objects->odb->next; -} - /* Returns 1 if we have successfully freshened the file, 0 otherwise. */ static int freshen_file(const char *fn) { @@ -1055,9 +466,9 @@ int parse_loose_header(const char *hdr, struct object_info *oi) return 0; } -static int loose_object_info(struct repository *r, - const struct object_id *oid, - struct object_info *oi, int flags) +int loose_object_info(struct repository *r, + const struct object_id *oid, + struct object_info *oi, int flags) { int status = 0; int fd; @@ -1153,345 +564,6 @@ static int loose_object_info(struct repository *r, return status; } -int obj_read_use_lock = 0; -pthread_mutex_t obj_read_mutex; - -void enable_obj_read_lock(void) -{ - if (obj_read_use_lock) - return; - - obj_read_use_lock = 1; - init_recursive_mutex(&obj_read_mutex); -} - -void disable_obj_read_lock(void) -{ - if (!obj_read_use_lock) - return; - - obj_read_use_lock = 0; - pthread_mutex_destroy(&obj_read_mutex); -} - -int fetch_if_missing = 1; - -static int do_oid_object_info_extended(struct repository *r, - const struct object_id *oid, - struct object_info *oi, unsigned flags) -{ - static struct object_info blank_oi = OBJECT_INFO_INIT; - const struct cached_object *co; - struct pack_entry e; - int rtype; - const struct object_id *real = oid; - int already_retried = 0; - - - if (flags & OBJECT_INFO_LOOKUP_REPLACE) - real = lookup_replace_object(r, oid); - - if (is_null_oid(real)) - return -1; - - if (!oi) - oi = &blank_oi; - - co = find_cached_object(real); - if (co) { - if (oi->typep) - *(oi->typep) = co->type; - if (oi->sizep) - *(oi->sizep) = co->size; - if (oi->disk_sizep) - *(oi->disk_sizep) = 0; - if (oi->delta_base_oid) - oidclr(oi->delta_base_oid, the_repository->hash_algo); - if (oi->type_name) - strbuf_addstr(oi->type_name, type_name(co->type)); - if (oi->contentp) - *oi->contentp = xmemdupz(co->buf, co->size); - oi->whence = OI_CACHED; - return 0; - } - - while (1) { - if (find_pack_entry(r, real, &e)) - break; - - /* Most likely it's a loose object. */ - if (!loose_object_info(r, real, oi, flags)) - return 0; - - /* Not a loose object; someone else may have just packed it. */ - if (!(flags & OBJECT_INFO_QUICK)) { - reprepare_packed_git(r); - if (find_pack_entry(r, real, &e)) - break; - } - - /* - * If r is the_repository, this might be an attempt at - * accessing a submodule object as if it were in the_repository - * (having called add_submodule_odb() on that submodule's ODB). - * If any such ODBs exist, register them and try again. - */ - if (r == the_repository && - register_all_submodule_odb_as_alternates()) - /* We added some alternates; retry */ - continue; - - /* Check if it is a missing object */ - if (fetch_if_missing && repo_has_promisor_remote(r) && - !already_retried && - !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { - promisor_remote_get_direct(r, real, 1); - already_retried = 1; - continue; - } - - if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { - const struct packed_git *p; - if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) - die(_("replacement %s not found for %s"), - oid_to_hex(real), oid_to_hex(oid)); - if ((p = has_packed_and_bad(r, real))) - die(_("packed object %s (stored in %s) is corrupt"), - oid_to_hex(real), p->pack_name); - } - return -1; - } - - if (oi == &blank_oi) - /* - * We know that the caller doesn't actually need the - * information below, so return early. - */ - return 0; - rtype = packed_object_info(r, e.p, e.offset, oi); - if (rtype < 0) { - mark_bad_packed_object(e.p, real); - return do_oid_object_info_extended(r, real, oi, 0); - } else if (oi->whence == OI_PACKED) { - oi->u.packed.offset = e.offset; - oi->u.packed.pack = e.p; - oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || - rtype == OBJ_OFS_DELTA); - } - - return 0; -} - -static int oid_object_info_convert(struct repository *r, - const struct object_id *input_oid, - struct object_info *input_oi, unsigned flags) -{ - const struct git_hash_algo *input_algo = &hash_algos[input_oid->algo]; - int do_die = flags & OBJECT_INFO_DIE_IF_CORRUPT; - struct strbuf type_name = STRBUF_INIT; - struct object_id oid, delta_base_oid; - struct object_info new_oi, *oi; - unsigned long size; - void *content; - int ret; - - if (repo_oid_to_algop(r, input_oid, the_hash_algo, &oid)) { - if (do_die) - die(_("missing mapping of %s to %s"), - oid_to_hex(input_oid), the_hash_algo->name); - return -1; - } - - /* Is new_oi needed? */ - oi = input_oi; - if (input_oi && (input_oi->delta_base_oid || input_oi->sizep || - input_oi->contentp)) { - new_oi = *input_oi; - /* Does delta_base_oid need to be converted? */ - if (input_oi->delta_base_oid) - new_oi.delta_base_oid = &delta_base_oid; - /* Will the attributes differ when converted? */ - if (input_oi->sizep || input_oi->contentp) { - new_oi.contentp = &content; - new_oi.sizep = &size; - new_oi.type_name = &type_name; - } - oi = &new_oi; - } - - ret = oid_object_info_extended(r, &oid, oi, flags); - if (ret) - return -1; - if (oi == input_oi) - return ret; - - if (new_oi.contentp) { - struct strbuf outbuf = STRBUF_INIT; - enum object_type type; - - type = type_from_string_gently(type_name.buf, type_name.len, - !do_die); - if (type == -1) - return -1; - if (type != OBJ_BLOB) { - ret = convert_object_file(the_repository, &outbuf, - the_hash_algo, input_algo, - content, size, type, !do_die); - free(content); - if (ret == -1) - return -1; - size = outbuf.len; - content = strbuf_detach(&outbuf, NULL); - } - if (input_oi->sizep) - *input_oi->sizep = size; - if (input_oi->contentp) - *input_oi->contentp = content; - else - free(content); - if (input_oi->type_name) - *input_oi->type_name = type_name; - else - strbuf_release(&type_name); - } - if (new_oi.delta_base_oid == &delta_base_oid) { - if (repo_oid_to_algop(r, &delta_base_oid, input_algo, - input_oi->delta_base_oid)) { - if (do_die) - die(_("missing mapping of %s to %s"), - oid_to_hex(&delta_base_oid), - input_algo->name); - return -1; - } - } - input_oi->whence = new_oi.whence; - input_oi->u = new_oi.u; - return ret; -} - -int oid_object_info_extended(struct repository *r, const struct object_id *oid, - struct object_info *oi, unsigned flags) -{ - int ret; - - if (oid->algo && (hash_algo_by_ptr(r->hash_algo) != oid->algo)) - return oid_object_info_convert(r, oid, oi, flags); - - obj_read_lock(); - ret = do_oid_object_info_extended(r, oid, oi, flags); - obj_read_unlock(); - return ret; -} - - -/* returns enum object_type or negative */ -int oid_object_info(struct repository *r, - const struct object_id *oid, - unsigned long *sizep) -{ - enum object_type type; - struct object_info oi = OBJECT_INFO_INIT; - - oi.typep = &type; - oi.sizep = sizep; - if (oid_object_info_extended(r, oid, &oi, - OBJECT_INFO_LOOKUP_REPLACE) < 0) - return -1; - return type; -} - -int pretend_object_file(void *buf, unsigned long len, enum object_type type, - struct object_id *oid) -{ - struct cached_object_entry *co; - char *co_buf; - - hash_object_file(the_hash_algo, buf, len, type, oid); - if (repo_has_object_file_with_flags(the_repository, oid, OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT) || - find_cached_object(oid)) - return 0; - ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc); - co = &cached_objects[cached_object_nr++]; - co->value.size = len; - co->value.type = type; - co_buf = xmalloc(len); - memcpy(co_buf, buf, len); - co->value.buf = co_buf; - oidcpy(&co->oid, oid); - return 0; -} - -/* - * This function dies on corrupt objects; the callers who want to - * deal with them should arrange to call oid_object_info_extended() and give - * error messages themselves. - */ -void *repo_read_object_file(struct repository *r, - const struct object_id *oid, - enum object_type *type, - unsigned long *size) -{ - struct object_info oi = OBJECT_INFO_INIT; - unsigned flags = OBJECT_INFO_DIE_IF_CORRUPT | OBJECT_INFO_LOOKUP_REPLACE; - void *data; - - oi.typep = type; - oi.sizep = size; - oi.contentp = &data; - if (oid_object_info_extended(r, oid, &oi, flags)) - return NULL; - - return data; -} - -void *read_object_with_reference(struct repository *r, - const struct object_id *oid, - enum object_type required_type, - unsigned long *size, - struct object_id *actual_oid_return) -{ - enum object_type type; - void *buffer; - unsigned long isize; - struct object_id actual_oid; - - oidcpy(&actual_oid, oid); - while (1) { - int ref_length = -1; - const char *ref_type = NULL; - - buffer = repo_read_object_file(r, &actual_oid, &type, &isize); - if (!buffer) - return NULL; - if (type == required_type) { - *size = isize; - if (actual_oid_return) - oidcpy(actual_oid_return, &actual_oid); - return buffer; - } - /* Handle references */ - else if (type == OBJ_COMMIT) - ref_type = "tree "; - else if (type == OBJ_TAG) - ref_type = "object "; - else { - free(buffer); - return NULL; - } - ref_length = strlen(ref_type); - - if (ref_length + the_hash_algo->hexsz > isize || - memcmp(buffer, ref_type, ref_length) || - get_oid_hex((char *) buffer + ref_length, &actual_oid)) { - free(buffer); - return NULL; - } - free(buffer); - /* Now we have the ID of the referred-to object in - * actual_oid. Check again. */ - } -} - static void hash_object_body(const struct git_hash_algo *algo, struct git_hash_ctx *c, const void *buf, unsigned long len, struct object_id *oid, @@ -2154,32 +1226,6 @@ int force_object_loose(const struct object_id *oid, time_t mtime) return ret; } -int has_object(struct repository *r, const struct object_id *oid, - unsigned flags) -{ - int quick = !(flags & HAS_OBJECT_RECHECK_PACKED); - unsigned object_info_flags = OBJECT_INFO_SKIP_FETCH_OBJECT | - (quick ? OBJECT_INFO_QUICK : 0); - - if (!startup_info->have_repository) - return 0; - return oid_object_info_extended(r, oid, NULL, object_info_flags) >= 0; -} - -int repo_has_object_file_with_flags(struct repository *r, - const struct object_id *oid, int flags) -{ - if (!startup_info->have_repository) - return 0; - return oid_object_info_extended(r, oid, NULL, flags) >= 0; -} - -int repo_has_object_file(struct repository *r, - const struct object_id *oid) -{ - return repo_has_object_file_with_flags(r, oid, 0); -} - /* * We can't use the normal fsck_error_function() for index_mem(), * because we don't yet have a valid oid for it to report. Instead, @@ -2407,16 +1453,6 @@ int read_pack_header(int fd, struct pack_header *header) return 0; } -void assert_oid_type(const struct object_id *oid, enum object_type expect) -{ - enum object_type type = oid_object_info(the_repository, oid, NULL); - if (type < 0) - die(_("%s is not a valid object"), oid_to_hex(oid)); - if (type != expect) - die(_("%s is not a valid '%s' object"), oid_to_hex(oid), - type_name(expect)); -} - int for_each_file_in_obj_subdir(unsigned int subdir_nr, struct strbuf *path, each_loose_object_fn obj_cb, diff --git a/object-file.h b/object-file.h index 353d8a85c3347e..78c84d970a9996 100644 --- a/object-file.h +++ b/object-file.h @@ -21,6 +21,29 @@ extern int fetch_if_missing; int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); +struct object_directory; + +const char *odb_loose_path(struct object_directory *odb, + struct strbuf *buf, + const struct object_id *oid); + +/* + * Return true iff an alternate object database has a loose object + * with the specified name. This function does not respect replace + * references. + */ +int has_loose_object_nonlocal(const struct object_id *); + +int has_loose_object(const struct object_id *); + +/** + * format_object_header() is a thin wrapper around s xsnprintf() that + * writes the initial " " part of the loose object + * header. It returns the size that snprintf() returns + 1. + */ +int format_object_header(char *str, size_t size, enum object_type type, + size_t objsize); + /** * unpack_loose_header() initializes the data stream needed to unpack * a loose object header. @@ -61,6 +84,29 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, struct object_info; int parse_loose_header(const char *hdr, struct object_info *oi); +int write_object_file_flags(const void *buf, unsigned long len, + enum object_type type, struct object_id *oid, + struct object_id *comapt_oid_in, unsigned flags); +static inline int write_object_file(const void *buf, unsigned long len, + enum object_type type, struct object_id *oid) +{ + return write_object_file_flags(buf, len, type, oid, NULL, 0); +} + +struct input_stream { + const void *(*read)(struct input_stream *, unsigned long *len); + void *data; + int is_finished; +}; + +int write_object_file_literally(const void *buf, unsigned long len, + const char *type, struct object_id *oid, + unsigned flags); +int stream_loose_object(struct input_stream *in_stream, size_t len, + struct object_id *oid); + +int force_object_loose(const struct object_id *oid, time_t mtime); + /** * With in-core object data in "buf", rehash it to make sure the * object name actually matches "oid" to detect object corruption. @@ -79,6 +125,10 @@ int check_object_signature(struct repository *r, const struct object_id *oid, */ int stream_object_signature(struct repository *r, const struct object_id *oid); +int loose_object_info(struct repository *r, + const struct object_id *oid, + struct object_info *oi, int flags); + enum finalize_object_file_flags { FOF_SKIP_COLLISION_CHECK = 1, }; @@ -90,10 +140,18 @@ int finalize_object_file_flags(const char *tmpfile, const char *filename, /* Helper to check and "touch" a file */ int check_and_freshen_file(const char *fn, int freshen); -void *read_object_with_reference(struct repository *r, - const struct object_id *oid, - enum object_type required_type, - unsigned long *size, - struct object_id *oid_ret); +/* + * Open the loose object at path, check its hash, and return the contents, + * use the "oi" argument to assert things about the object, or e.g. populate its + * type, and size. If the object is a blob, then "contents" may return NULL, + * to allow streaming of large blobs. + * + * Returns 0 on success, negative on error (details may be written to stderr). + */ +int read_loose_object(const char *path, + const struct object_id *expected_oid, + struct object_id *real_oid, + void **contents, + struct object_info *oi); #endif /* OBJECT_FILE_H */ diff --git a/object-store-ll.h b/object-store-ll.h index cd3bd5bd99f78c..8ae80b8a5fad64 100644 --- a/object-store-ll.h +++ b/object-store-ll.h @@ -49,12 +49,6 @@ struct object_directory { char *path; }; -struct input_stream { - const void *(*read)(struct input_stream *, unsigned long *len); - void *data; - int is_finished; -}; - void prepare_alt_odb(struct repository *r); int has_alt_odb(struct repository *r); char *compute_alternate_path(const char *path, struct strbuf *err); @@ -273,21 +267,6 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf, unsigned long len, enum object_type type, struct object_id *oid); -int write_object_file_flags(const void *buf, unsigned long len, - enum object_type type, struct object_id *oid, - struct object_id *comapt_oid_in, unsigned flags); -static inline int write_object_file(const void *buf, unsigned long len, - enum object_type type, struct object_id *oid) -{ - return write_object_file_flags(buf, len, type, oid, NULL, 0); -} - -int write_object_file_literally(const void *buf, unsigned long len, - const char *type, struct object_id *oid, - unsigned flags); -int stream_loose_object(struct input_stream *in_stream, size_t len, - struct object_id *oid); - /* * Add an object file to the in-memory object store, without writing it * to disk. @@ -299,8 +278,6 @@ int stream_loose_object(struct input_stream *in_stream, size_t len, int pretend_object_file(void *, unsigned long, enum object_type, struct object_id *oid); -int force_object_loose(const struct object_id *oid, time_t mtime); - struct object_info { /* Request */ enum object_type *typep; @@ -364,20 +341,6 @@ int oid_object_info_extended(struct repository *r, const struct object_id *, struct object_info *, unsigned flags); -/* - * Open the loose object at path, check its hash, and return the contents, - * use the "oi" argument to assert things about the object, or e.g. populate its - * type, and size. If the object is a blob, then "contents" may return NULL, - * to allow streaming of large blobs. - * - * Returns 0 on success, negative on error (details may be written to stderr). - */ -int read_loose_object(const char *path, - const struct object_id *expected_oid, - struct object_id *real_oid, - void **contents, - struct object_info *oi); - /* Retry packed storage after checking packed and loose storage */ #define HAS_OBJECT_RECHECK_PACKED 1 @@ -405,23 +368,6 @@ int repo_has_object_file(struct repository *r, const struct object_id *oid); int repo_has_object_file_with_flags(struct repository *r, const struct object_id *oid, int flags); -/* - * Return true iff an alternate object database has a loose object - * with the specified name. This function does not respect replace - * references. - */ -int has_loose_object_nonlocal(const struct object_id *); - -int has_loose_object(const struct object_id *); - -/** - * format_object_header() is a thin wrapper around s xsnprintf() that - * writes the initial " " part of the loose object - * header. It returns the size that snprintf() returns + 1. - */ -int format_object_header(char *str, size_t size, enum object_type type, - size_t objsize); - void assert_oid_type(const struct object_id *oid, enum object_type expect); /* @@ -553,4 +499,10 @@ int for_each_object_in_pack(struct packed_git *p, int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, void *data, enum for_each_object_flags flags); +void *read_object_with_reference(struct repository *r, + const struct object_id *oid, + enum object_type required_type, + unsigned long *size, + struct object_id *oid_ret); + #endif /* OBJECT_STORE_LL_H */ diff --git a/object-store.c b/object-store.c new file mode 100644 index 00000000000000..a2004dca15a635 --- /dev/null +++ b/object-store.c @@ -0,0 +1,972 @@ +#define USE_THE_REPOSITORY_VARIABLE + +#include "git-compat-util.h" +#include "abspath.h" +#include "config.h" +#include "environment.h" +#include "gettext.h" +#include "hex.h" +#include "lockfile.h" +#include "object-file-convert.h" +#include "object-file.h" +#include "object-store.h" +#include "packfile.h" +#include "path.h" +#include "promisor-remote.h" +#include "quote.h" +#include "replace-object.h" +#include "run-command.h" +#include "setup.h" +#include "strbuf.h" +#include "strvec.h" +#include "submodule.h" +#include "write-or-die.h" + +/* + * This is meant to hold a *small* number of objects that you would + * want repo_read_object_file() to be able to return, but yet you do not want + * to write them into the object store (e.g. a browse-only + * application). + */ +static struct cached_object_entry { + struct object_id oid; + struct cached_object { + enum object_type type; + const void *buf; + unsigned long size; + } value; +} *cached_objects; +static int cached_object_nr, cached_object_alloc; + +static const struct cached_object *find_cached_object(const struct object_id *oid) +{ + static const struct cached_object empty_tree = { + .type = OBJ_TREE, + .buf = "", + }; + int i; + const struct cached_object_entry *co = cached_objects; + + for (i = 0; i < cached_object_nr; i++, co++) { + if (oideq(&co->oid, oid)) + return &co->value; + } + if (oideq(oid, the_hash_algo->empty_tree)) + return &empty_tree; + return NULL; +} + +int odb_mkstemp(struct strbuf *temp_filename, const char *pattern) +{ + int fd; + /* + * we let the umask do its job, don't try to be more + * restrictive except to remove write permission. + */ + int mode = 0444; + repo_git_path_replace(the_repository, temp_filename, "objects/%s", pattern); + fd = git_mkstemp_mode(temp_filename->buf, mode); + if (0 <= fd) + return fd; + + /* slow path */ + /* some mkstemp implementations erase temp_filename on failure */ + repo_git_path_replace(the_repository, temp_filename, "objects/%s", pattern); + safe_create_leading_directories(the_repository, temp_filename->buf); + return xmkstemp_mode(temp_filename->buf, mode); +} + +int odb_pack_keep(const char *name) +{ + int fd; + + fd = open(name, O_RDWR|O_CREAT|O_EXCL, 0600); + if (0 <= fd) + return fd; + + /* slow path */ + safe_create_leading_directories_const(the_repository, name); + return open(name, O_RDWR|O_CREAT|O_EXCL, 0600); +} + +const char *loose_object_path(struct repository *r, struct strbuf *buf, + const struct object_id *oid) +{ + return odb_loose_path(r->objects->odb, buf, oid); +} + +/* + * Return non-zero iff the path is usable as an alternate object database. + */ +static int alt_odb_usable(struct raw_object_store *o, + struct strbuf *path, + const char *normalized_objdir, khiter_t *pos) +{ + int r; + + /* Detect cases where alternate disappeared */ + if (!is_directory(path->buf)) { + error(_("object directory %s does not exist; " + "check .git/objects/info/alternates"), + path->buf); + return 0; + } + + /* + * Prevent the common mistake of listing the same + * thing twice, or object directory itself. + */ + if (!o->odb_by_path) { + khiter_t p; + + o->odb_by_path = kh_init_odb_path_map(); + assert(!o->odb->next); + p = kh_put_odb_path_map(o->odb_by_path, o->odb->path, &r); + assert(r == 1); /* never used */ + kh_value(o->odb_by_path, p) = o->odb; + } + if (fspatheq(path->buf, normalized_objdir)) + return 0; + *pos = kh_put_odb_path_map(o->odb_by_path, path->buf, &r); + /* r: 0 = exists, 1 = never used, 2 = deleted */ + return r == 0 ? 0 : 1; +} + +/* + * Prepare alternate object database registry. + * + * The variable alt_odb_list points at the list of struct + * object_directory. The elements on this list come from + * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT + * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates, + * whose contents is similar to that environment variable but can be + * LF separated. Its base points at a statically allocated buffer that + * contains "/the/directory/corresponding/to/.git/objects/...", while + * its name points just after the slash at the end of ".git/objects/" + * in the example above, and has enough space to hold all hex characters + * of the object ID, an extra slash for the first level indirection, and + * the terminating NUL. + */ +static void read_info_alternates(struct repository *r, + const char *relative_base, + int depth); +static int link_alt_odb_entry(struct repository *r, const struct strbuf *entry, + const char *relative_base, int depth, const char *normalized_objdir) +{ + struct object_directory *ent; + struct strbuf pathbuf = STRBUF_INIT; + struct strbuf tmp = STRBUF_INIT; + khiter_t pos; + int ret = -1; + + if (!is_absolute_path(entry->buf) && relative_base) { + strbuf_realpath(&pathbuf, relative_base, 1); + strbuf_addch(&pathbuf, '/'); + } + strbuf_addbuf(&pathbuf, entry); + + if (!strbuf_realpath(&tmp, pathbuf.buf, 0)) { + error(_("unable to normalize alternate object path: %s"), + pathbuf.buf); + goto error; + } + strbuf_swap(&pathbuf, &tmp); + + /* + * The trailing slash after the directory name is given by + * this function at the end. Remove duplicates. + */ + while (pathbuf.len && pathbuf.buf[pathbuf.len - 1] == '/') + strbuf_setlen(&pathbuf, pathbuf.len - 1); + + if (!alt_odb_usable(r->objects, &pathbuf, normalized_objdir, &pos)) + goto error; + + CALLOC_ARRAY(ent, 1); + /* pathbuf.buf is already in r->objects->odb_by_path */ + ent->path = strbuf_detach(&pathbuf, NULL); + + /* add the alternate entry */ + *r->objects->odb_tail = ent; + r->objects->odb_tail = &(ent->next); + ent->next = NULL; + assert(r->objects->odb_by_path); + kh_value(r->objects->odb_by_path, pos) = ent; + + /* recursively add alternates */ + read_info_alternates(r, ent->path, depth + 1); + ret = 0; + error: + strbuf_release(&tmp); + strbuf_release(&pathbuf); + return ret; +} + +static const char *parse_alt_odb_entry(const char *string, + int sep, + struct strbuf *out) +{ + const char *end; + + strbuf_reset(out); + + if (*string == '#') { + /* comment; consume up to next separator */ + end = strchrnul(string, sep); + } else if (*string == '"' && !unquote_c_style(out, string, &end)) { + /* + * quoted path; unquote_c_style has copied the + * data for us and set "end". Broken quoting (e.g., + * an entry that doesn't end with a quote) falls + * back to the unquoted case below. + */ + } else { + /* normal, unquoted path */ + end = strchrnul(string, sep); + strbuf_add(out, string, end - string); + } + + if (*end) + end++; + return end; +} + +static void link_alt_odb_entries(struct repository *r, const char *alt, + int sep, const char *relative_base, int depth) +{ + struct strbuf objdirbuf = STRBUF_INIT; + struct strbuf entry = STRBUF_INIT; + + if (!alt || !*alt) + return; + + if (depth > 5) { + error(_("%s: ignoring alternate object stores, nesting too deep"), + relative_base); + return; + } + + strbuf_realpath(&objdirbuf, r->objects->odb->path, 1); + + while (*alt) { + alt = parse_alt_odb_entry(alt, sep, &entry); + if (!entry.len) + continue; + link_alt_odb_entry(r, &entry, + relative_base, depth, objdirbuf.buf); + } + strbuf_release(&entry); + strbuf_release(&objdirbuf); +} + +static void read_info_alternates(struct repository *r, + const char *relative_base, + int depth) +{ + char *path; + struct strbuf buf = STRBUF_INIT; + + path = xstrfmt("%s/info/alternates", relative_base); + if (strbuf_read_file(&buf, path, 1024) < 0) { + warn_on_fopen_errors(path); + free(path); + return; + } + + link_alt_odb_entries(r, buf.buf, '\n', relative_base, depth); + strbuf_release(&buf); + free(path); +} + +void add_to_alternates_file(const char *reference) +{ + struct lock_file lock = LOCK_INIT; + char *alts = repo_git_path(the_repository, "objects/info/alternates"); + FILE *in, *out; + int found = 0; + + hold_lock_file_for_update(&lock, alts, LOCK_DIE_ON_ERROR); + out = fdopen_lock_file(&lock, "w"); + if (!out) + die_errno(_("unable to fdopen alternates lockfile")); + + in = fopen(alts, "r"); + if (in) { + struct strbuf line = STRBUF_INIT; + + while (strbuf_getline(&line, in) != EOF) { + if (!strcmp(reference, line.buf)) { + found = 1; + break; + } + fprintf_or_die(out, "%s\n", line.buf); + } + + strbuf_release(&line); + fclose(in); + } + else if (errno != ENOENT) + die_errno(_("unable to read alternates file")); + + if (found) { + rollback_lock_file(&lock); + } else { + fprintf_or_die(out, "%s\n", reference); + if (commit_lock_file(&lock)) + die_errno(_("unable to move new alternates file into place")); + if (the_repository->objects->loaded_alternates) + link_alt_odb_entries(the_repository, reference, + '\n', NULL, 0); + } + free(alts); +} + +void add_to_alternates_memory(const char *reference) +{ + /* + * Make sure alternates are initialized, or else our entry may be + * overwritten when they are. + */ + prepare_alt_odb(the_repository); + + link_alt_odb_entries(the_repository, reference, + '\n', NULL, 0); +} + +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy) +{ + struct object_directory *new_odb; + + /* + * Make sure alternates are initialized, or else our entry may be + * overwritten when they are. + */ + prepare_alt_odb(the_repository); + + /* + * Make a new primary odb and link the old primary ODB in as an + * alternate + */ + new_odb = xcalloc(1, sizeof(*new_odb)); + new_odb->path = xstrdup(dir); + + /* + * Disable ref updates while a temporary odb is active, since + * the objects in the database may roll back. + */ + new_odb->disable_ref_updates = 1; + new_odb->will_destroy = will_destroy; + new_odb->next = the_repository->objects->odb; + the_repository->objects->odb = new_odb; + return new_odb->next; +} + +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path) +{ + struct object_directory *cur_odb = the_repository->objects->odb; + + if (strcmp(old_path, cur_odb->path)) + BUG("expected %s as primary object store; found %s", + old_path, cur_odb->path); + + if (cur_odb->next != restore_odb) + BUG("we expect the old primary object store to be the first alternate"); + + the_repository->objects->odb = restore_odb; + free_object_directory(cur_odb); +} + +/* + * Compute the exact path an alternate is at and returns it. In case of + * error NULL is returned and the human readable error is added to `err` + * `path` may be relative and should point to $GIT_DIR. + * `err` must not be null. + */ +char *compute_alternate_path(const char *path, struct strbuf *err) +{ + char *ref_git = NULL; + const char *repo; + int seen_error = 0; + + ref_git = real_pathdup(path, 0); + if (!ref_git) { + seen_error = 1; + strbuf_addf(err, _("path '%s' does not exist"), path); + goto out; + } + + repo = read_gitfile(ref_git); + if (!repo) + repo = read_gitfile(mkpath("%s/.git", ref_git)); + if (repo) { + free(ref_git); + ref_git = xstrdup(repo); + } + + if (!repo && is_directory(mkpath("%s/.git/objects", ref_git))) { + char *ref_git_git = mkpathdup("%s/.git", ref_git); + free(ref_git); + ref_git = ref_git_git; + } else if (!is_directory(mkpath("%s/objects", ref_git))) { + struct strbuf sb = STRBUF_INIT; + seen_error = 1; + if (get_common_dir(&sb, ref_git)) { + strbuf_addf(err, + _("reference repository '%s' as a linked " + "checkout is not supported yet."), + path); + goto out; + } + + strbuf_addf(err, _("reference repository '%s' is not a " + "local repository."), path); + goto out; + } + + if (!access(mkpath("%s/shallow", ref_git), F_OK)) { + strbuf_addf(err, _("reference repository '%s' is shallow"), + path); + seen_error = 1; + goto out; + } + + if (!access(mkpath("%s/info/grafts", ref_git), F_OK)) { + strbuf_addf(err, + _("reference repository '%s' is grafted"), + path); + seen_error = 1; + goto out; + } + +out: + if (seen_error) { + FREE_AND_NULL(ref_git); + } + + return ref_git; +} + +struct object_directory *find_odb(struct repository *r, const char *obj_dir) +{ + struct object_directory *odb; + char *obj_dir_real = real_pathdup(obj_dir, 1); + struct strbuf odb_path_real = STRBUF_INIT; + + prepare_alt_odb(r); + for (odb = r->objects->odb; odb; odb = odb->next) { + strbuf_realpath(&odb_path_real, odb->path, 1); + if (!strcmp(obj_dir_real, odb_path_real.buf)) + break; + } + + free(obj_dir_real); + strbuf_release(&odb_path_real); + + if (!odb) + die(_("could not find object directory matching %s"), obj_dir); + return odb; +} + +static void fill_alternate_refs_command(struct child_process *cmd, + const char *repo_path) +{ + const char *value; + + if (!git_config_get_value("core.alternateRefsCommand", &value)) { + cmd->use_shell = 1; + + strvec_push(&cmd->args, value); + strvec_push(&cmd->args, repo_path); + } else { + cmd->git_cmd = 1; + + strvec_pushf(&cmd->args, "--git-dir=%s", repo_path); + strvec_push(&cmd->args, "for-each-ref"); + strvec_push(&cmd->args, "--format=%(objectname)"); + + if (!git_config_get_value("core.alternateRefsPrefixes", &value)) { + strvec_push(&cmd->args, "--"); + strvec_split(&cmd->args, value); + } + } + + strvec_pushv(&cmd->env, (const char **)local_repo_env); + cmd->out = -1; +} + +static void read_alternate_refs(const char *path, + alternate_ref_fn *cb, + void *data) +{ + struct child_process cmd = CHILD_PROCESS_INIT; + struct strbuf line = STRBUF_INIT; + FILE *fh; + + fill_alternate_refs_command(&cmd, path); + + if (start_command(&cmd)) + return; + + fh = xfdopen(cmd.out, "r"); + while (strbuf_getline_lf(&line, fh) != EOF) { + struct object_id oid; + const char *p; + + if (parse_oid_hex(line.buf, &oid, &p) || *p) { + warning(_("invalid line while parsing alternate refs: %s"), + line.buf); + break; + } + + cb(&oid, data); + } + + fclose(fh); + finish_command(&cmd); + strbuf_release(&line); +} + +struct alternate_refs_data { + alternate_ref_fn *fn; + void *data; +}; + +static int refs_from_alternate_cb(struct object_directory *e, + void *data) +{ + struct strbuf path = STRBUF_INIT; + size_t base_len; + struct alternate_refs_data *cb = data; + + if (!strbuf_realpath(&path, e->path, 0)) + goto out; + if (!strbuf_strip_suffix(&path, "/objects")) + goto out; + base_len = path.len; + + /* Is this a git repository with refs? */ + strbuf_addstr(&path, "/refs"); + if (!is_directory(path.buf)) + goto out; + strbuf_setlen(&path, base_len); + + read_alternate_refs(path.buf, cb->fn, cb->data); + +out: + strbuf_release(&path); + return 0; +} + +void for_each_alternate_ref(alternate_ref_fn fn, void *data) +{ + struct alternate_refs_data cb; + cb.fn = fn; + cb.data = data; + foreach_alt_odb(refs_from_alternate_cb, &cb); +} + +int foreach_alt_odb(alt_odb_fn fn, void *cb) +{ + struct object_directory *ent; + int r = 0; + + prepare_alt_odb(the_repository); + for (ent = the_repository->objects->odb->next; ent; ent = ent->next) { + r = fn(ent, cb); + if (r) + break; + } + return r; +} + +void prepare_alt_odb(struct repository *r) +{ + if (r->objects->loaded_alternates) + return; + + link_alt_odb_entries(r, r->objects->alternate_db, PATH_SEP, NULL, 0); + + read_info_alternates(r, r->objects->odb->path, 0); + r->objects->loaded_alternates = 1; +} + +int has_alt_odb(struct repository *r) +{ + prepare_alt_odb(r); + return !!r->objects->odb->next; +} + +int obj_read_use_lock = 0; +pthread_mutex_t obj_read_mutex; + +void enable_obj_read_lock(void) +{ + if (obj_read_use_lock) + return; + + obj_read_use_lock = 1; + init_recursive_mutex(&obj_read_mutex); +} + +void disable_obj_read_lock(void) +{ + if (!obj_read_use_lock) + return; + + obj_read_use_lock = 0; + pthread_mutex_destroy(&obj_read_mutex); +} + +int fetch_if_missing = 1; + +static int do_oid_object_info_extended(struct repository *r, + const struct object_id *oid, + struct object_info *oi, unsigned flags) +{ + static struct object_info blank_oi = OBJECT_INFO_INIT; + const struct cached_object *co; + struct pack_entry e; + int rtype; + const struct object_id *real = oid; + int already_retried = 0; + + + if (flags & OBJECT_INFO_LOOKUP_REPLACE) + real = lookup_replace_object(r, oid); + + if (is_null_oid(real)) + return -1; + + if (!oi) + oi = &blank_oi; + + co = find_cached_object(real); + if (co) { + if (oi->typep) + *(oi->typep) = co->type; + if (oi->sizep) + *(oi->sizep) = co->size; + if (oi->disk_sizep) + *(oi->disk_sizep) = 0; + if (oi->delta_base_oid) + oidclr(oi->delta_base_oid, the_repository->hash_algo); + if (oi->type_name) + strbuf_addstr(oi->type_name, type_name(co->type)); + if (oi->contentp) + *oi->contentp = xmemdupz(co->buf, co->size); + oi->whence = OI_CACHED; + return 0; + } + + while (1) { + if (find_pack_entry(r, real, &e)) + break; + + /* Most likely it's a loose object. */ + if (!loose_object_info(r, real, oi, flags)) + return 0; + + /* Not a loose object; someone else may have just packed it. */ + if (!(flags & OBJECT_INFO_QUICK)) { + reprepare_packed_git(r); + if (find_pack_entry(r, real, &e)) + break; + } + + /* + * If r is the_repository, this might be an attempt at + * accessing a submodule object as if it were in the_repository + * (having called add_submodule_odb() on that submodule's ODB). + * If any such ODBs exist, register them and try again. + */ + if (r == the_repository && + register_all_submodule_odb_as_alternates()) + /* We added some alternates; retry */ + continue; + + /* Check if it is a missing object */ + if (fetch_if_missing && repo_has_promisor_remote(r) && + !already_retried && + !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) { + promisor_remote_get_direct(r, real, 1); + already_retried = 1; + continue; + } + + if (flags & OBJECT_INFO_DIE_IF_CORRUPT) { + const struct packed_git *p; + if ((flags & OBJECT_INFO_LOOKUP_REPLACE) && !oideq(real, oid)) + die(_("replacement %s not found for %s"), + oid_to_hex(real), oid_to_hex(oid)); + if ((p = has_packed_and_bad(r, real))) + die(_("packed object %s (stored in %s) is corrupt"), + oid_to_hex(real), p->pack_name); + } + return -1; + } + + if (oi == &blank_oi) + /* + * We know that the caller doesn't actually need the + * information below, so return early. + */ + return 0; + rtype = packed_object_info(r, e.p, e.offset, oi); + if (rtype < 0) { + mark_bad_packed_object(e.p, real); + return do_oid_object_info_extended(r, real, oi, 0); + } else if (oi->whence == OI_PACKED) { + oi->u.packed.offset = e.offset; + oi->u.packed.pack = e.p; + oi->u.packed.is_delta = (rtype == OBJ_REF_DELTA || + rtype == OBJ_OFS_DELTA); + } + + return 0; +} + +static int oid_object_info_convert(struct repository *r, + const struct object_id *input_oid, + struct object_info *input_oi, unsigned flags) +{ + const struct git_hash_algo *input_algo = &hash_algos[input_oid->algo]; + int do_die = flags & OBJECT_INFO_DIE_IF_CORRUPT; + struct strbuf type_name = STRBUF_INIT; + struct object_id oid, delta_base_oid; + struct object_info new_oi, *oi; + unsigned long size; + void *content; + int ret; + + if (repo_oid_to_algop(r, input_oid, the_hash_algo, &oid)) { + if (do_die) + die(_("missing mapping of %s to %s"), + oid_to_hex(input_oid), the_hash_algo->name); + return -1; + } + + /* Is new_oi needed? */ + oi = input_oi; + if (input_oi && (input_oi->delta_base_oid || input_oi->sizep || + input_oi->contentp)) { + new_oi = *input_oi; + /* Does delta_base_oid need to be converted? */ + if (input_oi->delta_base_oid) + new_oi.delta_base_oid = &delta_base_oid; + /* Will the attributes differ when converted? */ + if (input_oi->sizep || input_oi->contentp) { + new_oi.contentp = &content; + new_oi.sizep = &size; + new_oi.type_name = &type_name; + } + oi = &new_oi; + } + + ret = oid_object_info_extended(r, &oid, oi, flags); + if (ret) + return -1; + if (oi == input_oi) + return ret; + + if (new_oi.contentp) { + struct strbuf outbuf = STRBUF_INIT; + enum object_type type; + + type = type_from_string_gently(type_name.buf, type_name.len, + !do_die); + if (type == -1) + return -1; + if (type != OBJ_BLOB) { + ret = convert_object_file(the_repository, &outbuf, + the_hash_algo, input_algo, + content, size, type, !do_die); + free(content); + if (ret == -1) + return -1; + size = outbuf.len; + content = strbuf_detach(&outbuf, NULL); + } + if (input_oi->sizep) + *input_oi->sizep = size; + if (input_oi->contentp) + *input_oi->contentp = content; + else + free(content); + if (input_oi->type_name) + *input_oi->type_name = type_name; + else + strbuf_release(&type_name); + } + if (new_oi.delta_base_oid == &delta_base_oid) { + if (repo_oid_to_algop(r, &delta_base_oid, input_algo, + input_oi->delta_base_oid)) { + if (do_die) + die(_("missing mapping of %s to %s"), + oid_to_hex(&delta_base_oid), + input_algo->name); + return -1; + } + } + input_oi->whence = new_oi.whence; + input_oi->u = new_oi.u; + return ret; +} + +int oid_object_info_extended(struct repository *r, const struct object_id *oid, + struct object_info *oi, unsigned flags) +{ + int ret; + + if (oid->algo && (hash_algo_by_ptr(r->hash_algo) != oid->algo)) + return oid_object_info_convert(r, oid, oi, flags); + + obj_read_lock(); + ret = do_oid_object_info_extended(r, oid, oi, flags); + obj_read_unlock(); + return ret; +} + + +/* returns enum object_type or negative */ +int oid_object_info(struct repository *r, + const struct object_id *oid, + unsigned long *sizep) +{ + enum object_type type; + struct object_info oi = OBJECT_INFO_INIT; + + oi.typep = &type; + oi.sizep = sizep; + if (oid_object_info_extended(r, oid, &oi, + OBJECT_INFO_LOOKUP_REPLACE) < 0) + return -1; + return type; +} + +int pretend_object_file(void *buf, unsigned long len, enum object_type type, + struct object_id *oid) +{ + struct cached_object_entry *co; + char *co_buf; + + hash_object_file(the_hash_algo, buf, len, type, oid); + if (repo_has_object_file_with_flags(the_repository, oid, OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT) || + find_cached_object(oid)) + return 0; + ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc); + co = &cached_objects[cached_object_nr++]; + co->value.size = len; + co->value.type = type; + co_buf = xmalloc(len); + memcpy(co_buf, buf, len); + co->value.buf = co_buf; + oidcpy(&co->oid, oid); + return 0; +} + +/* + * This function dies on corrupt objects; the callers who want to + * deal with them should arrange to call oid_object_info_extended() and give + * error messages themselves. + */ +void *repo_read_object_file(struct repository *r, + const struct object_id *oid, + enum object_type *type, + unsigned long *size) +{ + struct object_info oi = OBJECT_INFO_INIT; + unsigned flags = OBJECT_INFO_DIE_IF_CORRUPT | OBJECT_INFO_LOOKUP_REPLACE; + void *data; + + oi.typep = type; + oi.sizep = size; + oi.contentp = &data; + if (oid_object_info_extended(r, oid, &oi, flags)) + return NULL; + + return data; +} + +void *read_object_with_reference(struct repository *r, + const struct object_id *oid, + enum object_type required_type, + unsigned long *size, + struct object_id *actual_oid_return) +{ + enum object_type type; + void *buffer; + unsigned long isize; + struct object_id actual_oid; + + oidcpy(&actual_oid, oid); + while (1) { + int ref_length = -1; + const char *ref_type = NULL; + + buffer = repo_read_object_file(r, &actual_oid, &type, &isize); + if (!buffer) + return NULL; + if (type == required_type) { + *size = isize; + if (actual_oid_return) + oidcpy(actual_oid_return, &actual_oid); + return buffer; + } + /* Handle references */ + else if (type == OBJ_COMMIT) + ref_type = "tree "; + else if (type == OBJ_TAG) + ref_type = "object "; + else { + free(buffer); + return NULL; + } + ref_length = strlen(ref_type); + + if (ref_length + the_hash_algo->hexsz > isize || + memcmp(buffer, ref_type, ref_length) || + get_oid_hex((char *) buffer + ref_length, &actual_oid)) { + free(buffer); + return NULL; + } + free(buffer); + /* Now we have the ID of the referred-to object in + * actual_oid. Check again. */ + } +} + +int has_object(struct repository *r, const struct object_id *oid, + unsigned flags) +{ + int quick = !(flags & HAS_OBJECT_RECHECK_PACKED); + unsigned object_info_flags = OBJECT_INFO_SKIP_FETCH_OBJECT | + (quick ? OBJECT_INFO_QUICK : 0); + + if (!startup_info->have_repository) + return 0; + return oid_object_info_extended(r, oid, NULL, object_info_flags) >= 0; +} + +int repo_has_object_file_with_flags(struct repository *r, + const struct object_id *oid, int flags) +{ + if (!startup_info->have_repository) + return 0; + return oid_object_info_extended(r, oid, NULL, flags) >= 0; +} + +int repo_has_object_file(struct repository *r, + const struct object_id *oid) +{ + return repo_has_object_file_with_flags(r, oid, 0); +} + +void assert_oid_type(const struct object_id *oid, enum object_type expect) +{ + enum object_type type = oid_object_info(the_repository, oid, NULL); + if (type < 0) + die(_("%s is not a valid object"), oid_to_hex(oid)); + if (type != expect) + die(_("%s is not a valid '%s' object"), oid_to_hex(oid), + type_name(expect)); +} From 70c0f9db4e00586e4df5cca24fe7ce05848ee59c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:19 +0200 Subject: [PATCH 06/36] object-file: split up concerns of `HASH_*` flags The functions `hash_object_file()`, `write_object_file()` and `index_fd()` reuse the same set of flags to alter their behaviour. This not only adds confusion, but given that every function only supports a subset of the flags it becomes very hard to see which flags can be passed to what function. Last but not least, this entangles the implementation of all three function families. Split up concerns by creating separate flags for each of the function families. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/hash-object.c | 23 +++++++++++++++++------ builtin/replace.c | 2 +- builtin/update-index.c | 2 +- bulk-checkin.c | 6 +++--- cache-tree.c | 2 +- notes-merge.c | 2 +- object-file.c | 18 +++++++++--------- object-file.h | 25 +++++++++++++++++++++---- read-cache.c | 4 ++-- 9 files changed, 56 insertions(+), 28 deletions(-) diff --git a/builtin/hash-object.c b/builtin/hash-object.c index a25f0403f444af..e7c0d6afdef539 100644 --- a/builtin/hash-object.c +++ b/builtin/hash-object.c @@ -19,6 +19,11 @@ #include "strbuf.h" #include "write-or-die.h" +enum { + HASH_OBJECT_CHECK = (1 << 0), + HASH_OBJECT_WRITE = (1 << 1), +}; + /* * This is to create corrupt objects for debugging and as such it * needs to bypass the data conversion performed by, and the type @@ -33,7 +38,7 @@ static int hash_literally(struct object_id *oid, int fd, const char *type, unsig ret = -1; else ret = write_object_file_literally(buf.buf, buf.len, type, oid, - flags); + (flags & HASH_OBJECT_WRITE) ? WRITE_OBJECT_FILE_PERSIST : 0); close(fd); strbuf_release(&buf); return ret; @@ -42,15 +47,21 @@ static int hash_literally(struct object_id *oid, int fd, const char *type, unsig static void hash_fd(int fd, const char *type, const char *path, unsigned flags, int literally) { + unsigned int index_flags = 0; struct stat st; struct object_id oid; + if (flags & HASH_OBJECT_WRITE) + index_flags |= INDEX_WRITE_OBJECT; + if (flags & HASH_OBJECT_CHECK) + index_flags |= INDEX_FORMAT_CHECK; + if (fstat(fd, &st) < 0 || (literally ? hash_literally(&oid, fd, type, flags) : index_fd(the_repository->index, &oid, fd, &st, - type_from_string(type), path, flags))) - die((flags & HASH_WRITE_OBJECT) + type_from_string(type), path, index_flags))) + die((flags & HASH_OBJECT_WRITE) ? "Unable to add %s to database" : "Unable to hash %s", path); printf("%s\n", oid_to_hex(&oid)); @@ -102,13 +113,13 @@ int cmd_hash_object(int argc, int no_filters = 0; int literally = 0; int nongit = 0; - unsigned flags = HASH_FORMAT_CHECK; + unsigned flags = HASH_OBJECT_CHECK; const char *vpath = NULL; char *vpath_free = NULL; const struct option hash_object_options[] = { OPT_STRING('t', NULL, &type, N_("type"), N_("object type")), OPT_BIT('w', NULL, &flags, N_("write the object into the object database"), - HASH_WRITE_OBJECT), + HASH_OBJECT_WRITE), OPT_COUNTUP( 0 , "stdin", &hashstdin, N_("read the object from stdin")), OPT_BOOL( 0 , "stdin-paths", &stdin_paths, N_("read file names from stdin")), OPT_BOOL( 0 , "no-filters", &no_filters, N_("store file as is without filters")), @@ -122,7 +133,7 @@ int cmd_hash_object(int argc, argc = parse_options(argc, argv, prefix, hash_object_options, hash_object_usage, 0); - if (flags & HASH_WRITE_OBJECT) + if (flags & HASH_OBJECT_WRITE) prefix = setup_git_directory(); else prefix = setup_git_directory_gently(&nongit); diff --git a/builtin/replace.c b/builtin/replace.c index 15ec0922ce14d8..2b4fc9a68b3819 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -305,7 +305,7 @@ static int import_object(struct object_id *oid, enum object_type type, strbuf_release(&result); } else { struct stat st; - int flags = HASH_FORMAT_CHECK | HASH_WRITE_OBJECT; + int flags = INDEX_FORMAT_CHECK | INDEX_WRITE_OBJECT; if (fstat(fd, &st) < 0) { error_errno(_("unable to fstat %s"), filename); diff --git a/builtin/update-index.c b/builtin/update-index.c index b2f6b1a3fbb6cd..f0cf964294d15d 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -304,7 +304,7 @@ static int add_one_path(const struct cache_entry *old, const char *path, int len ce->ce_mode = ce_mode_from_stat(old, st->st_mode); if (index_path(the_repository->index, &ce->oid, path, st, - info_only ? 0 : HASH_WRITE_OBJECT)) { + info_only ? 0 : INDEX_WRITE_OBJECT)) { discard_cache_entry(ce); return -1; } diff --git a/bulk-checkin.c b/bulk-checkin.c index 23ac00ea0a6b41..309201a76a62ae 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -171,7 +171,7 @@ static int stream_blob_to_pack(struct bulk_checkin_packfile *state, unsigned char obuf[16384]; unsigned hdrlen; int status = Z_OK; - int write_object = (flags & HASH_WRITE_OBJECT); + int write_object = (flags & INDEX_WRITE_OBJECT); off_t offset = 0; git_deflate_init(&s, pack_compression_level); @@ -241,7 +241,7 @@ static int stream_blob_to_pack(struct bulk_checkin_packfile *state, static void prepare_to_stream(struct bulk_checkin_packfile *state, unsigned flags) { - if (!(flags & HASH_WRITE_OBJECT) || state->f) + if (!(flags & INDEX_WRITE_OBJECT) || state->f) return; state->f = create_tmp_packfile(the_repository, &state->pack_tmp_name); @@ -275,7 +275,7 @@ static int deflate_blob_to_pack(struct bulk_checkin_packfile *state, git_hash_update(&ctx, obuf, header_len); /* Note: idx is non-NULL when we are writing */ - if ((flags & HASH_WRITE_OBJECT) != 0) { + if ((flags & INDEX_WRITE_OBJECT) != 0) { CALLOC_ARRAY(idx, 1); prepare_to_stream(state, flags); diff --git a/cache-tree.c b/cache-tree.c index bcbcad3d61a09c..4c8167ea927c54 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -452,7 +452,7 @@ static int update_one(struct cache_tree *it, OBJ_TREE, &it->oid); } else if (write_object_file_flags(buffer.buf, buffer.len, OBJ_TREE, &it->oid, NULL, flags & WRITE_TREE_SILENT - ? HASH_SILENT : 0)) { + ? WRITE_OBJECT_FILE_SILENT : 0)) { strbuf_release(&buffer); return -1; } diff --git a/notes-merge.c b/notes-merge.c index fce45043655edb..520b92942cd0e5 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -729,7 +729,7 @@ int notes_merge_commit(struct notes_merge_options *o, /* write file as blob, and add to partial_tree */ if (stat(path.buf, &st)) die_errno("Failed to stat '%s'", path.buf); - if (index_path(o->repo->index, &blob_oid, path.buf, &st, HASH_WRITE_OBJECT)) + if (index_path(o->repo->index, &blob_oid, path.buf, &st, INDEX_WRITE_OBJECT)) die("Failed to write blob object from '%s'", path.buf); if (add_note(partial_tree, &obj_oid, &blob_oid, NULL)) die("Failed to add resolved note '%s' to notes tree", diff --git a/object-file.c b/object-file.c index baa828822eae97..2051991f4deb42 100644 --- a/object-file.c +++ b/object-file.c @@ -33,9 +33,9 @@ static int get_conv_flags(unsigned flags) { - if (flags & HASH_RENORMALIZE) + if (flags & INDEX_RENORMALIZE) return CONV_EOL_RENORMALIZE; - else if (flags & HASH_WRITE_OBJECT) + else if (flags & INDEX_WRITE_OBJECT) return global_conv_flags_eol | CONV_WRITE_OBJECT; else return 0; @@ -835,7 +835,7 @@ static int start_loose_object_common(struct strbuf *tmp_file, fd = create_tmpfile(tmp_file, filename); if (fd < 0) { - if (flags & HASH_SILENT) + if (flags & WRITE_OBJECT_FILE_SILENT) return -1; else if (errno == EACCES) return error(_("insufficient permission for adding " @@ -967,7 +967,7 @@ static int write_loose_object(const struct object_id *oid, char *hdr, utb.actime = mtime; utb.modtime = mtime; if (utime(tmp_file.buf, &utb) < 0 && - !(flags & HASH_SILENT)) + !(flags & WRITE_OBJECT_FILE_SILENT)) warning_errno(_("failed utime() on %s"), tmp_file.buf); } @@ -1179,7 +1179,7 @@ int write_object_file_literally(const void *buf, unsigned long len, write_object_file_prepare_literally(the_hash_algo, buf, len, type, oid, header, &hdrlen); - if (!(flags & HASH_WRITE_OBJECT)) + if (!(flags & WRITE_OBJECT_FILE_PERSIST)) goto cleanup; if (freshen_packed_object(oid) || freshen_loose_object(oid)) goto cleanup; @@ -1250,7 +1250,7 @@ static int index_mem(struct index_state *istate, { struct strbuf nbuf = STRBUF_INIT; int ret = 0; - int write_object = flags & HASH_WRITE_OBJECT; + int write_object = flags & INDEX_WRITE_OBJECT; if (!type) type = OBJ_BLOB; @@ -1265,7 +1265,7 @@ static int index_mem(struct index_state *istate, size = nbuf.len; } } - if (flags & HASH_FORMAT_CHECK) { + if (flags & INDEX_FORMAT_CHECK) { struct fsck_options opts = FSCK_OPTIONS_DEFAULT; opts.strict = 1; @@ -1291,7 +1291,7 @@ static int index_stream_convert_blob(struct index_state *istate, unsigned flags) { int ret = 0; - const int write_object = flags & HASH_WRITE_OBJECT; + const int write_object = flags & INDEX_WRITE_OBJECT; struct strbuf sbuf = STRBUF_INIT; assert(path); @@ -1423,7 +1423,7 @@ int index_path(struct index_state *istate, struct object_id *oid, case S_IFLNK: if (strbuf_readlink(&sb, path, st->st_size)) return error_errno("readlink(\"%s\")", path); - if (!(flags & HASH_WRITE_OBJECT)) + if (!(flags & INDEX_WRITE_OBJECT)) hash_object_file(the_hash_algo, sb.buf, sb.len, OBJ_BLOB, oid); else if (write_object_file(sb.buf, sb.len, OBJ_BLOB, oid)) diff --git a/object-file.h b/object-file.h index 78c84d970a9996..c002fbe23451b3 100644 --- a/object-file.h +++ b/object-file.h @@ -14,10 +14,12 @@ struct index_state; */ extern int fetch_if_missing; -#define HASH_WRITE_OBJECT 1 -#define HASH_FORMAT_CHECK 2 -#define HASH_RENORMALIZE 4 -#define HASH_SILENT 8 +enum { + INDEX_WRITE_OBJECT = (1 << 0), + INDEX_FORMAT_CHECK = (1 << 1), + INDEX_RENORMALIZE = (1 << 2), +}; + int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags); int index_path(struct index_state *istate, struct object_id *oid, const char *path, struct stat *st, unsigned flags); @@ -84,6 +86,21 @@ enum unpack_loose_header_result unpack_loose_header(git_zstream *stream, struct object_info; int parse_loose_header(const char *hdr, struct object_info *oi); +enum { + /* + * By default, `write_object_file_literally()` does not actually write + * anything into the object store, but only computes the object ID. + * This flag changes that so that the object will be written as a loose + * object and persisted. + */ + WRITE_OBJECT_FILE_PERSIST = (1 << 0), + + /* + * Do not print an error in case something gose wrong. + */ + WRITE_OBJECT_FILE_SILENT = (1 << 1), +}; + int write_object_file_flags(const void *buf, unsigned long len, enum object_type type, struct object_id *oid, struct object_id *comapt_oid_in, unsigned flags); diff --git a/read-cache.c b/read-cache.c index 2f9e21c897d1f5..23028f43a11615 100644 --- a/read-cache.c +++ b/read-cache.c @@ -706,11 +706,11 @@ int add_to_index(struct index_state *istate, const char *path, struct stat *st, int intent_only = flags & ADD_CACHE_INTENT; int add_option = (ADD_CACHE_OK_TO_ADD|ADD_CACHE_OK_TO_REPLACE| (intent_only ? ADD_CACHE_NEW_ONLY : 0)); - unsigned hash_flags = pretend ? 0 : HASH_WRITE_OBJECT; + unsigned hash_flags = pretend ? 0 : INDEX_WRITE_OBJECT; struct object_id oid; if (flags & ADD_CACHE_RENORMALIZE) - hash_flags |= HASH_RENORMALIZE; + hash_flags |= INDEX_RENORMALIZE; if (!S_ISREG(st_mode) && !S_ISLNK(st_mode) && !S_ISDIR(st_mode)) return error(_("%s: can only add regular files, symbolic links or git-directories"), path); From 8a54ebd5ed724545811edfb4cd938f53bd25d1ae Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:20 +0200 Subject: [PATCH 07/36] object-file: drop `index_blob_stream()` The `index_blob_stream()` function is a mere wrapper around `index_blob_bulk_checkin()`. This has been the case since 568508e7657 (bulk-checkin: replace fast-import based implementation, 2011-10-28), which has moved the implementation from `index_blob_stream()` (which was still called `index_stream()`) into `index_bulk_checkin()` (which has since been renamed to `index_blob_bulk_checkin()`). Remove the redirection by dropping the wrapper. Move the comment to `index_blob_bulk_checkin()` to retain its context. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- bulk-checkin.h | 15 +++++++++++++++ object-file.c | 26 ++------------------------ 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/bulk-checkin.h b/bulk-checkin.h index aa7286a7b3e127..7246ea58dcf348 100644 --- a/bulk-checkin.h +++ b/bulk-checkin.h @@ -9,6 +9,21 @@ void prepare_loose_object_bulk_checkin(void); void fsync_loose_object_bulk_checkin(int fd, const char *filename); +/* + * This creates one packfile per large blob unless bulk-checkin + * machinery is "plugged". + * + * This also bypasses the usual "convert-to-git" dance, and that is on + * purpose. We could write a streaming version of the converting + * functions and insert that before feeding the data to fast-import + * (or equivalent in-core API described above). However, that is + * somewhat complicated, as we do not know the size of the filter + * result, which we need to know beforehand when writing a git object. + * Since the primary motivation for trying to stream from the working + * tree file and to avoid mmaping it in core is to deal with large + * binary blobs, they generally do not want to get any conversion, and + * callers should avoid this code path when filters are requested. + */ int index_blob_bulk_checkin(struct object_id *oid, int fd, size_t size, const char *path, unsigned flags); diff --git a/object-file.c b/object-file.c index 2051991f4deb42..6084d603136393 100644 --- a/object-file.c +++ b/object-file.c @@ -1356,28 +1356,6 @@ static int index_core(struct index_state *istate, return ret; } -/* - * This creates one packfile per large blob unless bulk-checkin - * machinery is "plugged". - * - * This also bypasses the usual "convert-to-git" dance, and that is on - * purpose. We could write a streaming version of the converting - * functions and insert that before feeding the data to fast-import - * (or equivalent in-core API described above). However, that is - * somewhat complicated, as we do not know the size of the filter - * result, which we need to know beforehand when writing a git object. - * Since the primary motivation for trying to stream from the working - * tree file and to avoid mmaping it in core is to deal with large - * binary blobs, they generally do not want to get any conversion, and - * callers should avoid this code path when filters are requested. - */ -static int index_blob_stream(struct object_id *oid, int fd, size_t size, - const char *path, - unsigned flags) -{ - return index_blob_bulk_checkin(oid, fd, size, path, flags); -} - int index_fd(struct index_state *istate, struct object_id *oid, int fd, struct stat *st, enum object_type type, const char *path, unsigned flags) @@ -1398,8 +1376,8 @@ int index_fd(struct index_state *istate, struct object_id *oid, ret = index_core(istate, oid, fd, xsize_t(st->st_size), type, path, flags); else - ret = index_blob_stream(oid, fd, xsize_t(st->st_size), path, - flags); + ret = index_blob_bulk_checkin(oid, fd, xsize_t(st->st_size), path, + flags); close(fd); return ret; } From a36d513eca23303a174083f1c0573dc1a6cd2264 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:21 +0200 Subject: [PATCH 08/36] object: split out functions relating to object store subsystem Split out functions relating to the object store subsystem from "object.c". This helps us to separate concerns. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- object-store-ll.h | 3 --- object-store.c | 66 ++++++++++++++++++++++++++++++++++++++++++++++ object.c | 67 ----------------------------------------------- 3 files changed, 66 insertions(+), 70 deletions(-) diff --git a/object-store-ll.h b/object-store-ll.h index 8ae80b8a5fad64..8bb0f33f9a88a8 100644 --- a/object-store-ll.h +++ b/object-store-ll.h @@ -92,9 +92,6 @@ struct oidtree *odb_loose_cache(struct object_directory *odb, /* Empty the loose object cache for the specified object directory. */ void odb_clear_loose_cache(struct object_directory *odb); -/* Clear and free the specified object directory */ -void free_object_directory(struct object_directory *odb); - struct packed_git { struct hashmap_entry packmap_ent; struct packed_git *next; diff --git a/object-store.c b/object-store.c index a2004dca15a635..896d9ac3509f51 100644 --- a/object-store.c +++ b/object-store.c @@ -2,11 +2,13 @@ #include "git-compat-util.h" #include "abspath.h" +#include "commit-graph.h" #include "config.h" #include "environment.h" #include "gettext.h" #include "hex.h" #include "lockfile.h" +#include "loose.h" #include "object-file-convert.h" #include "object-file.h" #include "object-store.h" @@ -361,6 +363,14 @@ struct object_directory *set_temporary_primary_odb(const char *dir, int will_des return new_odb->next; } +static void free_object_directory(struct object_directory *odb) +{ + free(odb->path); + odb_clear_loose_cache(odb); + loose_object_map_clear(&odb->loose_map); + free(odb); +} + void restore_primary_odb(struct object_directory *restore_odb, const char *old_path) { struct object_directory *cur_odb = the_repository->objects->odb; @@ -970,3 +980,59 @@ void assert_oid_type(const struct object_id *oid, enum object_type expect) die(_("%s is not a valid '%s' object"), oid_to_hex(oid), type_name(expect)); } + +struct raw_object_store *raw_object_store_new(void) +{ + struct raw_object_store *o = xmalloc(sizeof(*o)); + + memset(o, 0, sizeof(*o)); + INIT_LIST_HEAD(&o->packed_git_mru); + hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0); + pthread_mutex_init(&o->replace_mutex, NULL); + return o; +} + +static void free_object_directories(struct raw_object_store *o) +{ + while (o->odb) { + struct object_directory *next; + + next = o->odb->next; + free_object_directory(o->odb); + o->odb = next; + } + kh_destroy_odb_path_map(o->odb_by_path); + o->odb_by_path = NULL; +} + +void raw_object_store_clear(struct raw_object_store *o) +{ + FREE_AND_NULL(o->alternate_db); + + oidmap_free(o->replace_map, 1); + FREE_AND_NULL(o->replace_map); + pthread_mutex_destroy(&o->replace_mutex); + + free_commit_graph(o->commit_graph); + o->commit_graph = NULL; + o->commit_graph_attempted = 0; + + free_object_directories(o); + o->odb_tail = NULL; + o->loaded_alternates = 0; + + INIT_LIST_HEAD(&o->packed_git_mru); + close_object_store(o); + + /* + * `close_object_store()` only closes the packfiles, but doesn't free + * them. We thus have to do this manually. + */ + for (struct packed_git *p = o->packed_git, *next; p; p = next) { + next = p->next; + free(p); + } + o->packed_git = NULL; + + hashmap_clear(&o->pack_map); +} diff --git a/object.c b/object.c index 154525a497234c..ccda798b75f53d 100644 --- a/object.c +++ b/object.c @@ -6,16 +6,13 @@ #include "object.h" #include "replace-object.h" #include "object-file.h" -#include "object-store.h" #include "blob.h" #include "statinfo.h" #include "tree.h" #include "commit.h" #include "tag.h" #include "alloc.h" -#include "packfile.h" #include "commit-graph.h" -#include "loose.h" unsigned int get_max_object_index(const struct repository *repo) { @@ -567,70 +564,6 @@ struct parsed_object_pool *parsed_object_pool_new(struct repository *repo) return o; } -struct raw_object_store *raw_object_store_new(void) -{ - struct raw_object_store *o = xmalloc(sizeof(*o)); - - memset(o, 0, sizeof(*o)); - INIT_LIST_HEAD(&o->packed_git_mru); - hashmap_init(&o->pack_map, pack_map_entry_cmp, NULL, 0); - pthread_mutex_init(&o->replace_mutex, NULL); - return o; -} - -void free_object_directory(struct object_directory *odb) -{ - free(odb->path); - odb_clear_loose_cache(odb); - loose_object_map_clear(&odb->loose_map); - free(odb); -} - -static void free_object_directories(struct raw_object_store *o) -{ - while (o->odb) { - struct object_directory *next; - - next = o->odb->next; - free_object_directory(o->odb); - o->odb = next; - } - kh_destroy_odb_path_map(o->odb_by_path); - o->odb_by_path = NULL; -} - -void raw_object_store_clear(struct raw_object_store *o) -{ - FREE_AND_NULL(o->alternate_db); - - oidmap_free(o->replace_map, 1); - FREE_AND_NULL(o->replace_map); - pthread_mutex_destroy(&o->replace_mutex); - - free_commit_graph(o->commit_graph); - o->commit_graph = NULL; - o->commit_graph_attempted = 0; - - free_object_directories(o); - o->odb_tail = NULL; - o->loaded_alternates = 0; - - INIT_LIST_HEAD(&o->packed_git_mru); - close_object_store(o); - - /* - * `close_object_store()` only closes the packfiles, but doesn't free - * them. We thus have to do this manually. - */ - for (struct packed_git *p = o->packed_git, *next; p; p = next) { - next = p->next; - free(p); - } - o->packed_git = NULL; - - hashmap_clear(&o->pack_map); -} - void parsed_object_pool_reset_commit_grafts(struct parsed_object_pool *o) { for (int i = 0; i < o->grafts_nr; i++) { From 176a65ef09f8943439b5b21b8a3ec560fa47959f Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:22 +0200 Subject: [PATCH 09/36] object-store: remove global array of cached objects Cached objects are virtual objects that can be set up without writing anything into the object store directly, which is used by git-blame(1) to create fake commits for the working tree. These cached objects are stored in a global variable, which is another roadblock for libification of the object subsystem. Refactor the code so that we instead store the array as part of the raw object store. This refactoring raises the question whether virtual objects should really be specific to a single repository (or rather a single object store). Hypothetical usecases might for example span across submodules, and here it may or may not be the right thing to provide virtual objects across submodule boundaries. The only existing usecase is git-blame(1) though, which does not know to blame across submodule boundaries in the first place. As such, storing these objects both globally and per-repository would achieve the same result right now. But arguably, if we learned to blame across submodule boundaries, we would likely want to create separate fare working tree commits for each of the submodules so that the user can learn which worktree a specific uncommitted change belongs to. And even if we would want to create the same fake commit for each of the submodules we could do that when storing separate virtual objects per object store. While this is all rather hypothetical, the takeaway is that handling virtual objects per-object store gives us more flexibility compared to storing them globally. In a hypothetical future where we have achieved full libification one might be able to handle unrelated repositories in a single process, where the state of one repository should not have an impact on the state of another repository. As such, storing these cached objects per object store will enable more usecases and should lead to less surprising outcomes overall. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- blame.c | 2 +- object-store-ll.h | 14 +++++++++++++- object-store.c | 39 +++++++++++++++++++++++---------------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/blame.c b/blame.c index 703dab43e78f8b..b7c5bd692e6268 100644 --- a/blame.c +++ b/blame.c @@ -277,7 +277,7 @@ static struct commit *fake_working_tree_commit(struct repository *r, convert_to_git(r->index, path, buf.buf, buf.len, &buf, 0); origin->file.ptr = buf.buf; origin->file.size = buf.len; - pretend_object_file(buf.buf, buf.len, OBJ_BLOB, &origin->blob_oid); + pretend_object_file(the_repository, buf.buf, buf.len, OBJ_BLOB, &origin->blob_oid); /* * Read the current index, replace the path entry with diff --git a/object-store-ll.h b/object-store-ll.h index 8bb0f33f9a88a8..bb5e8798a1b5e7 100644 --- a/object-store-ll.h +++ b/object-store-ll.h @@ -151,6 +151,8 @@ static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, return strcmp(pg1->pack_name, key ? key : pg2->pack_name); } +struct cached_object_entry; + struct raw_object_store { /* * Set of all object directories; the main directory is first (and @@ -203,6 +205,15 @@ struct raw_object_store { unsigned flags; } kept_pack_cache; + /* + * This is meant to hold a *small* number of objects that you would + * want repo_read_object_file() to be able to return, but yet you do not want + * to write them into the object store (e.g. a browse-only + * application). + */ + struct cached_object_entry *cached_objects; + size_t cached_object_nr, cached_object_alloc; + /* * A map of packfiles to packed_git structs for tracking which * packs have been loaded already. @@ -272,7 +283,8 @@ void hash_object_file(const struct git_hash_algo *algo, const void *buf, * object in persistent storage before writing any other new objects * that reference it. */ -int pretend_object_file(void *, unsigned long, enum object_type, +int pretend_object_file(struct repository *repo, + void *buf, unsigned long len, enum object_type type, struct object_id *oid); struct object_info { diff --git a/object-store.c b/object-store.c index 896d9ac3509f51..0f1dcc113ed7b0 100644 --- a/object-store.c +++ b/object-store.c @@ -30,31 +30,31 @@ * to write them into the object store (e.g. a browse-only * application). */ -static struct cached_object_entry { +struct cached_object_entry { struct object_id oid; struct cached_object { enum object_type type; const void *buf; unsigned long size; } value; -} *cached_objects; -static int cached_object_nr, cached_object_alloc; +}; -static const struct cached_object *find_cached_object(const struct object_id *oid) +static const struct cached_object *find_cached_object(struct raw_object_store *object_store, + const struct object_id *oid) { static const struct cached_object empty_tree = { .type = OBJ_TREE, .buf = "", }; - int i; - const struct cached_object_entry *co = cached_objects; + const struct cached_object_entry *co = object_store->cached_objects; - for (i = 0; i < cached_object_nr; i++, co++) { + for (size_t i = 0; i < object_store->cached_object_nr; i++, co++) if (oideq(&co->oid, oid)) return &co->value; - } - if (oideq(oid, the_hash_algo->empty_tree)) + + if (oid->algo && oideq(oid, hash_algos[oid->algo].empty_tree)) return &empty_tree; + return NULL; } @@ -650,7 +650,7 @@ static int do_oid_object_info_extended(struct repository *r, if (!oi) oi = &blank_oi; - co = find_cached_object(real); + co = find_cached_object(r->objects, real); if (co) { if (oi->typep) *(oi->typep) = co->type; @@ -853,18 +853,21 @@ int oid_object_info(struct repository *r, return type; } -int pretend_object_file(void *buf, unsigned long len, enum object_type type, +int pretend_object_file(struct repository *repo, + void *buf, unsigned long len, enum object_type type, struct object_id *oid) { struct cached_object_entry *co; char *co_buf; - hash_object_file(the_hash_algo, buf, len, type, oid); - if (repo_has_object_file_with_flags(the_repository, oid, OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT) || - find_cached_object(oid)) + hash_object_file(repo->hash_algo, buf, len, type, oid); + if (repo_has_object_file_with_flags(repo, oid, OBJECT_INFO_QUICK | OBJECT_INFO_SKIP_FETCH_OBJECT) || + find_cached_object(repo->objects, oid)) return 0; - ALLOC_GROW(cached_objects, cached_object_nr + 1, cached_object_alloc); - co = &cached_objects[cached_object_nr++]; + + ALLOC_GROW(repo->objects->cached_objects, + repo->objects->cached_object_nr + 1, repo->objects->cached_object_alloc); + co = &repo->objects->cached_objects[repo->objects->cached_object_nr++]; co->value.size = len; co->value.type = type; co_buf = xmalloc(len); @@ -1021,6 +1024,10 @@ void raw_object_store_clear(struct raw_object_store *o) o->odb_tail = NULL; o->loaded_alternates = 0; + for (size_t i = 0; i < o->cached_object_nr; i++) + free((char *) o->cached_objects[i].value.buf); + FREE_AND_NULL(o->cached_objects); + INIT_LIST_HEAD(&o->packed_git_mru); close_object_store(o); From 68cd492a3e662c75dec364986c81e94716d4ac56 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 15 Apr 2025 11:38:23 +0200 Subject: [PATCH 10/36] object-store: merge "object-store-ll.h" and "object-store.h" The "object-store-ll.h" header has been introduced to keep transitive header dependendcies and compile times at bay. Now that we have created a new "object-store.c" file though we can easily move the last remaining additional bit of "object-store.h", the `odb_path_map`, out of the header. Do so. As the "object-store.h" header is now equivalent to its low-level alternative we drop the latter and inline it into the former. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- apply.c | 2 +- archive-tar.c | 2 +- archive-zip.c | 2 +- archive.c | 2 +- attr.c | 2 +- bisect.c | 2 +- blame.c | 2 +- builtin/backfill.c | 2 +- builtin/blame.c | 2 +- builtin/cat-file.c | 2 +- builtin/checkout.c | 2 +- builtin/clone.c | 2 +- builtin/commit-graph.c | 2 +- builtin/commit-tree.c | 2 +- builtin/count-objects.c | 2 +- builtin/describe.c | 2 +- builtin/difftool.c | 2 +- builtin/fast-export.c | 2 +- builtin/fast-import.c | 2 +- builtin/fetch.c | 2 +- builtin/fsck.c | 2 +- builtin/gc.c | 2 +- builtin/grep.c | 2 +- builtin/hash-object.c | 2 +- builtin/index-pack.c | 2 +- builtin/log.c | 2 +- builtin/ls-tree.c | 2 +- builtin/merge-tree.c | 2 +- builtin/mktag.c | 2 +- builtin/mktree.c | 2 +- builtin/multi-pack-index.c | 2 +- builtin/notes.c | 2 +- builtin/pack-objects.c | 2 +- builtin/pack-redundant.c | 2 +- builtin/prune.c | 2 +- builtin/receive-pack.c | 2 +- builtin/remote.c | 2 +- builtin/repack.c | 2 +- builtin/replace.c | 2 +- builtin/rev-list.c | 2 +- builtin/show-ref.c | 2 +- builtin/submodule--helper.c | 2 +- builtin/tag.c | 2 +- builtin/unpack-file.c | 2 +- builtin/unpack-objects.c | 2 +- bulk-checkin.c | 2 +- bundle-uri.c | 2 +- bundle.c | 2 +- cache-tree.c | 2 +- combine-diff.c | 2 +- commit-graph.c | 2 +- commit-graph.h | 2 +- commit.c | 2 +- config.c | 2 +- connected.c | 2 +- convert.c | 2 +- diagnose.c | 2 +- diff.c | 2 +- diffcore-rename.c | 2 +- dir.c | 2 +- entry.c | 2 +- fetch-pack.c | 2 +- fmt-merge-msg.c | 2 +- fsck.c | 2 +- grep.c | 2 +- http-backend.c | 2 +- http-push.c | 2 +- http-walker.c | 2 +- http.c | 2 +- list-objects-filter.c | 2 +- list-objects.c | 2 +- log-tree.c | 2 +- mailmap.c | 2 +- merge-blobs.c | 2 +- merge-recursive.c | 2 +- notes-merge.c | 2 +- object-file.c | 1 + object-name.c | 2 +- object-store-ll.h | 517 ---------------------------------- object-store.c | 5 + object-store.h | 516 ++++++++++++++++++++++++++++++++- oss-fuzz/fuzz-pack-idx.c | 2 +- pack-bitmap-write.c | 2 +- pack-bitmap.c | 2 +- pack-check.c | 2 +- pack-mtimes.c | 2 +- pack-objects.h | 2 +- pack-revindex.c | 2 +- packfile.c | 2 +- path.c | 2 +- promisor-remote.c | 2 +- protocol-caps.c | 2 +- prune-packed.c | 2 +- reachable.c | 2 +- read-cache.c | 2 +- ref-filter.c | 2 +- reflog.c | 2 +- refs.c | 2 +- remote.c | 2 +- replace-object.c | 2 +- replace-object.h | 2 +- repository.c | 2 +- rerere.c | 2 +- revision.c | 2 +- send-pack.c | 2 +- sequencer.c | 2 +- server-info.c | 2 +- shallow.c | 2 +- streaming.c | 2 +- submodule-config.c | 2 +- submodule.c | 2 +- t/helper/test-pack-mtimes.c | 2 +- t/helper/test-partial-clone.c | 2 +- t/helper/test-read-graph.c | 2 +- t/helper/test-read-midx.c | 2 +- t/helper/test-ref-store.c | 2 +- tag.c | 2 +- tmp-objdir.c | 2 +- tree-walk.c | 2 +- tree.c | 2 +- unpack-trees.c | 2 +- upload-pack.c | 2 +- walker.c | 2 +- xdiff-interface.c | 2 +- 124 files changed, 637 insertions(+), 642 deletions(-) delete mode 100644 object-store-ll.h diff --git a/apply.c b/apply.c index f274a3794877dc..2b6f4d0af87c83 100644 --- a/apply.c +++ b/apply.c @@ -14,7 +14,7 @@ #include "abspath.h" #include "base85.h" #include "config.h" -#include "object-store-ll.h" +#include "object-store.h" #include "delta.h" #include "diff.h" #include "dir.h" diff --git a/archive-tar.c b/archive-tar.c index 0edf13fba7568b..282b48196f9810 100644 --- a/archive-tar.c +++ b/archive-tar.c @@ -11,7 +11,7 @@ #include "hex.h" #include "tar.h" #include "archive.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strbuf.h" #include "streaming.h" #include "run-command.h" diff --git a/archive-zip.c b/archive-zip.c index 9f32730181bf62..405da6f3d83004 100644 --- a/archive-zip.c +++ b/archive-zip.c @@ -12,7 +12,7 @@ #include "hex.h" #include "streaming.h" #include "utf8.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strbuf.h" #include "userdiff.h" #include "write-or-die.h" diff --git a/archive.c b/archive.c index c95e398152498b..014c312178c98a 100644 --- a/archive.c +++ b/archive.c @@ -14,7 +14,7 @@ #include "pretty.h" #include "setup.h" #include "refs.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "tree.h" #include "tree-walk.h" diff --git a/attr.c b/attr.c index 0bd2750528fb84..86b6109fc4e328 100644 --- a/attr.c +++ b/attr.c @@ -22,7 +22,7 @@ #include "read-cache-ll.h" #include "refs.h" #include "revision.h" -#include "object-store-ll.h" +#include "object-store.h" #include "setup.h" #include "thread-utils.h" #include "tree-walk.h" diff --git a/bisect.c b/bisect.c index 269a98bf978de6..a327468c75bbcd 100644 --- a/bisect.c +++ b/bisect.c @@ -20,7 +20,7 @@ #include "commit-slab.h" #include "commit-reach.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "dir.h" diff --git a/blame.c b/blame.c index b7c5bd692e6268..57daa45e8996e8 100644 --- a/blame.c +++ b/blame.c @@ -3,7 +3,7 @@ #include "git-compat-util.h" #include "refs.h" -#include "object-store-ll.h" +#include "object-store.h" #include "cache-tree.h" #include "mergesort.h" #include "commit.h" diff --git a/builtin/backfill.c b/builtin/backfill.c index 33e1ea2f84ff6b..aaa104bc91d9fb 100644 --- a/builtin/backfill.c +++ b/builtin/backfill.c @@ -13,7 +13,7 @@ #include "tree.h" #include "tree-walk.h" #include "object.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "oidset.h" #include "promisor-remote.h" diff --git a/builtin/blame.c b/builtin/blame.c index c470654c7ec2c3..4e156bfd19df60 100644 --- a/builtin/blame.c +++ b/builtin/blame.c @@ -28,7 +28,7 @@ #include "line-log.h" #include "progress.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pager.h" #include "blame.h" #include "refs.h" diff --git a/builtin/cat-file.c b/builtin/cat-file.c index b13561cf73b11b..c870fde260adfd 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -22,7 +22,7 @@ #include "packfile.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "promisor-remote.h" #include "mailmap.h" diff --git a/builtin/checkout.c b/builtin/checkout.c index 3e68623838ac03..8136962e2b3518 100644 --- a/builtin/checkout.c +++ b/builtin/checkout.c @@ -20,7 +20,7 @@ #include "merge-recursive.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "parse-options.h" #include "path.h" #include "preload-index.h" diff --git a/builtin/clone.c b/builtin/clone.c index 31f2198c1b3c57..f64be7e1e15d59 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -25,7 +25,7 @@ #include "refs.h" #include "refspec.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "tree.h" #include "tree-walk.h" #include "unpack-trees.h" diff --git a/builtin/commit-graph.c b/builtin/commit-graph.c index 8ca75262c59c48..be06d0a811b20e 100644 --- a/builtin/commit-graph.c +++ b/builtin/commit-graph.c @@ -6,7 +6,7 @@ #include "hex.h" #include "parse-options.h" #include "commit-graph.h" -#include "object-store-ll.h" +#include "object-store.h" #include "progress.h" #include "replace-object.h" #include "strbuf.h" diff --git a/builtin/commit-tree.c b/builtin/commit-tree.c index 38457600a4e422..6f9975e7a88e70 100644 --- a/builtin/commit-tree.c +++ b/builtin/commit-tree.c @@ -9,7 +9,7 @@ #include "gettext.h" #include "hex.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "parse-options.h" diff --git a/builtin/count-objects.c b/builtin/count-objects.c index 1e89148ed742a8..0bb5360b2f262c 100644 --- a/builtin/count-objects.c +++ b/builtin/count-objects.c @@ -12,7 +12,7 @@ #include "parse-options.h" #include "quote.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" static unsigned long garbage; static off_t size_garbage; diff --git a/builtin/describe.c b/builtin/describe.c index 23df333fd04e56..0f87fbceef3704 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -19,7 +19,7 @@ #include "setup.h" #include "strvec.h" #include "run-command.h" -#include "object-store-ll.h" +#include "object-store.h" #include "list-objects.h" #include "commit-slab.h" #include "wildmatch.h" diff --git a/builtin/difftool.c b/builtin/difftool.c index 8292aedaaf06f8..f17a55b3cf9411 100644 --- a/builtin/difftool.c +++ b/builtin/difftool.c @@ -30,7 +30,7 @@ #include "strbuf.h" #include "lockfile.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "dir.h" #include "entry.h" #include "setup.h" diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 170126d41ac95d..afacd228b5d5b5 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -14,7 +14,7 @@ #include "refs.h" #include "refspec.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "object.h" #include "tag.h" diff --git a/builtin/fast-import.c b/builtin/fast-import.c index e4523cc6f1bd39..c1e198f4e34df9 100644 --- a/builtin/fast-import.c +++ b/builtin/fast-import.c @@ -24,7 +24,7 @@ #include "packfile.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "mem-pool.h" #include "commit-reach.h" #include "khash.h" diff --git a/builtin/fetch.c b/builtin/fetch.c index 02af50546908e1..b52a32a5e0d466 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -14,7 +14,7 @@ #include "refs.h" #include "refspec.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oidset.h" #include "oid-array.h" #include "commit.h" diff --git a/builtin/fsck.c b/builtin/fsck.c index 92312b5959174c..6cac28356ce14f 100644 --- a/builtin/fsck.c +++ b/builtin/fsck.c @@ -17,7 +17,7 @@ #include "packfile.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "read-cache-ll.h" #include "replace-object.h" diff --git a/builtin/gc.c b/builtin/gc.c index dae1e545514825..4247e0e7f7c3b4 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -28,7 +28,7 @@ #include "commit.h" #include "commit-graph.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pack.h" #include "pack-objects.h" #include "path.h" diff --git a/builtin/grep.c b/builtin/grep.c index 283d64cab80974..bcfbe5be5bab33 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -26,7 +26,7 @@ #include "submodule-config.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "pager.h" #include "path.h" diff --git a/builtin/hash-object.c b/builtin/hash-object.c index e7c0d6afdef539..cd53fa3bde8dc3 100644 --- a/builtin/hash-object.c +++ b/builtin/hash-object.c @@ -11,7 +11,7 @@ #include "gettext.h" #include "hex.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "blob.h" #include "quote.h" #include "parse-options.h" diff --git a/builtin/index-pack.c b/builtin/index-pack.c index de127c0ff13a28..60a8ee05dbc982 100644 --- a/builtin/index-pack.c +++ b/builtin/index-pack.c @@ -21,7 +21,7 @@ #include "packfile.h" #include "pack-revindex.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "oidset.h" #include "path.h" diff --git a/builtin/log.c b/builtin/log.c index 516c9ec8400b75..b450cd3bde821b 100644 --- a/builtin/log.c +++ b/builtin/log.c @@ -15,7 +15,7 @@ #include "hex.h" #include "refs.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pager.h" #include "color.h" #include "commit.h" diff --git a/builtin/ls-tree.c b/builtin/ls-tree.c index 8542b5d53e435d..8aafc30ca48f7a 100644 --- a/builtin/ls-tree.c +++ b/builtin/ls-tree.c @@ -10,7 +10,7 @@ #include "gettext.h" #include "hex.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "tree.h" #include "path.h" #include "quote.h" diff --git a/builtin/merge-tree.c b/builtin/merge-tree.c index 3ec7127b3a6b3f..4aafa73c61559e 100644 --- a/builtin/merge-tree.c +++ b/builtin/merge-tree.c @@ -10,7 +10,7 @@ #include "commit-reach.h" #include "merge-ort.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "parse-options.h" #include "blob.h" #include "merge-blobs.h" diff --git a/builtin/mktag.c b/builtin/mktag.c index 6e188dce50a4f5..7ac11c46d53f03 100644 --- a/builtin/mktag.c +++ b/builtin/mktag.c @@ -6,7 +6,7 @@ #include "strbuf.h" #include "replace-object.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "fsck.h" #include "config.h" diff --git a/builtin/mktree.c b/builtin/mktree.c index 0644f951161fc8..7ffe6eefd8a93d 100644 --- a/builtin/mktree.c +++ b/builtin/mktree.c @@ -12,7 +12,7 @@ #include "tree.h" #include "parse-options.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" static struct treeent { unsigned mode; diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 2a938466f53aaa..d98410ca6c6057 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -7,7 +7,7 @@ #include "midx.h" #include "strbuf.h" #include "trace2.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "repository.h" diff --git a/builtin/notes.c b/builtin/notes.c index 0dbc233752d311..a3f433ca4c06fa 100644 --- a/builtin/notes.c +++ b/builtin/notes.c @@ -16,7 +16,7 @@ #include "notes.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "pretty.h" diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 163aab547fe7fd..488c80f2cf3f63 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -32,7 +32,7 @@ #include "list.h" #include "packfile.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "dir.h" #include "midx.h" diff --git a/builtin/pack-redundant.c b/builtin/pack-redundant.c index 3febe732f8e1c7..5d1fc781761c79 100644 --- a/builtin/pack-redundant.c +++ b/builtin/pack-redundant.c @@ -13,7 +13,7 @@ #include "hex.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strbuf.h" #define BLKSIZE 512 diff --git a/builtin/prune.c b/builtin/prune.c index 8f52da8bd6608a..e930caa0c0af0e 100644 --- a/builtin/prune.c +++ b/builtin/prune.c @@ -17,7 +17,7 @@ #include "replace-object.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "shallow.h" static const char * const prune_usage[] = { diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index ee51bd76f60be6..be314879e82908 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -33,7 +33,7 @@ #include "packfile.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "protocol.h" #include "commit-reach.h" diff --git a/builtin/remote.c b/builtin/remote.c index 1b7aad88380735..59481b3a82c659 100644 --- a/builtin/remote.c +++ b/builtin/remote.c @@ -14,7 +14,7 @@ #include "rebase.h" #include "refs.h" #include "refspec.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strvec.h" #include "commit-reach.h" #include "progress.h" diff --git a/builtin/repack.c b/builtin/repack.c index f3330ade7b864f..1fd2874324aa8a 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -17,7 +17,7 @@ #include "midx.h" #include "packfile.h" #include "prune-packed.h" -#include "object-store-ll.h" +#include "object-store.h" #include "promisor-remote.h" #include "shallow.h" #include "pack.h" diff --git a/builtin/replace.c b/builtin/replace.c index 2b4fc9a68b3819..48c7c6a2d56d92 100644 --- a/builtin/replace.c +++ b/builtin/replace.c @@ -19,7 +19,7 @@ #include "run-command.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "tag.h" #include "wildmatch.h" diff --git a/builtin/rev-list.c b/builtin/rev-list.c index bb26bee0d4565a..0170d79b62f97b 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -14,7 +14,7 @@ #include "object.h" #include "object-name.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pack-bitmap.h" #include "log-tree.h" #include "graph.h" diff --git a/builtin/show-ref.c b/builtin/show-ref.c index 285cd3e43382cb..f81209f23c3386 100644 --- a/builtin/show-ref.c +++ b/builtin/show-ref.c @@ -5,7 +5,7 @@ #include "hex.h" #include "refs/refs-internal.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "object.h" #include "string-list.h" #include "parse-options.h" diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index cc001d0b4cd884..53da2116ddf576 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -28,7 +28,7 @@ #include "diff.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "advice.h" #include "branch.h" #include "list-objects-filter-options.h" diff --git a/builtin/tag.c b/builtin/tag.c index 536a01ff3ae2ff..e6b372cebf558a 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -19,7 +19,7 @@ #include "refs.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "tag.h" #include "parse-options.h" diff --git a/builtin/unpack-file.c b/builtin/unpack-file.c index b19e5cabd03dd0..e33acfc4ee4764 100644 --- a/builtin/unpack-file.c +++ b/builtin/unpack-file.c @@ -4,7 +4,7 @@ #include "hex.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" static char *create_temp_file(struct object_id *oid) { diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 4078eab92529cc..661be789f1340c 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -9,7 +9,7 @@ #include "git-zlib.h" #include "hex.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "object.h" #include "delta.h" #include "pack.h" diff --git a/bulk-checkin.c b/bulk-checkin.c index 309201a76a62ae..684d49023af73c 100644 --- a/bulk-checkin.c +++ b/bulk-checkin.c @@ -18,7 +18,7 @@ #include "tmp-objdir.h" #include "packfile.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" static int odb_transaction_nesting; diff --git a/bundle-uri.c b/bundle-uri.c index 744257c49c1328..96d2ba726d9909 100644 --- a/bundle-uri.c +++ b/bundle-uri.c @@ -14,7 +14,7 @@ #include "fetch-pack.h" #include "remote.h" #include "trace2.h" -#include "object-store-ll.h" +#include "object-store.h" static struct { enum bundle_list_heuristic heuristic; diff --git a/bundle.c b/bundle.c index d7ad6908433ecf..d661c4ec21418f 100644 --- a/bundle.c +++ b/bundle.c @@ -7,7 +7,7 @@ #include "environment.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "repository.h" #include "object.h" #include "commit.h" diff --git a/cache-tree.c b/cache-tree.c index 4c8167ea927c54..c0e1e9ee1d4af0 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -10,7 +10,7 @@ #include "cache-tree.h" #include "bulk-checkin.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "read-cache-ll.h" #include "replace-object.h" #include "repository.h" diff --git a/combine-diff.c b/combine-diff.c index 553bf59fed6734..dfae9f7995da51 100644 --- a/combine-diff.c +++ b/combine-diff.c @@ -2,7 +2,7 @@ #define DISABLE_SIGN_COMPARE_WARNINGS #include "git-compat-util.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "convert.h" #include "diff.h" diff --git a/commit-graph.c b/commit-graph.c index 9621c454972957..6394752b0b0868 100644 --- a/commit-graph.c +++ b/commit-graph.c @@ -13,7 +13,7 @@ #include "refs.h" #include "hash-lookup.h" #include "commit-graph.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "path.h" #include "alloc.h" diff --git a/commit-graph.h b/commit-graph.h index 6781940195435d..13f662827d49a9 100644 --- a/commit-graph.h +++ b/commit-graph.h @@ -1,7 +1,7 @@ #ifndef COMMIT_GRAPH_H #define COMMIT_GRAPH_H -#include "object-store-ll.h" +#include "object-store.h" #include "oidset.h" #define GIT_TEST_COMMIT_GRAPH "GIT_TEST_COMMIT_GRAPH" diff --git a/commit.c b/commit.c index fbf4f8e87fd9c3..00842678bd2122 100644 --- a/commit.c +++ b/commit.c @@ -9,7 +9,7 @@ #include "hex.h" #include "repository.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "utf8.h" #include "diff.h" #include "revision.h" diff --git a/config.c b/config.c index accb47e2d1880b..b18b5617fcd05d 100644 --- a/config.c +++ b/config.c @@ -31,7 +31,7 @@ #include "hashmap.h" #include "string-list.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pager.h" #include "path.h" #include "utf8.h" diff --git a/connected.c b/connected.c index 3099da84f3397f..4415388bebacdf 100644 --- a/connected.c +++ b/connected.c @@ -3,7 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "run-command.h" #include "sigchain.h" #include "connected.h" diff --git a/convert.c b/convert.c index 9cc0ca20ca0776..8783e17941ff70 100644 --- a/convert.c +++ b/convert.c @@ -8,7 +8,7 @@ #include "copy.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "attr.h" #include "run-command.h" #include "quote.h" diff --git a/diagnose.c b/diagnose.c index bd485effea22ce..b1be74be9831f8 100644 --- a/diagnose.c +++ b/diagnose.c @@ -7,7 +7,7 @@ #include "gettext.h" #include "hex.h" #include "strvec.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "parse-options.h" #include "repository.h" diff --git a/diff.c b/diff.c index 3bcf5028831dc2..91d5b962b57721 100644 --- a/diff.c +++ b/diff.c @@ -23,7 +23,7 @@ #include "color.h" #include "run-command.h" #include "utf8.h" -#include "object-store-ll.h" +#include "object-store.h" #include "userdiff.h" #include "submodule.h" #include "hashmap.h" diff --git a/diffcore-rename.c b/diffcore-rename.c index 5002e896aad8af..787a2cef5f4ed4 100644 --- a/diffcore-rename.c +++ b/diffcore-rename.c @@ -8,7 +8,7 @@ #include "git-compat-util.h" #include "diff.h" #include "diffcore.h" -#include "object-store-ll.h" +#include "object-store.h" #include "hashmap.h" #include "mem-pool.h" #include "oid-array.h" diff --git a/dir.c b/dir.c index 49008739b9bcae..5c4675b4ac40e8 100644 --- a/dir.c +++ b/dir.c @@ -17,7 +17,7 @@ #include "environment.h" #include "gettext.h" #include "name-hash.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "refs.h" #include "repository.h" diff --git a/entry.c b/entry.c index 81b321e53d1b96..f36ec5ad24205a 100644 --- a/entry.c +++ b/entry.c @@ -1,7 +1,7 @@ #define USE_THE_REPOSITORY_VARIABLE #include "git-compat-util.h" -#include "object-store-ll.h" +#include "object-store.h" #include "dir.h" #include "environment.h" #include "gettext.h" diff --git a/fetch-pack.c b/fetch-pack.c index 1ed5e11dd56857..210dc30d50f6d6 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -24,7 +24,7 @@ #include "oid-array.h" #include "oidset.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "connected.h" #include "fetch-negotiator.h" diff --git a/fmt-merge-msg.c b/fmt-merge-msg.c index 5b63c3b088a17a..501b5acdd44c22 100644 --- a/fmt-merge-msg.c +++ b/fmt-merge-msg.c @@ -6,7 +6,7 @@ #include "environment.h" #include "refs.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "diff.h" #include "diff-merges.h" #include "hex.h" diff --git a/fsck.c b/fsck.c index 9fc4c25ffd59ba..8dc8472ceb3781 100644 --- a/fsck.c +++ b/fsck.c @@ -4,7 +4,7 @@ #include "date.h" #include "dir.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "repository.h" #include "object.h" diff --git a/grep.c b/grep.c index 9284b5741f5aca..f8d535182c36d3 100644 --- a/grep.c +++ b/grep.c @@ -5,7 +5,7 @@ #include "gettext.h" #include "grep.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pretty.h" #include "userdiff.h" #include "xdiff-interface.h" diff --git a/http-backend.c b/http-backend.c index 50b2858fad634a..0c575aa88aaf32 100644 --- a/http-backend.c +++ b/http-backend.c @@ -18,7 +18,7 @@ #include "url.h" #include "strvec.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "protocol.h" #include "date.h" #include "write-or-die.h" diff --git a/http-push.c b/http-push.c index 806eb67cf1b3b2..32e37565f4e08f 100644 --- a/http-push.c +++ b/http-push.c @@ -20,7 +20,7 @@ #include "url.h" #include "packfile.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit-reach.h" #ifdef EXPAT_NEEDS_XMLPARSE_H diff --git a/http-walker.c b/http-walker.c index 7918ddc0968dd3..882cae19c2468c 100644 --- a/http-walker.c +++ b/http-walker.c @@ -9,7 +9,7 @@ #include "list.h" #include "transport.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" struct alt_base { char *base; diff --git a/http.c b/http.c index 0c9a872809f872..bc18ff83c4bd43 100644 --- a/http.c +++ b/http.c @@ -19,7 +19,7 @@ #include "packfile.h" #include "string-list.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "tempfile.h" static struct trace_key trace_curl = TRACE_KEY_INIT(CURL); diff --git a/list-objects-filter.c b/list-objects-filter.c index dc598a081bb252..7765761b3c6e2a 100644 --- a/list-objects-filter.c +++ b/list-objects-filter.c @@ -12,7 +12,7 @@ #include "oidmap.h" #include "oidset.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" /* Remember to update object flag allocation in object.h */ /* diff --git a/list-objects.c b/list-objects.c index 943e62e868ffe7..1e5512e1318a2c 100644 --- a/list-objects.c +++ b/list-objects.c @@ -14,7 +14,7 @@ #include "list-objects-filter.h" #include "list-objects-filter-options.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "trace.h" #include "environment.h" diff --git a/log-tree.c b/log-tree.c index 5dd1b63076f20c..a4d4ab59ca0714 100644 --- a/log-tree.c +++ b/log-tree.c @@ -9,7 +9,7 @@ #include "environment.h" #include "hex.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "repository.h" #include "tmp-objdir.h" #include "commit.h" diff --git a/mailmap.c b/mailmap.c index f35d20ed7fd1ef..9e2642a043b1fb 100644 --- a/mailmap.c +++ b/mailmap.c @@ -6,7 +6,7 @@ #include "string-list.h" #include "mailmap.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "setup.h" char *git_mailmap_file; diff --git a/merge-blobs.c b/merge-blobs.c index 0ad0390fea568f..53f36dbc1750e7 100644 --- a/merge-blobs.c +++ b/merge-blobs.c @@ -4,7 +4,7 @@ #include "merge-ll.h" #include "blob.h" #include "merge-blobs.h" -#include "object-store-ll.h" +#include "object-store.h" static int fill_mmfile_blob(mmfile_t *f, struct blob *obj) { diff --git a/merge-recursive.c b/merge-recursive.c index f71490517e1ff8..b852f4676740e4 100644 --- a/merge-recursive.c +++ b/merge-recursive.c @@ -27,7 +27,7 @@ #include "name-hash.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "repository.h" #include "revision.h" diff --git a/notes-merge.c b/notes-merge.c index 520b92942cd0e5..dae8e6a281aab0 100644 --- a/notes-merge.c +++ b/notes-merge.c @@ -8,7 +8,7 @@ #include "refs.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "repository.h" #include "diff.h" diff --git a/object-file.c b/object-file.c index 6084d603136393..450e1cae3464f3 100644 --- a/object-file.c +++ b/object-file.c @@ -13,6 +13,7 @@ #include "git-compat-util.h" #include "bulk-checkin.h" #include "convert.h" +#include "dir.h" #include "environment.h" #include "fsck.h" #include "gettext.h" diff --git a/object-name.c b/object-name.c index 91f731373a10a4..2c751a5352a75d 100644 --- a/object-name.c +++ b/object-name.c @@ -19,7 +19,7 @@ #include "oidtree.h" #include "packfile.h" #include "pretty.h" -#include "object-store-ll.h" +#include "object-store.h" #include "read-cache-ll.h" #include "repo-settings.h" #include "repository.h" diff --git a/object-store-ll.h b/object-store-ll.h deleted file mode 100644 index bb5e8798a1b5e7..00000000000000 --- a/object-store-ll.h +++ /dev/null @@ -1,517 +0,0 @@ -#ifndef OBJECT_STORE_LL_H -#define OBJECT_STORE_LL_H - -#include "hashmap.h" -#include "object.h" -#include "list.h" -#include "thread-utils.h" -#include "oidset.h" - -struct oidmap; -struct oidtree; -struct strbuf; -struct repository; - -struct object_directory { - struct object_directory *next; - - /* - * Used to store the results of readdir(3) calls when we are OK - * sacrificing accuracy due to races for speed. That includes - * object existence with OBJECT_INFO_QUICK, as well as - * our search for unique abbreviated hashes. Don't use it for tasks - * requiring greater accuracy! - * - * Be sure to call odb_load_loose_cache() before using. - */ - uint32_t loose_objects_subdir_seen[8]; /* 256 bits */ - struct oidtree *loose_objects_cache; - - /* Map between object IDs for loose objects. */ - struct loose_object_map *loose_map; - - /* - * This is a temporary object store created by the tmp_objdir - * facility. Disable ref updates since the objects in the store - * might be discarded on rollback. - */ - int disable_ref_updates; - - /* - * This object store is ephemeral, so there is no need to fsync. - */ - int will_destroy; - - /* - * Path to the alternative object store. If this is a relative path, - * it is relative to the current working directory. - */ - char *path; -}; - -void prepare_alt_odb(struct repository *r); -int has_alt_odb(struct repository *r); -char *compute_alternate_path(const char *path, struct strbuf *err); -struct object_directory *find_odb(struct repository *r, const char *obj_dir); -typedef int alt_odb_fn(struct object_directory *, void *); -int foreach_alt_odb(alt_odb_fn, void*); -typedef void alternate_ref_fn(const struct object_id *oid, void *); -void for_each_alternate_ref(alternate_ref_fn, void *); - -/* - * Add the directory to the on-disk alternates file; the new entry will also - * take effect in the current process. - */ -void add_to_alternates_file(const char *dir); - -/* - * Add the directory to the in-memory list of alternates (along with any - * recursive alternates it points to), but do not modify the on-disk alternates - * file. - */ -void add_to_alternates_memory(const char *dir); - -/* - * Replace the current writable object directory with the specified temporary - * object directory; returns the former primary object directory. - */ -struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy); - -/* - * Restore a previous ODB replaced by set_temporary_main_odb. - */ -void restore_primary_odb(struct object_directory *restore_odb, const char *old_path); - -/* - * Populate and return the loose object cache array corresponding to the - * given object ID. - */ -struct oidtree *odb_loose_cache(struct object_directory *odb, - const struct object_id *oid); - -/* Empty the loose object cache for the specified object directory. */ -void odb_clear_loose_cache(struct object_directory *odb); - -struct packed_git { - struct hashmap_entry packmap_ent; - struct packed_git *next; - struct list_head mru; - struct pack_window *windows; - off_t pack_size; - const void *index_data; - size_t index_size; - uint32_t num_objects; - size_t crc_offset; - struct oidset bad_objects; - int index_version; - time_t mtime; - int pack_fd; - int index; /* for builtin/pack-objects.c */ - unsigned pack_local:1, - pack_keep:1, - pack_keep_in_core:1, - freshened:1, - do_not_close:1, - pack_promisor:1, - multi_pack_index:1, - is_cruft:1; - unsigned char hash[GIT_MAX_RAWSZ]; - struct revindex_entry *revindex; - const uint32_t *revindex_data; - const uint32_t *revindex_map; - size_t revindex_size; - /* - * mtimes_map points at the beginning of the memory mapped region of - * this pack's corresponding .mtimes file, and mtimes_size is the size - * of that .mtimes file - */ - const uint32_t *mtimes_map; - size_t mtimes_size; - - /* repo denotes the repository this packfile belongs to */ - struct repository *repo; - - /* something like ".git/objects/pack/xxxxx.pack" */ - char pack_name[FLEX_ARRAY]; /* more */ -}; - -struct multi_pack_index; - -static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, - const struct hashmap_entry *entry, - const struct hashmap_entry *entry2, - const void *keydata) -{ - const char *key = keydata; - const struct packed_git *pg1, *pg2; - - pg1 = container_of(entry, const struct packed_git, packmap_ent); - pg2 = container_of(entry2, const struct packed_git, packmap_ent); - - return strcmp(pg1->pack_name, key ? key : pg2->pack_name); -} - -struct cached_object_entry; - -struct raw_object_store { - /* - * Set of all object directories; the main directory is first (and - * cannot be NULL after initialization). Subsequent directories are - * alternates. - */ - struct object_directory *odb; - struct object_directory **odb_tail; - struct kh_odb_path_map *odb_by_path; - - int loaded_alternates; - - /* - * A list of alternate object directories loaded from the environment; - * this should not generally need to be accessed directly, but will - * populate the "odb" list when prepare_alt_odb() is run. - */ - char *alternate_db; - - /* - * Objects that should be substituted by other objects - * (see git-replace(1)). - */ - struct oidmap *replace_map; - unsigned replace_map_initialized : 1; - pthread_mutex_t replace_mutex; /* protect object replace functions */ - - struct commit_graph *commit_graph; - unsigned commit_graph_attempted : 1; /* if loading has been attempted */ - - /* - * private data - * - * should only be accessed directly by packfile.c and midx.c - */ - struct multi_pack_index *multi_pack_index; - - /* - * private data - * - * should only be accessed directly by packfile.c - */ - - struct packed_git *packed_git; - /* A most-recently-used ordered version of the packed_git list. */ - struct list_head packed_git_mru; - - struct { - struct packed_git **packs; - unsigned flags; - } kept_pack_cache; - - /* - * This is meant to hold a *small* number of objects that you would - * want repo_read_object_file() to be able to return, but yet you do not want - * to write them into the object store (e.g. a browse-only - * application). - */ - struct cached_object_entry *cached_objects; - size_t cached_object_nr, cached_object_alloc; - - /* - * A map of packfiles to packed_git structs for tracking which - * packs have been loaded already. - */ - struct hashmap pack_map; - - /* - * A fast, rough count of the number of objects in the repository. - * These two fields are not meant for direct access. Use - * repo_approximate_object_count() instead. - */ - unsigned long approximate_object_count; - unsigned approximate_object_count_valid : 1; - - /* - * Whether packed_git has already been populated with this repository's - * packs. - */ - unsigned packed_git_initialized : 1; -}; - -struct raw_object_store *raw_object_store_new(void); -void raw_object_store_clear(struct raw_object_store *o); - -/* - * Create a temporary file rooted in the object database directory, or - * die on failure. The filename is taken from "pattern", which should have the - * usual "XXXXXX" trailer, and the resulting filename is written into the - * "template" buffer. Returns the open descriptor. - */ -int odb_mkstemp(struct strbuf *temp_filename, const char *pattern); - -/* - * Create a pack .keep file named "name" (which should generally be the output - * of odb_pack_name). Returns a file descriptor opened for writing, or -1 on - * error. - */ -int odb_pack_keep(const char *name); - -/* - * Put in `buf` the name of the file in the local object database that - * would be used to store a loose object with the specified oid. - */ -const char *loose_object_path(struct repository *r, struct strbuf *buf, - const struct object_id *oid); - -void *map_loose_object(struct repository *r, const struct object_id *oid, - unsigned long *size); - -void *repo_read_object_file(struct repository *r, - const struct object_id *oid, - enum object_type *type, - unsigned long *size); - -/* Read and unpack an object file into memory, write memory to an object file */ -int oid_object_info(struct repository *r, const struct object_id *, unsigned long *); - -void hash_object_file(const struct git_hash_algo *algo, const void *buf, - unsigned long len, enum object_type type, - struct object_id *oid); - -/* - * Add an object file to the in-memory object store, without writing it - * to disk. - * - * Callers are responsible for calling write_object_file to record the - * object in persistent storage before writing any other new objects - * that reference it. - */ -int pretend_object_file(struct repository *repo, - void *buf, unsigned long len, enum object_type type, - struct object_id *oid); - -struct object_info { - /* Request */ - enum object_type *typep; - unsigned long *sizep; - off_t *disk_sizep; - struct object_id *delta_base_oid; - struct strbuf *type_name; - void **contentp; - - /* Response */ - enum { - OI_CACHED, - OI_LOOSE, - OI_PACKED, - OI_DBCACHED - } whence; - union { - /* - * struct { - * ... Nothing to expose in this case - * } cached; - * struct { - * ... Nothing to expose in this case - * } loose; - */ - struct { - struct packed_git *pack; - off_t offset; - unsigned int is_delta; - } packed; - } u; -}; - -/* - * Initializer for a "struct object_info" that wants no items. You may - * also memset() the memory to all-zeroes. - */ -#define OBJECT_INFO_INIT { 0 } - -/* Invoke lookup_replace_object() on the given hash */ -#define OBJECT_INFO_LOOKUP_REPLACE 1 -/* Allow reading from a loose object file of unknown/bogus type */ -#define OBJECT_INFO_ALLOW_UNKNOWN_TYPE 2 -/* Do not retry packed storage after checking packed and loose storage */ -#define OBJECT_INFO_QUICK 8 -/* - * Do not attempt to fetch the object if missing (even if fetch_is_missing is - * nonzero). - */ -#define OBJECT_INFO_SKIP_FETCH_OBJECT 16 -/* - * This is meant for bulk prefetching of missing blobs in a partial - * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK - */ -#define OBJECT_INFO_FOR_PREFETCH (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK) - -/* Die if object corruption (not just an object being missing) was detected. */ -#define OBJECT_INFO_DIE_IF_CORRUPT 32 - -int oid_object_info_extended(struct repository *r, - const struct object_id *, - struct object_info *, unsigned flags); - -/* Retry packed storage after checking packed and loose storage */ -#define HAS_OBJECT_RECHECK_PACKED 1 - -/* - * Returns 1 if the object exists. This function will not lazily fetch objects - * in a partial clone. - */ -int has_object(struct repository *r, const struct object_id *oid, - unsigned flags); - -/* - * These macros and functions are deprecated. If checking existence for an - * object that is likely to be missing and/or whose absence is relatively - * inconsequential (or is consequential but the caller is prepared to handle - * it), use has_object(), which has better defaults (no lazy fetch in a partial - * clone and no rechecking of packed storage). In the unlikely event that a - * caller needs to assert existence of an object that it fully expects to - * exist, and wants to trigger a lazy fetch in a partial clone, use - * oid_object_info_extended() with a NULL struct object_info. - * - * These functions can be removed once all callers have migrated to - * has_object() and/or oid_object_info_extended(). - */ -int repo_has_object_file(struct repository *r, const struct object_id *oid); -int repo_has_object_file_with_flags(struct repository *r, - const struct object_id *oid, int flags); - -void assert_oid_type(const struct object_id *oid, enum object_type expect); - -/* - * Enabling the object read lock allows multiple threads to safely call the - * following functions in parallel: repo_read_object_file(), - * read_object_with_reference(), oid_object_info() and oid_object_info_extended(). - * - * obj_read_lock() and obj_read_unlock() may also be used to protect other - * section which cannot execute in parallel with object reading. Since the used - * lock is a recursive mutex, these sections can even contain calls to object - * reading functions. However, beware that in these cases zlib inflation won't - * be performed in parallel, losing performance. - * - * TODO: oid_object_info_extended()'s call stack has a recursive behavior. If - * any of its callees end up calling it, this recursive call won't benefit from - * parallel inflation. - */ -void enable_obj_read_lock(void); -void disable_obj_read_lock(void); - -extern int obj_read_use_lock; -extern pthread_mutex_t obj_read_mutex; - -static inline void obj_read_lock(void) -{ - if(obj_read_use_lock) - pthread_mutex_lock(&obj_read_mutex); -} - -static inline void obj_read_unlock(void) -{ - if(obj_read_use_lock) - pthread_mutex_unlock(&obj_read_mutex); -} - -/* - * Iterate over the files in the loose-object parts of the object - * directory "path", triggering the following callbacks: - * - * - loose_object is called for each loose object we find. - * - * - loose_cruft is called for any files that do not appear to be - * loose objects. Note that we only look in the loose object - * directories "objects/[0-9a-f]{2}/", so we will not report - * "objects/foobar" as cruft. - * - * - loose_subdir is called for each top-level hashed subdirectory - * of the object directory (e.g., "$OBJDIR/f0"). It is called - * after the objects in the directory are processed. - * - * Any callback that is NULL will be ignored. Callbacks returning non-zero - * will end the iteration. - * - * In the "buf" variant, "path" is a strbuf which will also be used as a - * scratch buffer, but restored to its original contents before - * the function returns. - */ -typedef int each_loose_object_fn(const struct object_id *oid, - const char *path, - void *data); -typedef int each_loose_cruft_fn(const char *basename, - const char *path, - void *data); -typedef int each_loose_subdir_fn(unsigned int nr, - const char *path, - void *data); -int for_each_file_in_obj_subdir(unsigned int subdir_nr, - struct strbuf *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); -int for_each_loose_file_in_objdir(const char *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); -int for_each_loose_file_in_objdir_buf(struct strbuf *path, - each_loose_object_fn obj_cb, - each_loose_cruft_fn cruft_cb, - each_loose_subdir_fn subdir_cb, - void *data); - -/* Flags for for_each_*_object() below. */ -enum for_each_object_flags { - /* Iterate only over local objects, not alternates. */ - FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0), - - /* Only iterate over packs obtained from the promisor remote. */ - FOR_EACH_OBJECT_PROMISOR_ONLY = (1<<1), - - /* - * Visit objects within a pack in packfile order rather than .idx order - */ - FOR_EACH_OBJECT_PACK_ORDER = (1<<2), - - /* Only iterate over packs that are not marked as kept in-core. */ - FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS = (1<<3), - - /* Only iterate over packs that do not have .keep files. */ - FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4), -}; - -/* - * Iterate over all accessible loose objects without respect to - * reachability. By default, this includes both local and alternate objects. - * The order in which objects are visited is unspecified. - * - * Any flags specific to packs are ignored. - */ -int for_each_loose_object(each_loose_object_fn, void *, - enum for_each_object_flags flags); - -/* - * Iterate over all accessible packed objects without respect to reachability. - * By default, this includes both local and alternate packs. - * - * Note that some objects may appear twice if they are found in multiple packs. - * Each pack is visited in an unspecified order. By default, objects within a - * pack are visited in pack-idx order (i.e., sorted by oid). - */ -typedef int each_packed_object_fn(const struct object_id *oid, - struct packed_git *pack, - uint32_t pos, - void *data); -int for_each_object_in_pack(struct packed_git *p, - each_packed_object_fn, void *data, - enum for_each_object_flags flags); -int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, - void *data, enum for_each_object_flags flags); - -void *read_object_with_reference(struct repository *r, - const struct object_id *oid, - enum object_type required_type, - unsigned long *size, - struct object_id *oid_ret); - -#endif /* OBJECT_STORE_LL_H */ diff --git a/object-store.c b/object-store.c index 0f1dcc113ed7b0..6ab50d25d3eb4f 100644 --- a/object-store.c +++ b/object-store.c @@ -4,9 +4,11 @@ #include "abspath.h" #include "commit-graph.h" #include "config.h" +#include "dir.h" #include "environment.h" #include "gettext.h" #include "hex.h" +#include "khash.h" #include "lockfile.h" #include "loose.h" #include "object-file-convert.h" @@ -24,6 +26,9 @@ #include "submodule.h" #include "write-or-die.h" +KHASH_INIT(odb_path_map, const char * /* key: odb_path */, + struct object_directory *, 1, fspathhash, fspatheq) + /* * This is meant to hold a *small* number of objects that you would * want repo_read_object_file() to be able to return, but yet you do not want diff --git a/object-store.h b/object-store.h index 1b3e3d7d0145be..46961dc954257b 100644 --- a/object-store.h +++ b/object-store.h @@ -1,11 +1,517 @@ #ifndef OBJECT_STORE_H #define OBJECT_STORE_H -#include "khash.h" -#include "dir.h" -#include "object-store-ll.h" +#include "hashmap.h" +#include "object.h" +#include "list.h" +#include "oidset.h" +#include "thread-utils.h" -KHASH_INIT(odb_path_map, const char * /* key: odb_path */, - struct object_directory *, 1, fspathhash, fspatheq) +struct oidmap; +struct oidtree; +struct strbuf; +struct repository; + +struct object_directory { + struct object_directory *next; + + /* + * Used to store the results of readdir(3) calls when we are OK + * sacrificing accuracy due to races for speed. That includes + * object existence with OBJECT_INFO_QUICK, as well as + * our search for unique abbreviated hashes. Don't use it for tasks + * requiring greater accuracy! + * + * Be sure to call odb_load_loose_cache() before using. + */ + uint32_t loose_objects_subdir_seen[8]; /* 256 bits */ + struct oidtree *loose_objects_cache; + + /* Map between object IDs for loose objects. */ + struct loose_object_map *loose_map; + + /* + * This is a temporary object store created by the tmp_objdir + * facility. Disable ref updates since the objects in the store + * might be discarded on rollback. + */ + int disable_ref_updates; + + /* + * This object store is ephemeral, so there is no need to fsync. + */ + int will_destroy; + + /* + * Path to the alternative object store. If this is a relative path, + * it is relative to the current working directory. + */ + char *path; +}; + +void prepare_alt_odb(struct repository *r); +int has_alt_odb(struct repository *r); +char *compute_alternate_path(const char *path, struct strbuf *err); +struct object_directory *find_odb(struct repository *r, const char *obj_dir); +typedef int alt_odb_fn(struct object_directory *, void *); +int foreach_alt_odb(alt_odb_fn, void*); +typedef void alternate_ref_fn(const struct object_id *oid, void *); +void for_each_alternate_ref(alternate_ref_fn, void *); + +/* + * Add the directory to the on-disk alternates file; the new entry will also + * take effect in the current process. + */ +void add_to_alternates_file(const char *dir); + +/* + * Add the directory to the in-memory list of alternates (along with any + * recursive alternates it points to), but do not modify the on-disk alternates + * file. + */ +void add_to_alternates_memory(const char *dir); + +/* + * Replace the current writable object directory with the specified temporary + * object directory; returns the former primary object directory. + */ +struct object_directory *set_temporary_primary_odb(const char *dir, int will_destroy); + +/* + * Restore a previous ODB replaced by set_temporary_main_odb. + */ +void restore_primary_odb(struct object_directory *restore_odb, const char *old_path); + +/* + * Populate and return the loose object cache array corresponding to the + * given object ID. + */ +struct oidtree *odb_loose_cache(struct object_directory *odb, + const struct object_id *oid); + +/* Empty the loose object cache for the specified object directory. */ +void odb_clear_loose_cache(struct object_directory *odb); + +struct packed_git { + struct hashmap_entry packmap_ent; + struct packed_git *next; + struct list_head mru; + struct pack_window *windows; + off_t pack_size; + const void *index_data; + size_t index_size; + uint32_t num_objects; + size_t crc_offset; + struct oidset bad_objects; + int index_version; + time_t mtime; + int pack_fd; + int index; /* for builtin/pack-objects.c */ + unsigned pack_local:1, + pack_keep:1, + pack_keep_in_core:1, + freshened:1, + do_not_close:1, + pack_promisor:1, + multi_pack_index:1, + is_cruft:1; + unsigned char hash[GIT_MAX_RAWSZ]; + struct revindex_entry *revindex; + const uint32_t *revindex_data; + const uint32_t *revindex_map; + size_t revindex_size; + /* + * mtimes_map points at the beginning of the memory mapped region of + * this pack's corresponding .mtimes file, and mtimes_size is the size + * of that .mtimes file + */ + const uint32_t *mtimes_map; + size_t mtimes_size; + + /* repo denotes the repository this packfile belongs to */ + struct repository *repo; + + /* something like ".git/objects/pack/xxxxx.pack" */ + char pack_name[FLEX_ARRAY]; /* more */ +}; + +struct multi_pack_index; + +static inline int pack_map_entry_cmp(const void *cmp_data UNUSED, + const struct hashmap_entry *entry, + const struct hashmap_entry *entry2, + const void *keydata) +{ + const char *key = keydata; + const struct packed_git *pg1, *pg2; + + pg1 = container_of(entry, const struct packed_git, packmap_ent); + pg2 = container_of(entry2, const struct packed_git, packmap_ent); + + return strcmp(pg1->pack_name, key ? key : pg2->pack_name); +} + +struct cached_object_entry; + +struct raw_object_store { + /* + * Set of all object directories; the main directory is first (and + * cannot be NULL after initialization). Subsequent directories are + * alternates. + */ + struct object_directory *odb; + struct object_directory **odb_tail; + struct kh_odb_path_map *odb_by_path; + + int loaded_alternates; + + /* + * A list of alternate object directories loaded from the environment; + * this should not generally need to be accessed directly, but will + * populate the "odb" list when prepare_alt_odb() is run. + */ + char *alternate_db; + + /* + * Objects that should be substituted by other objects + * (see git-replace(1)). + */ + struct oidmap *replace_map; + unsigned replace_map_initialized : 1; + pthread_mutex_t replace_mutex; /* protect object replace functions */ + + struct commit_graph *commit_graph; + unsigned commit_graph_attempted : 1; /* if loading has been attempted */ + + /* + * private data + * + * should only be accessed directly by packfile.c and midx.c + */ + struct multi_pack_index *multi_pack_index; + + /* + * private data + * + * should only be accessed directly by packfile.c + */ + + struct packed_git *packed_git; + /* A most-recently-used ordered version of the packed_git list. */ + struct list_head packed_git_mru; + + struct { + struct packed_git **packs; + unsigned flags; + } kept_pack_cache; + + /* + * This is meant to hold a *small* number of objects that you would + * want repo_read_object_file() to be able to return, but yet you do not want + * to write them into the object store (e.g. a browse-only + * application). + */ + struct cached_object_entry *cached_objects; + size_t cached_object_nr, cached_object_alloc; + + /* + * A map of packfiles to packed_git structs for tracking which + * packs have been loaded already. + */ + struct hashmap pack_map; + + /* + * A fast, rough count of the number of objects in the repository. + * These two fields are not meant for direct access. Use + * repo_approximate_object_count() instead. + */ + unsigned long approximate_object_count; + unsigned approximate_object_count_valid : 1; + + /* + * Whether packed_git has already been populated with this repository's + * packs. + */ + unsigned packed_git_initialized : 1; +}; + +struct raw_object_store *raw_object_store_new(void); +void raw_object_store_clear(struct raw_object_store *o); + +/* + * Create a temporary file rooted in the object database directory, or + * die on failure. The filename is taken from "pattern", which should have the + * usual "XXXXXX" trailer, and the resulting filename is written into the + * "template" buffer. Returns the open descriptor. + */ +int odb_mkstemp(struct strbuf *temp_filename, const char *pattern); + +/* + * Create a pack .keep file named "name" (which should generally be the output + * of odb_pack_name). Returns a file descriptor opened for writing, or -1 on + * error. + */ +int odb_pack_keep(const char *name); + +/* + * Put in `buf` the name of the file in the local object database that + * would be used to store a loose object with the specified oid. + */ +const char *loose_object_path(struct repository *r, struct strbuf *buf, + const struct object_id *oid); + +void *map_loose_object(struct repository *r, const struct object_id *oid, + unsigned long *size); + +void *repo_read_object_file(struct repository *r, + const struct object_id *oid, + enum object_type *type, + unsigned long *size); + +/* Read and unpack an object file into memory, write memory to an object file */ +int oid_object_info(struct repository *r, const struct object_id *, unsigned long *); + +void hash_object_file(const struct git_hash_algo *algo, const void *buf, + unsigned long len, enum object_type type, + struct object_id *oid); + +/* + * Add an object file to the in-memory object store, without writing it + * to disk. + * + * Callers are responsible for calling write_object_file to record the + * object in persistent storage before writing any other new objects + * that reference it. + */ +int pretend_object_file(struct repository *repo, + void *buf, unsigned long len, enum object_type type, + struct object_id *oid); + +struct object_info { + /* Request */ + enum object_type *typep; + unsigned long *sizep; + off_t *disk_sizep; + struct object_id *delta_base_oid; + struct strbuf *type_name; + void **contentp; + + /* Response */ + enum { + OI_CACHED, + OI_LOOSE, + OI_PACKED, + OI_DBCACHED + } whence; + union { + /* + * struct { + * ... Nothing to expose in this case + * } cached; + * struct { + * ... Nothing to expose in this case + * } loose; + */ + struct { + struct packed_git *pack; + off_t offset; + unsigned int is_delta; + } packed; + } u; +}; + +/* + * Initializer for a "struct object_info" that wants no items. You may + * also memset() the memory to all-zeroes. + */ +#define OBJECT_INFO_INIT { 0 } + +/* Invoke lookup_replace_object() on the given hash */ +#define OBJECT_INFO_LOOKUP_REPLACE 1 +/* Allow reading from a loose object file of unknown/bogus type */ +#define OBJECT_INFO_ALLOW_UNKNOWN_TYPE 2 +/* Do not retry packed storage after checking packed and loose storage */ +#define OBJECT_INFO_QUICK 8 +/* + * Do not attempt to fetch the object if missing (even if fetch_is_missing is + * nonzero). + */ +#define OBJECT_INFO_SKIP_FETCH_OBJECT 16 +/* + * This is meant for bulk prefetching of missing blobs in a partial + * clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK + */ +#define OBJECT_INFO_FOR_PREFETCH (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK) + +/* Die if object corruption (not just an object being missing) was detected. */ +#define OBJECT_INFO_DIE_IF_CORRUPT 32 + +int oid_object_info_extended(struct repository *r, + const struct object_id *, + struct object_info *, unsigned flags); + +/* Retry packed storage after checking packed and loose storage */ +#define HAS_OBJECT_RECHECK_PACKED 1 + +/* + * Returns 1 if the object exists. This function will not lazily fetch objects + * in a partial clone. + */ +int has_object(struct repository *r, const struct object_id *oid, + unsigned flags); + +/* + * These macros and functions are deprecated. If checking existence for an + * object that is likely to be missing and/or whose absence is relatively + * inconsequential (or is consequential but the caller is prepared to handle + * it), use has_object(), which has better defaults (no lazy fetch in a partial + * clone and no rechecking of packed storage). In the unlikely event that a + * caller needs to assert existence of an object that it fully expects to + * exist, and wants to trigger a lazy fetch in a partial clone, use + * oid_object_info_extended() with a NULL struct object_info. + * + * These functions can be removed once all callers have migrated to + * has_object() and/or oid_object_info_extended(). + */ +int repo_has_object_file(struct repository *r, const struct object_id *oid); +int repo_has_object_file_with_flags(struct repository *r, + const struct object_id *oid, int flags); + +void assert_oid_type(const struct object_id *oid, enum object_type expect); + +/* + * Enabling the object read lock allows multiple threads to safely call the + * following functions in parallel: repo_read_object_file(), + * read_object_with_reference(), oid_object_info() and oid_object_info_extended(). + * + * obj_read_lock() and obj_read_unlock() may also be used to protect other + * section which cannot execute in parallel with object reading. Since the used + * lock is a recursive mutex, these sections can even contain calls to object + * reading functions. However, beware that in these cases zlib inflation won't + * be performed in parallel, losing performance. + * + * TODO: oid_object_info_extended()'s call stack has a recursive behavior. If + * any of its callees end up calling it, this recursive call won't benefit from + * parallel inflation. + */ +void enable_obj_read_lock(void); +void disable_obj_read_lock(void); + +extern int obj_read_use_lock; +extern pthread_mutex_t obj_read_mutex; + +static inline void obj_read_lock(void) +{ + if(obj_read_use_lock) + pthread_mutex_lock(&obj_read_mutex); +} + +static inline void obj_read_unlock(void) +{ + if(obj_read_use_lock) + pthread_mutex_unlock(&obj_read_mutex); +} + +/* + * Iterate over the files in the loose-object parts of the object + * directory "path", triggering the following callbacks: + * + * - loose_object is called for each loose object we find. + * + * - loose_cruft is called for any files that do not appear to be + * loose objects. Note that we only look in the loose object + * directories "objects/[0-9a-f]{2}/", so we will not report + * "objects/foobar" as cruft. + * + * - loose_subdir is called for each top-level hashed subdirectory + * of the object directory (e.g., "$OBJDIR/f0"). It is called + * after the objects in the directory are processed. + * + * Any callback that is NULL will be ignored. Callbacks returning non-zero + * will end the iteration. + * + * In the "buf" variant, "path" is a strbuf which will also be used as a + * scratch buffer, but restored to its original contents before + * the function returns. + */ +typedef int each_loose_object_fn(const struct object_id *oid, + const char *path, + void *data); +typedef int each_loose_cruft_fn(const char *basename, + const char *path, + void *data); +typedef int each_loose_subdir_fn(unsigned int nr, + const char *path, + void *data); +int for_each_file_in_obj_subdir(unsigned int subdir_nr, + struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); +int for_each_loose_file_in_objdir(const char *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); +int for_each_loose_file_in_objdir_buf(struct strbuf *path, + each_loose_object_fn obj_cb, + each_loose_cruft_fn cruft_cb, + each_loose_subdir_fn subdir_cb, + void *data); + +/* Flags for for_each_*_object() below. */ +enum for_each_object_flags { + /* Iterate only over local objects, not alternates. */ + FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0), + + /* Only iterate over packs obtained from the promisor remote. */ + FOR_EACH_OBJECT_PROMISOR_ONLY = (1<<1), + + /* + * Visit objects within a pack in packfile order rather than .idx order + */ + FOR_EACH_OBJECT_PACK_ORDER = (1<<2), + + /* Only iterate over packs that are not marked as kept in-core. */ + FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS = (1<<3), + + /* Only iterate over packs that do not have .keep files. */ + FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4), +}; + +/* + * Iterate over all accessible loose objects without respect to + * reachability. By default, this includes both local and alternate objects. + * The order in which objects are visited is unspecified. + * + * Any flags specific to packs are ignored. + */ +int for_each_loose_object(each_loose_object_fn, void *, + enum for_each_object_flags flags); + +/* + * Iterate over all accessible packed objects without respect to reachability. + * By default, this includes both local and alternate packs. + * + * Note that some objects may appear twice if they are found in multiple packs. + * Each pack is visited in an unspecified order. By default, objects within a + * pack are visited in pack-idx order (i.e., sorted by oid). + */ +typedef int each_packed_object_fn(const struct object_id *oid, + struct packed_git *pack, + uint32_t pos, + void *data); +int for_each_object_in_pack(struct packed_git *p, + each_packed_object_fn, void *data, + enum for_each_object_flags flags); +int for_each_packed_object(struct repository *repo, each_packed_object_fn cb, + void *data, enum for_each_object_flags flags); + +void *read_object_with_reference(struct repository *r, + const struct object_id *oid, + enum object_type required_type, + unsigned long *size, + struct object_id *oid_ret); #endif /* OBJECT_STORE_H */ diff --git a/oss-fuzz/fuzz-pack-idx.c b/oss-fuzz/fuzz-pack-idx.c index 3e190214d1487e..609a343ee3ea41 100644 --- a/oss-fuzz/fuzz-pack-idx.c +++ b/oss-fuzz/fuzz-pack-idx.c @@ -1,5 +1,5 @@ #include "git-compat-util.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size); diff --git a/pack-bitmap-write.c b/pack-bitmap-write.c index 6a97b52b36d424..4c7f7985e78153 100644 --- a/pack-bitmap-write.c +++ b/pack-bitmap-write.c @@ -4,7 +4,7 @@ #include "environment.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "diff.h" #include "revision.h" diff --git a/pack-bitmap.c b/pack-bitmap.c index 0dbd7c4ffe161a..fb68d2ae6374ae 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -17,7 +17,7 @@ #include "packfile.h" #include "repository.h" #include "trace2.h" -#include "object-store-ll.h" +#include "object-store.h" #include "list-objects-filter-options.h" #include "midx.h" #include "config.h" diff --git a/pack-check.c b/pack-check.c index 95dcbbe985265a..874897d6cbab12 100644 --- a/pack-check.c +++ b/pack-check.c @@ -8,7 +8,7 @@ #include "progress.h" #include "packfile.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" struct idx_entry { off_t offset; diff --git a/pack-mtimes.c b/pack-mtimes.c index bcea28e521dfb3..20900ca88d377a 100644 --- a/pack-mtimes.c +++ b/pack-mtimes.c @@ -1,7 +1,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "pack-mtimes.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "strbuf.h" diff --git a/pack-objects.h b/pack-objects.h index d73e3843c92e9c..d1c4ae7f9b6189 100644 --- a/pack-objects.h +++ b/pack-objects.h @@ -1,7 +1,7 @@ #ifndef PACK_OBJECTS_H #define PACK_OBJECTS_H -#include "object-store-ll.h" +#include "object-store.h" #include "thread-utils.h" #include "pack.h" diff --git a/pack-revindex.c b/pack-revindex.c index 1ee7b49e206ad4..37ad9c7926877b 100644 --- a/pack-revindex.c +++ b/pack-revindex.c @@ -1,7 +1,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "pack-revindex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "strbuf.h" #include "trace2.h" diff --git a/packfile.c b/packfile.c index 9d09f8bc72633a..d91016f1c7ff40 100644 --- a/packfile.c +++ b/packfile.c @@ -19,7 +19,7 @@ #include "tree-walk.h" #include "tree.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "midx.h" #include "commit-graph.h" #include "pack-revindex.h" diff --git a/path.c b/path.c index 62d67166dff945..4505bb78e8b470 100644 --- a/path.c +++ b/path.c @@ -15,7 +15,7 @@ #include "submodule-config.h" #include "path.h" #include "packfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "lockfile.h" #include "exec-cmd.h" diff --git a/promisor-remote.c b/promisor-remote.c index 5801ebfd9b2c2b..9d058586dfa8a4 100644 --- a/promisor-remote.c +++ b/promisor-remote.c @@ -3,7 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "promisor-remote.h" #include "config.h" #include "trace2.h" diff --git a/protocol-caps.c b/protocol-caps.c index 855f279c2f7a4f..9b8db37a2105fc 100644 --- a/protocol-caps.c +++ b/protocol-caps.c @@ -6,7 +6,7 @@ #include "hash.h" #include "hex.h" #include "object.h" -#include "object-store-ll.h" +#include "object-store.h" #include "repository.h" #include "string-list.h" #include "strbuf.h" diff --git a/prune-packed.c b/prune-packed.c index 7dad2fc0c169cf..c1d95a519d7479 100644 --- a/prune-packed.c +++ b/prune-packed.c @@ -2,7 +2,7 @@ #include "git-compat-util.h" #include "gettext.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "progress.h" #include "prune-packed.h" diff --git a/reachable.c b/reachable.c index 1b26b9b1d76d8c..16e23a380372f6 100644 --- a/reachable.c +++ b/reachable.c @@ -14,7 +14,7 @@ #include "list-objects.h" #include "packfile.h" #include "worktree.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pack-bitmap.h" #include "pack-mtimes.h" #include "config.h" diff --git a/read-cache.c b/read-cache.c index 23028f43a11615..570744bb56633b 100644 --- a/read-cache.c +++ b/read-cache.c @@ -20,7 +20,7 @@ #include "refs.h" #include "dir.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "tree.h" #include "commit.h" diff --git a/ref-filter.c b/ref-filter.c index 6da8d4c03b62bf..7a274633cfc7e7 100644 --- a/ref-filter.c +++ b/ref-filter.c @@ -12,7 +12,7 @@ #include "refs.h" #include "wildmatch.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "repo-settings.h" #include "repository.h" diff --git a/reflog.c b/reflog.c index 1b5f031f6d787f..60834a124d9a95 100644 --- a/reflog.c +++ b/reflog.c @@ -3,7 +3,7 @@ #include "git-compat-util.h" #include "gettext.h" -#include "object-store-ll.h" +#include "object-store.h" #include "reflog.h" #include "refs.h" #include "revision.h" diff --git a/refs.c b/refs.c index 1208f86629b7f5..14f25c499cdc7c 100644 --- a/refs.c +++ b/refs.c @@ -19,7 +19,7 @@ #include "run-command.h" #include "hook.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "object.h" #include "path.h" #include "submodule.h" diff --git a/remote.c b/remote.c index e609cf5c56a772..7fc657d98b83f2 100644 --- a/remote.c +++ b/remote.c @@ -12,7 +12,7 @@ #include "refs.h" #include "refspec.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "commit.h" #include "diff.h" diff --git a/replace-object.c b/replace-object.c index 9a3cdd809a9b48..7b8a09b5cb4959 100644 --- a/replace-object.c +++ b/replace-object.c @@ -2,7 +2,7 @@ #include "gettext.h" #include "hex.h" #include "oidmap.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "refs.h" #include "repository.h" diff --git a/replace-object.h b/replace-object.h index 66c41b938b44f4..ba478eb30c47a6 100644 --- a/replace-object.h +++ b/replace-object.h @@ -3,7 +3,7 @@ #include "oidmap.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" struct replace_object { struct oidmap_entry original; diff --git a/repository.c b/repository.c index 6cbaf2e3daa93a..9b3d6665fc6d4f 100644 --- a/repository.c +++ b/repository.c @@ -1,7 +1,7 @@ #include "git-compat-util.h" #include "abspath.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "config.h" #include "object.h" #include "lockfile.h" diff --git a/rerere.c b/rerere.c index 0832cc54840a95..3cd37c5f0ae70c 100644 --- a/rerere.c +++ b/rerere.c @@ -18,7 +18,7 @@ #include "path.h" #include "pathspec.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strmap.h" #define RESOLVED 0 diff --git a/revision.c b/revision.c index b536c4a29ad1d7..352e18b1f97c2a 100644 --- a/revision.c +++ b/revision.c @@ -8,7 +8,7 @@ #include "hex.h" #include "object-name.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oidset.h" #include "tag.h" #include "blob.h" diff --git a/send-pack.c b/send-pack.c index 856a65d5f5abce..5005689cb55a48 100644 --- a/send-pack.c +++ b/send-pack.c @@ -4,7 +4,7 @@ #include "date.h" #include "gettext.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pkt-line.h" #include "sideband.h" #include "run-command.h" diff --git a/sequencer.c b/sequencer.c index 9fda9be9266b26..552051e275c25e 100644 --- a/sequencer.c +++ b/sequencer.c @@ -13,7 +13,7 @@ #include "dir.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "object.h" #include "pager.h" #include "commit.h" diff --git a/server-info.c b/server-info.c index f0646ac92a9eed..d6cd20a39d74fc 100644 --- a/server-info.c +++ b/server-info.c @@ -11,7 +11,7 @@ #include "packfile.h" #include "path.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "server-info.h" #include "strbuf.h" #include "tempfile.h" diff --git a/shallow.c b/shallow.c index 06c3266a3e0b14..2f82ebd6e3f5e5 100644 --- a/shallow.c +++ b/shallow.c @@ -5,7 +5,7 @@ #include "repository.h" #include "tempfile.h" #include "lockfile.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "tag.h" #include "pkt-line.h" diff --git a/streaming.c b/streaming.c index 018b794d252819..127d6b5d6ac2d7 100644 --- a/streaming.c +++ b/streaming.c @@ -10,7 +10,7 @@ #include "streaming.h" #include "repository.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "replace-object.h" #include "packfile.h" diff --git a/submodule-config.c b/submodule-config.c index d82b404b73e4f8..8630e27947d394 100644 --- a/submodule-config.c +++ b/submodule-config.c @@ -13,7 +13,7 @@ #include "submodule.h" #include "strbuf.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "parse-options.h" #include "thread-utils.h" #include "tree-walk.h" diff --git a/submodule.c b/submodule.c index 218c8c17603ff2..ead3fb5dadca3a 100644 --- a/submodule.c +++ b/submodule.c @@ -27,7 +27,7 @@ #include "parse-options.h" #include "object-file.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit-reach.h" #include "read-cache-ll.h" #include "setup.h" diff --git a/t/helper/test-pack-mtimes.c b/t/helper/test-pack-mtimes.c index f8f9afbb5b1f57..50f5941bff745a 100644 --- a/t/helper/test-pack-mtimes.c +++ b/t/helper/test-pack-mtimes.c @@ -3,7 +3,7 @@ #include "test-tool.h" #include "hex.h" #include "strbuf.h" -#include "object-store-ll.h" +#include "object-store.h" #include "packfile.h" #include "pack-mtimes.h" #include "setup.h" diff --git a/t/helper/test-partial-clone.c b/t/helper/test-partial-clone.c index a1af9710c31002..34f1aee5581762 100644 --- a/t/helper/test-partial-clone.c +++ b/t/helper/test-partial-clone.c @@ -1,7 +1,7 @@ #include "test-tool.h" #include "hex.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "setup.h" /* diff --git a/t/helper/test-read-graph.c b/t/helper/test-read-graph.c index 811dde1cb3c880..8b413b644be677 100644 --- a/t/helper/test-read-graph.c +++ b/t/helper/test-read-graph.c @@ -3,7 +3,7 @@ #include "test-tool.h" #include "commit-graph.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "bloom.h" #include "setup.h" diff --git a/t/helper/test-read-midx.c b/t/helper/test-read-midx.c index fc632369618917..ac81390899aef0 100644 --- a/t/helper/test-read-midx.c +++ b/t/helper/test-read-midx.c @@ -4,7 +4,7 @@ #include "hex.h" #include "midx.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "pack-bitmap.h" #include "packfile.h" #include "setup.h" diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c index 2ff67c067acf67..4cfc7c90b59bea 100644 --- a/t/helper/test-ref-store.c +++ b/t/helper/test-ref-store.c @@ -5,7 +5,7 @@ #include "refs.h" #include "setup.h" #include "worktree.h" -#include "object-store-ll.h" +#include "object-store.h" #include "path.h" #include "repository.h" #include "strbuf.h" diff --git a/tag.c b/tag.c index 8d9e9e29304c58..05be39067cfe4e 100644 --- a/tag.c +++ b/tag.c @@ -5,7 +5,7 @@ #include "environment.h" #include "tag.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "tree.h" #include "blob.h" diff --git a/tmp-objdir.c b/tmp-objdir.c index 31d16a4c2c576e..c38fbeb5e8aa3a 100644 --- a/tmp-objdir.c +++ b/tmp-objdir.c @@ -10,7 +10,7 @@ #include "strbuf.h" #include "strvec.h" #include "quote.h" -#include "object-store-ll.h" +#include "object-store.h" #include "repository.h" struct tmp_objdir { diff --git a/tree-walk.c b/tree-walk.c index a03339796561b4..90655d52378319 100644 --- a/tree-walk.c +++ b/tree-walk.c @@ -6,7 +6,7 @@ #include "gettext.h" #include "hex.h" #include "object-file.h" -#include "object-store-ll.h" +#include "object-store.h" #include "trace2.h" #include "tree.h" #include "pathspec.h" diff --git a/tree.c b/tree.c index ad86ad1ba99b74..b85f56267fb3bb 100644 --- a/tree.c +++ b/tree.c @@ -4,7 +4,7 @@ #include "hex.h" #include "tree.h" #include "object-name.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "alloc.h" #include "tree-walk.h" diff --git a/unpack-trees.c b/unpack-trees.c index cf5b73c84be2ff..471837f0329d63 100644 --- a/unpack-trees.c +++ b/unpack-trees.c @@ -26,7 +26,7 @@ #include "symlinks.h" #include "trace2.h" #include "fsmonitor.h" -#include "object-store-ll.h" +#include "object-store.h" #include "promisor-remote.h" #include "entry.h" #include "parallel-checkout.h" diff --git a/upload-pack.c b/upload-pack.c index 02ce6336028047..30e4630f3a1cb3 100644 --- a/upload-pack.c +++ b/upload-pack.c @@ -10,7 +10,7 @@ #include "pkt-line.h" #include "sideband.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "oid-array.h" #include "object.h" #include "commit.h" diff --git a/walker.c b/walker.c index 1cf3da02193531..4fedc19f346e66 100644 --- a/walker.c +++ b/walker.c @@ -5,7 +5,7 @@ #include "hex.h" #include "walker.h" #include "repository.h" -#include "object-store-ll.h" +#include "object-store.h" #include "commit.h" #include "strbuf.h" #include "tree.h" diff --git a/xdiff-interface.c b/xdiff-interface.c index 77712811ff1b77..1edcd319e6ef2c 100644 --- a/xdiff-interface.c +++ b/xdiff-interface.c @@ -5,7 +5,7 @@ #include "gettext.h" #include "config.h" #include "hex.h" -#include "object-store-ll.h" +#include "object-store.h" #include "strbuf.h" #include "xdiff-interface.h" #include "xdiff/xtypes.h" From 1b2eee94f120969690d772af5a14f681f7d875a7 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 16 Apr 2025 15:42:15 +0000 Subject: [PATCH 11/36] docs: document core.hooksPath=/dev/null If a user wishes to disable hooks, then they can do so using the established pattern of setting 'core.hooksPath' to /dev/null. This is already tested in t1350-config-hooks-path.sh, but has not previously been visible in the documentation. Update the documentation to include this as an option. Signed-off-by: Derrick Stolee Signed-off-by: Junio C Hamano --- Documentation/config/core.adoc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Documentation/config/core.adoc b/Documentation/config/core.adoc index 8f6d8e77541261..9fde1ab63a70ea 100644 --- a/Documentation/config/core.adoc +++ b/Documentation/config/core.adoc @@ -512,6 +512,11 @@ centrally configure your Git hooks instead of configuring them on a per-repository basis, or as a more flexible and centralized alternative to having an `init.templateDir` where you've changed default hooks. ++ +You can also disable all hooks entirely by setting `core.hooksPath` +to `/dev/null`. This is usually only advisable for expert users and +on a per-command basis using configuration parameters of the form +`git -c core.hooksPath=/dev/null ...`. core.editor:: Commands such as `commit` and `tag` that let you edit From dccf1296d85aac93156d151253f2669f1b34a152 Mon Sep 17 00:00:00 2001 From: Anthony Wang Date: Wed, 16 Apr 2025 16:59:39 +0200 Subject: [PATCH 12/36] t9811: be more precise to check importing of tags The tests use grep to search the output of `git tag` for tagnames they expect to exist, which can incorrectly pass if an unxpected tag has the expected tag as its substring. We fix this by using `git show-ref --verify` instead. Additionally, we add a negative test to verify that a possible uninteded tag does not show up in the imported repository. This change also fixes an additional problem, where piping the output of `git tag` caused the exit codes to be lost. Signed-off-by: Anthony Wang Signed-off-by: Junio C Hamano --- t/t9811-git-p4-label-import.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/t/t9811-git-p4-label-import.sh b/t/t9811-git-p4-label-import.sh index 5ac5383fb71715..39856629c00657 100755 --- a/t/t9811-git-p4-label-import.sh +++ b/t/t9811-git-p4-label-import.sh @@ -95,9 +95,9 @@ test_expect_success 'two labels on the same changelist' ' cd "$git" && git p4 sync --import-labels && - git tag | grep TAG_F1 && - git tag | grep -q TAG_F1_1 && - git tag | grep -q TAG_F1_2 && + git show-ref --verify refs/tags/TAG_F1_1 && + git show-ref --verify refs/tags/TAG_F1_2 && + test_must_fail git show-ref --verify refs/tags/TAG_F1_ONLY && cd main && @@ -207,8 +207,7 @@ test_expect_success 'use git config to enable import/export of tags' ' git tag CFG_A_GIT_TAG && git p4 rebase --verbose && git p4 submit --verbose && - git tag && - git tag | grep TAG_F1_1 + git show-ref --verify refs/tags/TAG_F1_1 && ) && ( cd "$cli" && From 0d07e0617c0de87eae402c57c20ed137abeeeb08 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:22 +0100 Subject: [PATCH 13/36] meson.build: remove -DCURL_DISABLE_TYPECHECK Commit 9371322a60 ("sparse: suppress some \"using sizeof on a function\" warnings", 2013-10-06) used target-specific variable assignments to add -DCURL_DISABLE_TYPECHECK to SPARSE_FLAGS for each of the files affected by the "typecheck-gcc.h" warnings. (http-push.c, http.c, http-walker.c and remote-curl.c). These warnings are only issued by sparse, and not by gcc, so we do not want to disable the 'type checking' for non-sparse targets. The meson build does not provide any sparse targets, so there is no need to use the CURL_DISABLE_TYPECHECK preprocessor flag with the c compiler. In order to re-enable the curl 'type checking' in the meson build, remove the assignment of -DCURL_DISABLE_TYPECHECK to libgit_c_args. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- meson.build | 1 - 1 file changed, 1 deletion(-) diff --git a/meson.build b/meson.build index efe2871c9dba13..88a29fd0436a31 100644 --- a/meson.build +++ b/meson.build @@ -966,7 +966,6 @@ if curl.found() # Most executables don't have to link against libcurl, but we still need its # include directories so that we can resolve LIBCURL_VERSION in "help.c". libgit_dependencies += curl.partial_dependency(includes: true) - libgit_c_args += '-DCURL_DISABLE_TYPECHECK' build_options_config.set('NO_CURL', '') else libgit_c_args += '-DNO_CURL' From 507c63f96e02a27bdcc2c48377e7b0b9d1981458 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:23 +0100 Subject: [PATCH 14/36] Makefile: only set some BASIC_CFLAGS when RUNTIME_PREFIX is set Several build variables only have any meaning when the RUNTIME_PREFIX variable has been set. In particular, the following build variables are otherwise ignored: HAVE_BSD_KERN_PROC_SYSCTL PROCFS_EXECUTABLE_PATH HAVE_NS_GET_EXECUTABLE_PATH HAVE_ZOS_GET_EXECUTABLE_PATH HAVE_WPGMPTR Make setting BASIC_CFLAGS, for each of these variables, conditional on the RUNTIME_PREFIX being defined. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- Makefile | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 7315507381e146..818dde32a9a34e 100644 --- a/Makefile +++ b/Makefile @@ -2179,10 +2179,6 @@ ifdef HAVE_BSD_SYSCTL BASIC_CFLAGS += -DHAVE_BSD_SYSCTL endif -ifdef HAVE_BSD_KERN_PROC_SYSCTL - BASIC_CFLAGS += -DHAVE_BSD_KERN_PROC_SYSCTL -endif - ifdef HAVE_GETDELIM BASIC_CFLAGS += -DHAVE_GETDELIM endif @@ -2213,25 +2209,33 @@ ifneq ($(findstring openssl,$(CSPRNG_METHOD)),) EXTLIBS += -lcrypto -lssl endif -ifneq ($(PROCFS_EXECUTABLE_PATH),) - procfs_executable_path_SQ = $(subst ','\'',$(PROCFS_EXECUTABLE_PATH)) - BASIC_CFLAGS += '-DPROCFS_EXECUTABLE_PATH="$(procfs_executable_path_SQ)"' -endif - ifndef HAVE_PLATFORM_PROCINFO COMPAT_OBJS += compat/stub/procinfo.o endif -ifdef HAVE_NS_GET_EXECUTABLE_PATH - BASIC_CFLAGS += -DHAVE_NS_GET_EXECUTABLE_PATH -endif +ifdef RUNTIME_PREFIX -ifdef HAVE_ZOS_GET_EXECUTABLE_PATH - BASIC_CFLAGS += -DHAVE_ZOS_GET_EXECUTABLE_PATH -endif + ifdef HAVE_BSD_KERN_PROC_SYSCTL + BASIC_CFLAGS += -DHAVE_BSD_KERN_PROC_SYSCTL + endif + + ifneq ($(PROCFS_EXECUTABLE_PATH),) + pep_SQ = $(subst ','\'',$(PROCFS_EXECUTABLE_PATH)) + BASIC_CFLAGS += '-DPROCFS_EXECUTABLE_PATH="$(pep_SQ)"' + endif + + ifdef HAVE_NS_GET_EXECUTABLE_PATH + BASIC_CFLAGS += -DHAVE_NS_GET_EXECUTABLE_PATH + endif + + ifdef HAVE_ZOS_GET_EXECUTABLE_PATH + BASIC_CFLAGS += -DHAVE_ZOS_GET_EXECUTABLE_PATH + endif + + ifdef HAVE_WPGMPTR + BASIC_CFLAGS += -DHAVE_WPGMPTR + endif -ifdef HAVE_WPGMPTR - BASIC_CFLAGS += -DHAVE_WPGMPTR endif ifdef FILENO_IS_A_MACRO From 65e3757c240422f4e1ffb8892786a1dfc24894f7 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:24 +0100 Subject: [PATCH 15/36] meson.build: only set build variables for non-default values Some preprocessor -Defines have defaults set in the source code when they have not been provided to the C compiler. In this case, there is no need to pass them on the command-line, unless the build requires a non-standard value. The build variables for DEFAULT_EDITOR and DEFAULT_PAGER have appropriate defaults ('vi' and 'less') set in the code. Add the preprocessor -Defines to the 'libgit_c_args' only if the values set with the corresponding 'options' are different to these standard values. Also, the 'git-var' documentation contains some conditional text which documents the chosen compiled in value, which would not read well for the standard values. Similar to the above, only add the corresponding '-a' attribute arguments to the 'asciidoc_common_options' variable, if the values set in the 'options' are different to these standard values. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- Documentation/meson.build | 20 ++++++++++++++++++++ meson.build | 13 +++++++++++-- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/Documentation/meson.build b/Documentation/meson.build index 594546d68b12c1..1642b6e2a3ab74 100644 --- a/Documentation/meson.build +++ b/Documentation/meson.build @@ -242,6 +242,16 @@ if docs_backend == 'asciidoc' '--attribute=build_dir=' + meson.current_build_dir(), ] + pager_opt = get_option('default_pager') + if pager_opt != '' and pager_opt != 'less' + asciidoc_common_options += '-agit-default-pager=' + pager_opt + endif + + editor_opt = get_option('default_editor') + if editor_opt != '' and editor_opt != 'vi' + asciidoc_common_options += '-agit-default-editor=' + editor_opt + endif + documentation_deps = [ asciidoc_conf, ] @@ -279,6 +289,16 @@ elif docs_backend == 'asciidoctor' '--require', 'asciidoctor-extensions', ] + pager_opt = get_option('default_pager') + if pager_opt != '' and pager_opt != 'less' + asciidoc_common_options += '-agit-default-pager=' + pager_opt + endif + + editor_opt = get_option('default_editor') + if editor_opt != '' and editor_opt != 'vi' + asciidoc_common_options += '-agit-default-editor=' + editor_opt + endif + documentation_deps = [ asciidoctor_extensions, ] diff --git a/meson.build b/meson.build index 88a29fd0436a31..efd0bd3319889c 100644 --- a/meson.build +++ b/meson.build @@ -693,10 +693,8 @@ endif # These variables are used for building libgit.a. libgit_c_args = [ '-DBINDIR="' + get_option('bindir') + '"', - '-DDEFAULT_EDITOR="' + get_option('default_editor') + '"', '-DDEFAULT_GIT_TEMPLATE_DIR="' + get_option('datadir') / 'git-core/templates' + '"', '-DDEFAULT_HELP_FORMAT="' + get_option('default_help_format') + '"', - '-DDEFAULT_PAGER="' + get_option('default_pager') + '"', '-DETC_GITATTRIBUTES="' + get_option('gitattributes') + '"', '-DETC_GITCONFIG="' + get_option('gitconfig') + '"', '-DFALLBACK_RUNTIME_PREFIX="' + get_option('prefix') + '"', @@ -708,6 +706,17 @@ libgit_c_args = [ '-DPAGER_ENV="' + get_option('pager_environment') + '"', '-DSHELL_PATH="' + fs.as_posix(shell.full_path()) + '"', ] + +editor_opt = get_option('default_editor') +if editor_opt != '' and editor_opt != 'vi' + libgit_c_args += '-DDEFAULT_EDITOR="' + editor_opt + '"' +endif + +pager_opt = get_option('default_pager') +if pager_opt != '' and pager_opt != 'less' + libgit_c_args += '-DDEFAULT_PAGER="' + pager_opt + '"' +endif + libgit_include_directories = [ '.' ] libgit_dependencies = [ ] From 214e2c6856b43fb8b053766fe47af12dfe4356e3 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:25 +0100 Subject: [PATCH 16/36] meson.build: set default help format to html on windows The build variable DEFAULT_HELP_FORMAT has an appropriate default ('man') set in the code, so there is no need to pass the -Define on the compiler command-line, unless the build requires a non-standard value. In addition, on windows the make build overrides the default help format to 'html', rather than 'man', in the 'config.mak.uname' file. In order to suppress the -Define on the C compiler command-line, only add the -Define to the 'libgit_c_args' variable when the requested value is not the standard 'man'. In order to override the default value on windows, add a 'platform' value to the 'default_help_format' combo option and set it as the default choice. When this option is set to 'platform', use the 'host_machine.system()' method call to determine the appropriate default value for the host system. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- meson.build | 13 ++++++++++++- meson_options.txt | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/meson.build b/meson.build index efd0bd3319889c..8f8a2580647bf2 100644 --- a/meson.build +++ b/meson.build @@ -694,7 +694,6 @@ endif libgit_c_args = [ '-DBINDIR="' + get_option('bindir') + '"', '-DDEFAULT_GIT_TEMPLATE_DIR="' + get_option('datadir') / 'git-core/templates' + '"', - '-DDEFAULT_HELP_FORMAT="' + get_option('default_help_format') + '"', '-DETC_GITATTRIBUTES="' + get_option('gitattributes') + '"', '-DETC_GITCONFIG="' + get_option('gitconfig') + '"', '-DFALLBACK_RUNTIME_PREFIX="' + get_option('prefix') + '"', @@ -717,6 +716,18 @@ if pager_opt != '' and pager_opt != 'less' libgit_c_args += '-DDEFAULT_PAGER="' + pager_opt + '"' endif +help_format_opt = get_option('default_help_format') +if help_format_opt == 'platform' + if host_machine.system() == 'windows' + help_format_opt = 'html' + else + help_format_opt = 'man' + endif +endif +if help_format_opt != 'man' + libgit_c_args += '-DDEFAULT_HELP_FORMAT="' + help_format_opt + '"' +endif + libgit_include_directories = [ '.' ] libgit_dependencies = [ ] diff --git a/meson_options.txt b/meson_options.txt index 78d172a74019a4..8ac30a522311c5 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -95,7 +95,7 @@ option('highlight_bin', type: 'string', value: 'highlight') # Documentation. option('docs', type: 'array', choices: ['man', 'html'], value: [], description: 'Which documenattion formats to build and install.') -option('default_help_format', type: 'combo', choices: ['man', 'html'], value: 'man', +option('default_help_format', type: 'combo', choices: ['man', 'html', 'platform'], value: 'platform', description: 'Default format used when executing git-help(1).') option('docs_backend', type: 'combo', choices: ['asciidoc', 'asciidoctor', 'auto'], value: 'auto', description: 'Which backend to use to generate documentation.') From ead39b241cfb31c1c6a0244f3593772637ed30c9 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:26 +0100 Subject: [PATCH 17/36] Makefile: remove NEEDS_LIBRT build variable Commit d19e3a5b21 ("Makefile: add NEEDS_LIBRT to optionally link with librt", 2016-07-07) introduced the NEEDS_LIBRT build variable to disassociate the HAVE_CLOCK_GETTIME variable with the unconditional linking of the librt library. At one time, the clock_gettime() function was not available as part of the libc library and (on some unix systems) required linking with librt. Commit 52fcec75ce ("config.mak.uname: define NEEDS_LIBRT under Linux, for now", 2016-07-10) set the NEEDS_LIBRT variable in the Linux section of the config.mak.uname file, since Debian 7 (wheezy) was one of the few remaining distributions, with glibc 2.13, that required linking with librt for clock_gettime(). Note that from glibc version 2.17, this is no longer necessary. Note that Debian 7.0 was released on May 4th, 2013 and benefited from long term support until May 2018 when it went end-of-life. Since that time, Linux distributions use a more up-to-date library, for example: Distribution version end of support Debian 8 2.19 30th June 2020 RHEL 8 2.28 31st May 2024 * Ubuntu 16.04 2.23 30th Apr 2021 * paid 'Maintenance support' ends 31st May 2029 Since it is no longer required, remove NEEDS_LIBRT from the Makefile and config.mak.uname. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- Makefile | 7 ------- config.mak.uname | 2 -- 2 files changed, 9 deletions(-) diff --git a/Makefile b/Makefile index 818dde32a9a34e..8fa4d2664e162b 100644 --- a/Makefile +++ b/Makefile @@ -340,9 +340,6 @@ include shared.mak # # Define HAVE_SYNC_FILE_RANGE if your platform has sync_file_range. # -# Define NEEDS_LIBRT if your platform requires linking with librt (glibc version -# before 2.17) for clock_gettime and CLOCK_MONOTONIC. -# # Define HAVE_BSD_SYSCTL if your platform has a BSD-compatible sysctl function. # # Define HAVE_GETDELIM if your system has the getdelim() function. @@ -2171,10 +2168,6 @@ ifdef HAVE_SYNC_FILE_RANGE BASIC_CFLAGS += -DHAVE_SYNC_FILE_RANGE endif -ifdef NEEDS_LIBRT - EXTLIBS += -lrt -endif - ifdef HAVE_BSD_SYSCTL BASIC_CFLAGS += -DHAVE_BSD_SYSCTL endif diff --git a/config.mak.uname b/config.mak.uname index b12d4e168ae119..88007c4f1320b0 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -54,8 +54,6 @@ ifeq ($(uname_S),Linux) HAVE_DEV_TTY = YesPlease HAVE_CLOCK_GETTIME = YesPlease HAVE_CLOCK_MONOTONIC = YesPlease - # -lrt is needed for clock_gettime on glibc <= 2.16 - NEEDS_LIBRT = YesPlease HAVE_SYNC_FILE_RANGE = YesPlease HAVE_GETDELIM = YesPlease FREAD_READS_DIRECTORIES = UnfortunatelyYes From 26a8b236eec02aeb730f2c6dfa4c5268fe5e7c38 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:27 +0100 Subject: [PATCH 18/36] config.mak.uname: add a note about NO_STRLCPY for Linux Commit 817151e61a ("Rename safe_strncpy() to strlcpy().", 2006-06-24) added the NO_STRLCPY make variable to allow the conditional use of the gitstrlcpy() compat function on those platforms which didn't provide the 'standard' strlcpy() function. Recently, in the summer of 2023, the strlcpy() and strlcat() functions were added to the glibc library (v2.38), so some of the more up-to-date Linux distributions no longer need to set NO_STRLCPY. For example, both Ubuntu 24.04 LTS and RHEL 10 beta have glibc v2.39. However, several distributions, which are still within their support window, have an earlier version and must still use the 'compat' version of strlcpy(). If the meson or autoconf build systems are used on newer platforms, then they will be configured to to use strlcpy() from glibc, whereas the make build will always choose the 'compat' function instead. Add a note to the config.mak.uname file, in the Linux section, to prompt make users to override NO_STRLCPY in the config.mak file, if appropriate. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 88007c4f1320b0..ae6ba1558603f7 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -48,6 +48,7 @@ ifeq ($(uname_S),OSF1) endif ifeq ($(uname_S),Linux) HAVE_ALLOCA_H = YesPlease + # override in config.mak if you have glibc >= 2.38 NO_STRLCPY = YesPlease HAVE_PATHS_H = YesPlease LIBC_CONTAINS_LIBINTL = YesPlease From 064eed36c7f6f291ed4c1966b382d87293fac3a3 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:28 +0100 Subject: [PATCH 19/36] config.mak.uname: only set NO_REGEX on cygwin for v1.7 Commit 92f63d2b05 ("Cygwin 1.7 needs compat/regex", 2013-07-19) set the NO_REGEX build variable because the platform regex library failed some of the tests (t4018 and t4034), which passed just fine with the compat library. After some time (maybe a year or two), the platform library had been updated (with an import from FreeBSD, I believe) and now passed the full test-suite. This would be about the time of the v1.7 -> v2.0 transition in 2015. I had a patch ready to send, but just didn't get around to submitting it to the list. At some point in the interim, the official cygwin git package used the autoconf build system, which sets the NO_REGEX variable to use the platform regex library functions. The new meson build system does likewise. The cygwin platform regex library, in addition to now passing the tests which formerly failed, now passes an 'test_expect_failure' test in the t7815-grep-binary test file. In particular, test #12 'git grep .fi a' which determines that the regex pattern '.' matches a NUL character. The commit f96e56733a ("grep: use REG_STARTEND for all matching if available", 2010-05-22) added the test in question, but it does not give any indication as to why the test was framed as an expected fail, rather than a 'positive' test that the 'git grep' command fails to match a NUL. Note that the previous test #11 was also originally marked in that commit as a 'test_expect_failure', but was flipped to an 'success' test in commit 7e36de5859 ("t/t7008-grep-binary.sh: un-TODO a test that needs REG_STARTEND", 2010-08-17). In order to produce the same NO_REGEX configuration from autoconf, meson and make, modify config.mak.uname to only set NO_REGEX for cygwin v1.7. In addition, skip test t7815.12 on cygwin, by adding the !CYGWIN pre- requisite to the test header, which (among other things) removes an '...; please update test(s)' comment. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 4 +++- t/t7815-grep-binary.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config.mak.uname b/config.mak.uname index ae6ba1558603f7..b6adce0bc467fd 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -245,7 +245,9 @@ ifeq ($(uname_O),Cygwin) # Try commenting this out if you suspect MMAP is more efficient NO_MMAP = YesPlease else - NO_REGEX = UnfortunatelyYes + ifeq ($(shell expr "$(uname_R)" : '1\.7\.'),4) + NO_REGEX = UnfortunatelyYes + endif endif HAVE_DEV_TTY = YesPlease HAVE_ALLOCA_H = YesPlease diff --git a/t/t7815-grep-binary.sh b/t/t7815-grep-binary.sh index 90ebb64f46ebfa..b7000dfa3226ff 100755 --- a/t/t7815-grep-binary.sh +++ b/t/t7815-grep-binary.sh @@ -63,7 +63,7 @@ test_expect_success 'git grep ile a' ' git grep ile a ' -test_expect_failure 'git grep .fi a' ' +test_expect_failure !CYGWIN 'git grep .fi a' ' git grep .fi a ' From 394c4dfd510b0612f66b08bbfa87324c01b33ac0 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:29 +0100 Subject: [PATCH 20/36] config.mak.uname: add HAVE_GETDELIM to the cygwin section Cygwin has provided the getdelim() function as far back as (at least) 2011. The autoconf and meson builds enable the use of this symbol. In order to have the same configuration for autoconf, meson and make, enable the HAVE_GETDELIM build variable in the cygwin section of the config.mak.uname file. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index b6adce0bc467fd..1b3ba8a34143c9 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -250,6 +250,7 @@ ifeq ($(uname_O),Cygwin) endif endif HAVE_DEV_TTY = YesPlease + HAVE_GETDELIM = YesPlease HAVE_ALLOCA_H = YesPlease NEEDS_LIBICONV = YesPlease NO_FAST_WORKING_DIRECTORY = UnfortunatelyYes From a45ca6fcfe3aeb31aa9c12de5eeced225e253052 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:30 +0100 Subject: [PATCH 21/36] config.mak.uname: add clock_gettime() to the cygwin build Cygwin supports the clock_gettime() function, along with the associated CLOCK_MONOTONIC preprocessor symbol. The autoconf and meson builds both enable the use of those symbols. In order to have the same configuration for the make builds, add the HAVE_CLOCK_GETTIME and HAVE_CLOCK_MONOTONIC build variables to the cygwin section of the config.mak.uname file. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config.mak.uname b/config.mak.uname index 1b3ba8a34143c9..d058b34292a159 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -251,6 +251,8 @@ ifeq ($(uname_O),Cygwin) endif HAVE_DEV_TTY = YesPlease HAVE_GETDELIM = YesPlease + HAVE_CLOCK_GETTIME = YesPlease + HAVE_CLOCK_MONOTONIC = YesPlease HAVE_ALLOCA_H = YesPlease NEEDS_LIBICONV = YesPlease NO_FAST_WORKING_DIRECTORY = UnfortunatelyYes From c9a51775a36b424189d7cb65634e8e657a75a05f Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:31 +0100 Subject: [PATCH 22/36] builtin/gc.c: correct RAM calculation when using sysinfo The man page for sysinfo(2) on Linux states that (from v2.3.48) the sizes of the memory and swap fields, of the returned structure, are given as multiples of 'mem_unit' bytes. In earlier versions (prior to v2.3.23 on i386 in particular), the 'mem_unit' field was not part of the structure, and all sizes were measured in bytes. The man page does not discuss the motivation for this change, but it is possible that the change was intended for the, relatively rare, 32-bit platform with more than 4GB of memory. The total_ram() function makes the assumption that the 'totalram' field of the 'struct sysinfo' is measured in bytes, or alternatively that the 'mem_unit' field is always equal to one. Having writen a program to call the sysinfo() function and print the structure fields, it seems that, on Linux x84_64 and i686 anyway, the 'mem_unit' field is indeed set to one (note that the 32-bit system had only 2GB ram). However, cygwin also has an sysinfo() implementation, which gives the following values: $ ./sysinfo uptime: 21381 loads: 0, 0, 0 total ram: 2074637 free ram: 843237 shared ram: 0 buffer ram: 0 total swap: 327680 free swap: 306932 procs: 15 total high: 0 free high: 0 mem_unit: 4096 total ram: 8497713152 $ [This laptop has 8GB ram, so a little bit seems to be missing. ;) ] Modify the total_ram() function to allow for the possibility that the memory size is not specified in bytes (ie 'mem_unit' is greater than one). Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- builtin/gc.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/builtin/gc.c b/builtin/gc.c index 99431fd46744cd..cdcf1dc6e7d551 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -373,8 +373,13 @@ static uint64_t total_ram(void) #if defined(HAVE_SYSINFO) struct sysinfo si; - if (!sysinfo(&si)) - return si.totalram; + if (!sysinfo(&si)) { + uint64_t total = si.totalram; + + if (si.mem_unit > 1) + total *= (uint64_t)si.mem_unit; + return total; + } #elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM)) int64_t physical_memory; int mib[2]; From 50dec7c566d87838aaaa6017b5fb10b0f1cc8af7 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:32 +0100 Subject: [PATCH 23/36] config.mak.uname: add sysinfo() configuration for cygwin Although sysinfo() is a 'Linux only' function, cygwin provides an implementation which appears to be functional. The assumption that this function is Linux only is reflected in the way the HAVE_SYSINFO build variable is handled by the Makefile and config.mak.uname. Rework the setting of HAVE_SYSINFO in the Linux section of the system specific config file, along with the corresponding setting of the BASIC_CFLAGS in the Makefile. Add the setting of HAVE_SYSINFO to the cygwin section of 'config.mak.uname'. While here, add a test for the sysinfo() function to the autoconf build system. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- Makefile | 4 ++++ config.mak.uname | 3 ++- configure.ac | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8fa4d2664e162b..ff89f2b81f47b6 100644 --- a/Makefile +++ b/Makefile @@ -2168,6 +2168,10 @@ ifdef HAVE_SYNC_FILE_RANGE BASIC_CFLAGS += -DHAVE_SYNC_FILE_RANGE endif +ifdef HAVE_SYSINFO + BASIC_CFLAGS += -DHAVE_SYSINFO +endif + ifdef HAVE_BSD_SYSCTL BASIC_CFLAGS += -DHAVE_BSD_SYSCTL endif diff --git a/config.mak.uname b/config.mak.uname index d058b34292a159..e7bd07cf6ec578 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -58,7 +58,7 @@ ifeq ($(uname_S),Linux) HAVE_SYNC_FILE_RANGE = YesPlease HAVE_GETDELIM = YesPlease FREAD_READS_DIRECTORIES = UnfortunatelyYes - BASIC_CFLAGS += -DHAVE_SYSINFO + HAVE_SYSINFO = YesPlease PROCFS_EXECUTABLE_PATH = /proc/self/exe HAVE_PLATFORM_PROCINFO = YesPlease COMPAT_OBJS += compat/linux/procinfo.o @@ -253,6 +253,7 @@ ifeq ($(uname_O),Cygwin) HAVE_GETDELIM = YesPlease HAVE_CLOCK_GETTIME = YesPlease HAVE_CLOCK_MONOTONIC = YesPlease + HAVE_SYSINFO = YesPlease HAVE_ALLOCA_H = YesPlease NEEDS_LIBICONV = YesPlease NO_FAST_WORKING_DIRECTORY = UnfortunatelyYes diff --git a/configure.ac b/configure.ac index 5923edc44aa7b6..d7e0503f1ec580 100644 --- a/configure.ac +++ b/configure.ac @@ -1066,6 +1066,14 @@ AC_CHECK_LIB([iconv], [locale_charset], [AC_CHECK_LIB([charset], [locale_charset], [CHARSET_LIB=-lcharset])]) GIT_CONF_SUBST([CHARSET_LIB]) + +# +# Define HAVE_SYSINFO=YesPlease if sysinfo is available. +GIT_CHECK_FUNC(sysinfo, + [HAVE_SYSINFO=YesPlease], + [HAVE_SYSINFO=]) +GIT_CONF_SUBST([HAVE_SYSINFO]) + # # Define HAVE_CLOCK_GETTIME=YesPlease if clock_gettime is available. GIT_CHECK_FUNC(clock_gettime, From 70ef34c8d8001e677a346666f32040adfa90e89c Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:33 +0100 Subject: [PATCH 24/36] config.mak.uname: add arc4random to the cygwin build The arc4random_buf() function has been available in cygwin since about 2016 (somewhere in the v2.x branch). Set the CSPRNG_METHOD build variable to 'arc4random', in the cygwin section, to enable the use of this cryptographically-secure pseudorandom number function. Note that the autoconf and new meson builds also enable this function. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index e7bd07cf6ec578..330741eb5a1cdb 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -254,6 +254,7 @@ ifeq ($(uname_O),Cygwin) HAVE_CLOCK_GETTIME = YesPlease HAVE_CLOCK_MONOTONIC = YesPlease HAVE_SYSINFO = YesPlease + CSPRNG_METHOD = arc4random HAVE_ALLOCA_H = YesPlease NEEDS_LIBICONV = YesPlease NO_FAST_WORKING_DIRECTORY = UnfortunatelyYes From 8f282bdff0b49744b45d619075b59a5e8b596613 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:36 +0200 Subject: [PATCH 25/36] parse: fix off-by-one for minimum signed values We accept a maximum value in `git_parse_signed()` that restricts the range of accepted integers. As the intent is to pass `INT*_MAX` values here, this maximum doesn't only act as the upper bound, but also as the implicit lower bound of the accepted range. This lower bound is calculated by negating the maximum. But given that the maximum value of a signed integer with N bits is `2^(N-1)-1` whereas the minimum value is `-2^(N-1)` we have an off-by-one error in the lower bound. Fix this off-by-one error by using `-max - 1` as lower bound instead. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- parse.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parse.c b/parse.c index 7a60a4f816c3e4..3c47448ca675fb 100644 --- a/parse.c +++ b/parse.c @@ -38,7 +38,7 @@ int git_parse_signed(const char *value, intmax_t *ret, intmax_t max) errno = EINVAL; return 0; } - if ((val < 0 && -max / factor > val) || + if ((val < 0 && (-max - 1) / factor > val) || (val > 0 && max / factor < val)) { errno = ERANGE; return 0; From d012ceb5f3351af0589a0c82b07059bce8c7b24b Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:37 +0200 Subject: [PATCH 26/36] global: use designated initializers for options While we expose macros for most of our different option types understood by the "parse-options" subsystem, not every combination of fields that has one as that would otherwise quickly lead to an explosion of macros. Instead, we just initialize structures manually for those variants of fields that don't have a macro. Callsites that open-code these structure initialization don't use designated initializers though and instead just provide values for each of the fields that they want to initialize. This has three significant downsides: - Callsites need to specify all values up to the last field that they care about. This often includes fields that should simply be left at their default zero-initialized state, which adds distraction. - Any reader not deeply familiar with the layout of the structure has a hard time figuring out what the respective initializers mean. - Reordering or introducing new fields in the middle of the structure is impossible without adapting all callsites. Convert all sites to instead use designated initializers, which we have started using in our codebase quite a while ago. This allows us to skip any default-initialized fields, gives the reader context by specifying the field names and allows us to reorder or introduce new fields where we want to. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- archive.c | 35 ++++++--- builtin/am.c | 28 +++++--- builtin/clone.c | 13 +++- builtin/commit-tree.c | 12 +++- builtin/commit.c | 62 ++++++++++++---- builtin/config.c | 13 +++- builtin/describe.c | 24 +++++-- builtin/fetch.c | 10 ++- builtin/fmt-merge-msg.c | 25 +++++-- builtin/gc.c | 12 +++- builtin/grep.c | 14 ++-- builtin/init-db.c | 13 ++-- builtin/ls-remote.c | 11 ++- builtin/merge.c | 37 +++++++--- builtin/read-tree.c | 11 ++- builtin/rebase.c | 25 +++++-- builtin/revert.c | 12 +++- builtin/show-branch.c | 12 +++- builtin/tag.c | 23 ++++-- builtin/update-index.c | 131 +++++++++++++++++++++++----------- builtin/write-tree.c | 12 ++-- diff.c | 13 ++-- ref-filter.h | 15 ++-- t/helper/test-parse-options.c | 38 +++++++--- 24 files changed, 443 insertions(+), 158 deletions(-) diff --git a/archive.c b/archive.c index 8be4e7ac8db50c..67bba3cd301fc5 100644 --- a/archive.c +++ b/archive.c @@ -650,20 +650,37 @@ static int parse_archive_args(int argc, const char **argv, OPT_STRING(0, "format", &format, N_("fmt"), N_("archive format")), OPT_STRING(0, "prefix", &base, N_("prefix"), N_("prepend prefix to each pathname in the archive")), - { OPTION_CALLBACK, 0, "add-file", args, N_("file"), - N_("add untracked file to archive"), 0, add_file_cb, - (intptr_t)&base }, - { OPTION_CALLBACK, 0, "add-virtual-file", args, - N_("path:content"), N_("add untracked file to archive"), 0, - add_file_cb, (intptr_t)&base }, + { + .type = OPTION_CALLBACK, + .long_name = "add-file", + .value = args, + .argh = N_("file"), + .help = N_("add untracked file to archive"), + .callback = add_file_cb, + .defval = (intptr_t) &base, + }, + { + .type = OPTION_CALLBACK, + .long_name = "add-virtual-file", + .value = args, + .argh = N_("path:content"), + .help = N_("add untracked file to archive"), + .callback = add_file_cb, + .defval = (intptr_t) &base, + }, OPT_STRING('o', "output", &output, N_("file"), N_("write the archive to this file")), OPT_BOOL(0, "worktree-attributes", &worktree_attributes, N_("read .gitattributes in working directory")), OPT__VERBOSE(&verbose, N_("report archived files on stderr")), - { OPTION_STRING, 0, "mtime", &mtime_option, N_("time"), - N_("set modification time of archive entries"), - PARSE_OPT_NONEG }, + { + .type = OPTION_STRING, + .long_name = "mtime", + .value = &mtime_option, + .argh = N_("time"), + .help = N_("set modification time of archive entries"), + .flags = PARSE_OPT_NONEG, + }, OPT_NUMBER_CALLBACK(&compression_level, N_("set compression level"), number_callback), OPT_GROUP(""), diff --git a/builtin/am.c b/builtin/am.c index 3b61bd4c333c4b..4afb519830f48f 100644 --- a/builtin/am.c +++ b/builtin/am.c @@ -2400,11 +2400,16 @@ int cmd_am(int argc, OPT_CMDMODE(0, "quit", &resume_mode, N_("abort the patching operation but keep HEAD where it is"), RESUME_QUIT), - { OPTION_CALLBACK, 0, "show-current-patch", &resume_mode, - "(diff|raw)", - N_("show the patch being applied"), - PARSE_OPT_CMDMODE | PARSE_OPT_OPTARG | PARSE_OPT_NONEG | PARSE_OPT_LITERAL_ARGHELP, - parse_opt_show_current_patch, RESUME_SHOW_PATCH_RAW }, + { + .type = OPTION_CALLBACK, + .long_name = "show-current-patch", + .value = &resume_mode, + .argh = "(diff|raw)", + .help = N_("show the patch being applied"), + .flags = PARSE_OPT_CMDMODE | PARSE_OPT_OPTARG | PARSE_OPT_NONEG | PARSE_OPT_LITERAL_ARGHELP, + .callback = parse_opt_show_current_patch, + .defval = RESUME_SHOW_PATCH_RAW, + }, OPT_CMDMODE(0, "retry", &resume_mode, N_("try to apply current patch again"), RESUME_APPLY), @@ -2417,9 +2422,16 @@ int cmd_am(int argc, OPT_BOOL(0, "ignore-date", &state.ignore_date, N_("use current timestamp for author date")), OPT_RERERE_AUTOUPDATE(&state.allow_rerere_autoupdate), - { OPTION_STRING, 'S', "gpg-sign", &state.sign_commit, N_("key-id"), - N_("GPG-sign commits"), - PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &state.sign_commit, + .argh = N_("key-id"), + .help = N_("GPG-sign commits"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, OPT_CALLBACK_F(0, "empty", &state.empty_type, "(stop|drop|keep)", N_("how to handle empty patches"), PARSE_OPT_NONEG, am_option_parse_empty), diff --git a/builtin/clone.c b/builtin/clone.c index 88276e5b7ab2fd..9c3547f41e3677 100644 --- a/builtin/clone.c +++ b/builtin/clone.c @@ -930,9 +930,16 @@ int cmd_clone(int argc, N_("don't use local hardlinks, always copy")), OPT_BOOL('s', "shared", &option_shared, N_("setup as shared repository")), - { OPTION_CALLBACK, 0, "recurse-submodules", &option_recurse_submodules, - N_("pathspec"), N_("initialize submodules in the clone"), - PARSE_OPT_OPTARG, recurse_submodules_cb, (intptr_t)"." }, + { + .type = OPTION_CALLBACK, + .long_name = "recurse-submodules", + .value = &option_recurse_submodules, + .argh = N_("pathspec"), + .help = N_("initialize submodules in the clone"), + .flags = PARSE_OPT_OPTARG, + .callback = recurse_submodules_cb, + .defval = (intptr_t)".", + }, OPT_ALIAS(0, "recursive", "recurse-submodules"), OPT_INTEGER('j', "jobs", &max_jobs, N_("number of submodules cloned in parallel")), diff --git a/builtin/commit-tree.c b/builtin/commit-tree.c index 38457600a4e422..c787133d004c7e 100644 --- a/builtin/commit-tree.c +++ b/builtin/commit-tree.c @@ -111,8 +111,16 @@ int cmd_commit_tree(int argc, OPT_CALLBACK_F('F', NULL, &buffer, N_("file"), N_("read commit log message from file"), PARSE_OPT_NONEG, parse_file_arg_callback), - { OPTION_STRING, 'S', "gpg-sign", &sign_commit, N_("key-id"), - N_("GPG sign commit"), PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &sign_commit, + .argh = N_("key-id"), + .help = N_("GPG sign commit"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, OPT_END() }; int ret; diff --git a/builtin/commit.c b/builtin/commit.c index 2f459682221d6a..66bd91fd523dd7 100644 --- a/builtin/commit.c +++ b/builtin/commit.c @@ -1542,17 +1542,34 @@ struct repository *repo UNUSED) STATUS_FORMAT_LONG), OPT_BOOL('z', "null", &s.null_termination, N_("terminate entries with NUL")), - { OPTION_STRING, 'u', "untracked-files", &untracked_files_arg, - N_("mode"), - N_("show untracked files, optional modes: all, normal, no. (Default: all)"), - PARSE_OPT_OPTARG, NULL, (intptr_t)"all" }, - { OPTION_STRING, 0, "ignored", &ignored_arg, - N_("mode"), - N_("show ignored files, optional modes: traditional, matching, no. (Default: traditional)"), - PARSE_OPT_OPTARG, NULL, (intptr_t)"traditional" }, - { OPTION_STRING, 0, "ignore-submodules", &ignore_submodule_arg, N_("when"), - N_("ignore changes to submodules, optional when: all, dirty, untracked. (Default: all)"), - PARSE_OPT_OPTARG, NULL, (intptr_t)"all" }, + { + .type = OPTION_STRING, + .short_name = 'u', + .long_name = "untracked-files", + .value = &untracked_files_arg, + .argh = N_("mode"), + .help = N_("show untracked files, optional modes: all, normal, no. (Default: all)"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)"all", + }, + { + .type = OPTION_STRING, + .long_name = "ignored", + .value = &ignored_arg, + .argh = N_("mode"), + .help = N_("show ignored files, optional modes: traditional, matching, no. (Default: traditional)"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)"traditional", + }, + { + .type = OPTION_STRING, + .long_name = "ignore-submodules", + .value = &ignore_submodule_arg, + .argh = N_("when"), + .help = N_("ignore changes to submodules, optional when: all, dirty, untracked. (Default: all)"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)"all", + }, OPT_COLUMN(0, "column", &s.colopts, N_("list untracked files in columns")), OPT_BOOL(0, "no-renames", &no_renames, N_("do not detect renames")), OPT_CALLBACK_F('M', "find-renames", &rename_score_arg, @@ -1688,8 +1705,16 @@ int cmd_commit(int argc, OPT_BOOL('e', "edit", &edit_flag, N_("force edit of commit")), OPT_CLEANUP(&cleanup_arg), OPT_BOOL(0, "status", &include_status, N_("include status in commit message template")), - { OPTION_STRING, 'S', "gpg-sign", &sign_commit, N_("key-id"), - N_("GPG sign commit"), PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &sign_commit, + .argh = N_("key-id"), + .help = N_("GPG sign commit"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, /* end commit message options */ OPT_GROUP(N_("Commit contents options")), @@ -1714,7 +1739,16 @@ int cmd_commit(int argc, N_("terminate entries with NUL")), OPT_BOOL(0, "amend", &amend, N_("amend previous commit")), OPT_BOOL(0, "no-post-rewrite", &no_post_rewrite, N_("bypass post-rewrite hook")), - { OPTION_STRING, 'u', "untracked-files", &untracked_files_arg, N_("mode"), N_("show untracked files, optional modes: all, normal, no. (Default: all)"), PARSE_OPT_OPTARG, NULL, (intptr_t)"all" }, + { + .type = OPTION_STRING, + .short_name = 'u', + .long_name = "untracked-files", + .value = &untracked_files_arg, + .argh = N_("mode"), + .help = N_("show untracked files, optional modes: all, normal, no. (Default: all)"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)"all", + }, OPT_PATHSPEC_FROM_FILE(&pathspec_from_file), OPT_PATHSPEC_FILE_NUL(&pathspec_file_nul), /* end commit contents options */ diff --git a/builtin/config.c b/builtin/config.c index 53a90094e31454..f70d6354772259 100644 --- a/builtin/config.c +++ b/builtin/config.c @@ -131,9 +131,16 @@ struct config_display_options { #define TYPE_COLOR 6 #define TYPE_BOOL_OR_STR 7 -#define OPT_CALLBACK_VALUE(s, l, v, h, i) \ - { OPTION_CALLBACK, (s), (l), (v), NULL, (h), PARSE_OPT_NOARG | \ - PARSE_OPT_NONEG, option_parse_type, (i) } +#define OPT_CALLBACK_VALUE(s, l, v, h, i) { \ + .type = OPTION_CALLBACK, \ + .short_name = (s), \ + .long_name = (l), \ + .value = (v), \ + .help = (h), \ + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, \ + .callback = option_parse_type, \ + .defval = (i), \ +} static int option_parse_type(const struct option *opt, const char *arg, int unset) diff --git a/builtin/describe.c b/builtin/describe.c index e2e73f3d757cab..2da9f4fed018ee 100644 --- a/builtin/describe.c +++ b/builtin/describe.c @@ -601,12 +601,24 @@ int cmd_describe(int argc, N_("do not consider tags matching ")), OPT_BOOL(0, "always", &always, N_("show abbreviated commit object as fallback")), - {OPTION_STRING, 0, "dirty", &dirty, N_("mark"), - N_("append on dirty working tree (default: \"-dirty\")"), - PARSE_OPT_OPTARG, NULL, (intptr_t) "-dirty"}, - {OPTION_STRING, 0, "broken", &broken, N_("mark"), - N_("append on broken working tree (default: \"-broken\")"), - PARSE_OPT_OPTARG, NULL, (intptr_t) "-broken"}, + { + .type = OPTION_STRING, + .long_name = "dirty", + .value = &dirty, + .argh = N_("mark"), + .help = N_("append on dirty working tree (default: \"-dirty\")"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "-dirty", + }, + { + .type = OPTION_STRING, + .long_name = "broken", + .value = &broken, + .argh = N_("mark"), + .help = N_("append on broken working tree (default: \"-broken\")"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "-broken", + }, OPT_END(), }; diff --git a/builtin/fetch.c b/builtin/fetch.c index 02af50546908e1..3a5159d9e696d4 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -2359,8 +2359,14 @@ int cmd_fetch(int argc, OPT_SET_INT_F(0, "refetch", &refetch, N_("re-fetch without negotiating common commits"), 1, PARSE_OPT_NONEG), - { OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, N_("dir"), - N_("prepend this to submodule path output"), PARSE_OPT_HIDDEN }, + { + .type = OPTION_STRING, + .long_name = "submodule-prefix", + .value = &submodule_prefix, + .argh = N_("dir"), + .help = N_("prepend this to submodule path output"), + .flags = PARSE_OPT_HIDDEN, + }, OPT_CALLBACK_F(0, "recurse-submodules-default", &recurse_submodules_default, N_("on-demand"), N_("default for recursive fetching of submodules " diff --git a/builtin/fmt-merge-msg.c b/builtin/fmt-merge-msg.c index 189cd1096a0b92..240cdb474bc49b 100644 --- a/builtin/fmt-merge-msg.c +++ b/builtin/fmt-merge-msg.c @@ -20,13 +20,24 @@ int cmd_fmt_merge_msg(int argc, char *into_name = NULL; int shortlog_len = -1; struct option options[] = { - { OPTION_INTEGER, 0, "log", &shortlog_len, N_("n"), - N_("populate log with at most entries from shortlog"), - PARSE_OPT_OPTARG, NULL, DEFAULT_MERGE_LOG_LEN }, - { OPTION_INTEGER, 0, "summary", &shortlog_len, N_("n"), - N_("alias for --log (deprecated)"), - PARSE_OPT_OPTARG | PARSE_OPT_HIDDEN, NULL, - DEFAULT_MERGE_LOG_LEN }, + { + .type = OPTION_INTEGER, + .long_name = "log", + .value = &shortlog_len, + .argh = N_("n"), + .help = N_("populate log with at most entries from shortlog"), + .flags = PARSE_OPT_OPTARG, + .defval = DEFAULT_MERGE_LOG_LEN, + }, + { + .type = OPTION_INTEGER, + .long_name = "summary", + .value = &shortlog_len, + .argh = N_("n"), + .help = N_("alias for --log (deprecated)"), + .flags = PARSE_OPT_OPTARG | PARSE_OPT_HIDDEN, + .defval = DEFAULT_MERGE_LOG_LEN, + }, OPT_STRING('m', "message", &message, N_("text"), N_("use as start of message")), OPT_STRING(0, "into-name", &into_name, N_("name"), diff --git a/builtin/gc.c b/builtin/gc.c index 99431fd46744cd..6707a26bc6e140 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -699,9 +699,15 @@ struct repository *repo UNUSED) int ret; struct option builtin_gc_options[] = { OPT__QUIET(&quiet, N_("suppress progress reporting")), - { OPTION_STRING, 0, "prune", &prune_expire_arg, N_("date"), - N_("prune unreferenced objects"), - PARSE_OPT_OPTARG, NULL, (intptr_t)prune_expire_arg }, + { + .type = OPTION_STRING, + .long_name = "prune", + .value = &prune_expire_arg, + .argh = N_("date"), + .help = N_("prune unreferenced objects"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t)prune_expire_arg, + }, OPT_BOOL(0, "cruft", &cfg.cruft_packs, N_("pack unreferenced objects separately")), OPT_MAGNITUDE(0, "max-cruft-size", &cfg.max_cruft_size, N_("with --cruft, limit the size of new cruft packs")), diff --git a/builtin/grep.c b/builtin/grep.c index d1427290f773b6..c4869733e1bab6 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -1017,10 +1017,16 @@ int cmd_grep(int argc, OPT_BOOL(0, "all-match", &opt.all_match, N_("show only matches from files that match all patterns")), OPT_GROUP(""), - { OPTION_STRING, 'O', "open-files-in-pager", &show_in_pager, - N_("pager"), N_("show matching files in the pager"), - PARSE_OPT_OPTARG | PARSE_OPT_NOCOMPLETE, - NULL, (intptr_t)default_pager }, + { + .type = OPTION_STRING, + .short_name = 'O', + .long_name = "open-files-in-pager", + .value = &show_in_pager, + .argh = N_("pager"), + .help = N_("show matching files in the pager"), + .flags = PARSE_OPT_OPTARG | PARSE_OPT_NOCOMPLETE, + .defval = (intptr_t)default_pager, + }, OPT_BOOL_F(0, "ext-grep", &external_grep_allowed__ignored, N_("allow calling of grep(1) (ignored by this build)"), PARSE_OPT_NOCOMPLETE), diff --git a/builtin/init-db.c b/builtin/init-db.c index 196dccdd77acb8..4a950e44d8d46a 100644 --- a/builtin/init-db.c +++ b/builtin/init-db.c @@ -93,10 +93,15 @@ int cmd_init_db(int argc, N_("directory from which templates will be used")), OPT_SET_INT(0, "bare", &is_bare_repository_cfg, N_("create a bare repository"), 1), - { OPTION_CALLBACK, 0, "shared", &init_shared_repository, - N_("permissions"), - N_("specify that the git repository is to be shared amongst several users"), - PARSE_OPT_OPTARG | PARSE_OPT_NONEG, shared_callback, 0}, + { + .type = OPTION_CALLBACK, + .long_name = "shared", + .value = &init_shared_repository, + .argh = N_("permissions"), + .help = N_("specify that the git repository is to be shared amongst several users"), + .flags = PARSE_OPT_OPTARG | PARSE_OPT_NONEG, + .callback = shared_callback + }, OPT_BIT('q', "quiet", &flags, N_("be quiet"), INIT_DB_QUIET), OPT_STRING(0, "separate-git-dir", &real_git_dir, N_("gitdir"), N_("separate git dir from working tree")), diff --git a/builtin/ls-remote.c b/builtin/ls-remote.c index 42f34e123610c1..01a4d4daa1faba 100644 --- a/builtin/ls-remote.c +++ b/builtin/ls-remote.c @@ -67,9 +67,14 @@ int cmd_ls_remote(int argc, OPT__QUIET(&quiet, N_("do not print remote URL")), OPT_STRING(0, "upload-pack", &uploadpack, N_("exec"), N_("path of git-upload-pack on the remote host")), - { OPTION_STRING, 0, "exec", &uploadpack, N_("exec"), - N_("path of git-upload-pack on the remote host"), - PARSE_OPT_HIDDEN }, + { + .type = OPTION_STRING, + .long_name = "exec", + .value = &uploadpack, + .argh = N_("exec"), + .help = N_("path of git-upload-pack on the remote host"), + .flags = PARSE_OPT_HIDDEN, + }, OPT_BIT('t', "tags", &flags, N_("limit to tags"), REF_TAGS), OPT_BIT('b', "branches", &flags, N_("limit to branches"), REF_BRANCHES), OPT_BIT_F('h', "heads", &flags, diff --git a/builtin/merge.c b/builtin/merge.c index ba9faf126aa7a4..21787d45165e6d 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -250,9 +250,15 @@ static struct option builtin_merge_options[] = { OPT_BOOL(0, "stat", &show_diffstat, N_("show a diffstat at the end of the merge")), OPT_BOOL(0, "summary", &show_diffstat, N_("(synonym to --stat)")), - { OPTION_INTEGER, 0, "log", &shortlog_len, N_("n"), - N_("add (at most ) entries from shortlog to merge commit message"), - PARSE_OPT_OPTARG, NULL, DEFAULT_MERGE_LOG_LEN }, + { + .type = OPTION_INTEGER, + .long_name = "log", + .value = &shortlog_len, + .argh = N_("n"), + .help = N_("add (at most ) entries from shortlog to merge commit message"), + .flags = PARSE_OPT_OPTARG, + .defval = DEFAULT_MERGE_LOG_LEN, + }, OPT_BOOL(0, "squash", &squash, N_("create a single commit instead of doing a merge")), OPT_BOOL(0, "commit", &option_commit, @@ -274,9 +280,16 @@ static struct option builtin_merge_options[] = { OPT_CALLBACK('m', "message", &merge_msg, N_("message"), N_("merge commit message (for a non-fast-forward merge)"), option_parse_message), - { OPTION_LOWLEVEL_CALLBACK, 'F', "file", &merge_msg, N_("path"), - N_("read message from file"), PARSE_OPT_NONEG, - NULL, 0, option_read_message }, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .short_name = 'F', + .long_name = "file", + .value = &merge_msg, + .argh = N_("path"), + .help = N_("read message from file"), + .flags = PARSE_OPT_NONEG, + .ll_callback = option_read_message, + }, OPT_STRING(0, "into-name", &into_name, N_("name"), N_("use instead of the real target")), OPT__VERBOSITY(&verbosity), @@ -289,8 +302,16 @@ static struct option builtin_merge_options[] = { OPT_BOOL(0, "allow-unrelated-histories", &allow_unrelated_histories, N_("allow merging unrelated histories")), OPT_SET_INT(0, "progress", &show_progress, N_("force progress reporting"), 1), - { OPTION_STRING, 'S', "gpg-sign", &sign_commit, N_("key-id"), - N_("GPG sign commit"), PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &sign_commit, + .argh = N_("key-id"), + .help = N_("GPG sign commit"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, OPT_AUTOSTASH(&autostash), OPT_BOOL(0, "overwrite-ignore", &overwrite_ignore, N_("update ignored files (default)")), OPT_BOOL(0, "signoff", &signoff, N_("add a Signed-off-by trailer")), diff --git a/builtin/read-tree.c b/builtin/read-tree.c index d2a807a828b6ab..a8f352f7cd9b10 100644 --- a/builtin/read-tree.c +++ b/builtin/read-tree.c @@ -135,9 +135,14 @@ int cmd_read_tree(int argc, N_("3-way merge in presence of adds and removes")), OPT_BOOL(0, "reset", &opts.reset, N_("same as -m, but discard unmerged entries")), - { OPTION_STRING, 0, "prefix", &opts.prefix, N_("/"), - N_("read the tree into the index under /"), - PARSE_OPT_NONEG }, + { + .type = OPTION_STRING, + .long_name = "prefix", + .value = &opts.prefix, + .argh = N_("/"), + .help = N_("read the tree into the index under /"), + .flags = PARSE_OPT_NONEG, + }, OPT_BOOL('u', NULL, &opts.update, N_("update working tree with merge result")), OPT_CALLBACK_F(0, "exclude-per-directory", &opts, diff --git a/builtin/rebase.c b/builtin/rebase.c index d4715ed35d77ed..d4083350090341 100644 --- a/builtin/rebase.c +++ b/builtin/rebase.c @@ -1122,9 +1122,15 @@ int cmd_rebase(int argc, OPT_BIT('v', "verbose", &options.flags, N_("display a diffstat of what changed upstream"), REBASE_NO_QUIET | REBASE_VERBOSE | REBASE_DIFFSTAT), - {OPTION_NEGBIT, 'n', "no-stat", &options.flags, NULL, - N_("do not show diffstat of what changed upstream"), - PARSE_OPT_NOARG, NULL, REBASE_DIFFSTAT }, + { + .type = OPTION_NEGBIT, + .short_name = 'n', + .long_name = "no-stat", + .value = &options.flags, + .help = N_("do not show diffstat of what changed upstream"), + .flags = PARSE_OPT_NOARG, + .defval = REBASE_DIFFSTAT, + }, OPT_BOOL(0, "signoff", &options.signoff, N_("add a Signed-off-by trailer to each commit")), OPT_BOOL(0, "committer-date-is-author-date", @@ -1190,9 +1196,16 @@ int cmd_rebase(int argc, OPT_BOOL(0, "update-refs", &options.update_refs, N_("update branches that point to commits " "that are being rebased")), - { OPTION_STRING, 'S', "gpg-sign", &gpg_sign, N_("key-id"), - N_("GPG-sign commits"), - PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &gpg_sign, + .argh = N_("key-id"), + .help = N_("GPG-sign commits"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, OPT_AUTOSTASH(&options.autostash), OPT_STRING_LIST('x', "exec", &options.exec, N_("exec"), N_("add exec lines after each commit of the " diff --git a/builtin/revert.c b/builtin/revert.c index aca6c293cdfb2f..4f5ef9754949f1 100644 --- a/builtin/revert.c +++ b/builtin/revert.c @@ -132,8 +132,16 @@ static int run_sequencer(int argc, const char **argv, const char *prefix, OPT_STRING(0, "strategy", &strategy, N_("strategy"), N_("merge strategy")), OPT_STRVEC('X', "strategy-option", &opts->xopts, N_("option"), N_("option for merge strategy")), - { OPTION_STRING, 'S', "gpg-sign", &gpg_sign, N_("key-id"), - N_("GPG sign commit"), PARSE_OPT_OPTARG, NULL, (intptr_t) "" }, + { + .type = OPTION_STRING, + .short_name = 'S', + .long_name = "gpg-sign", + .value = &gpg_sign, + .argh = N_("key-id"), + .help = N_("GPG sign commit"), + .flags = PARSE_OPT_OPTARG, + .defval = (intptr_t) "", + }, OPT_END() }; struct option *options = base_options; diff --git a/builtin/show-branch.c b/builtin/show-branch.c index fce6b404e9265b..dab37019d29210 100644 --- a/builtin/show-branch.c +++ b/builtin/show-branch.c @@ -667,9 +667,15 @@ int cmd_show_branch(int ac, N_("show remote-tracking branches")), OPT__COLOR(&showbranch_use_color, N_("color '*!+-' corresponding to the branch")), - { OPTION_INTEGER, 0, "more", &extra, N_("n"), - N_("show more commits after the common ancestor"), - PARSE_OPT_OPTARG, NULL, (intptr_t)1 }, + { + .type = OPTION_INTEGER, + .long_name = "more", + .value = &extra, + .argh = N_("n"), + .help = N_("show more commits after the common ancestor"), + .flags = PARSE_OPT_OPTARG, + .defval = 1, + }, OPT_SET_INT(0, "list", &extra, N_("synonym to more=-1"), -1), OPT_BOOL(0, "no-name", &no_name, N_("suppress naming strings")), OPT_BOOL(0, "current", &with_current_branch, diff --git a/builtin/tag.c b/builtin/tag.c index d3e0943b734759..b266f12bb48d12 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -479,9 +479,15 @@ int cmd_tag(int argc, int edit_flag = 0; struct option options[] = { OPT_CMDMODE('l', "list", &cmdmode, N_("list tag names"), 'l'), - { OPTION_INTEGER, 'n', NULL, &filter.lines, N_("n"), - N_("print lines of each tag message"), - PARSE_OPT_OPTARG, NULL, 1 }, + { + .type = OPTION_INTEGER, + .short_name = 'n', + .value = &filter.lines, + .argh = N_("n"), + .help = N_("print lines of each tag message"), + .flags = PARSE_OPT_OPTARG, + .defval = 1, + }, OPT_CMDMODE('d', "delete", &cmdmode, N_("delete tags"), 'd'), OPT_CMDMODE('v', "verify", &cmdmode, N_("verify tags"), 'v'), @@ -513,9 +519,14 @@ int cmd_tag(int argc, N_("do not output a newline after empty formatted refs")), OPT_REF_SORT(&sorting_options), { - OPTION_CALLBACK, 0, "points-at", &filter.points_at, N_("object"), - N_("print only tags of the object"), PARSE_OPT_LASTARG_DEFAULT, - parse_opt_object_name, (intptr_t) "HEAD" + .type = OPTION_CALLBACK, + .long_name = "points-at", + .value = &filter.points_at, + .argh = N_("object"), + .help = N_("print only tags of the object"), + .flags = PARSE_OPT_LASTARG_DEFAULT, + .callback = parse_opt_object_name, + .defval = (intptr_t) "HEAD", }, OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")), diff --git a/builtin/update-index.c b/builtin/update-index.c index b2f6b1a3fbb6cd..ee64b0226797bc 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -964,29 +964,51 @@ int cmd_update_index(int argc, N_("like --refresh, but ignore assume-unchanged setting"), PARSE_OPT_NOARG | PARSE_OPT_NONEG, really_refresh_callback), - {OPTION_LOWLEVEL_CALLBACK, 0, "cacheinfo", NULL, - N_(",,"), - N_("add the specified entry to the index"), - PARSE_OPT_NOARG | /* disallow --cacheinfo= form */ - PARSE_OPT_NONEG | PARSE_OPT_LITERAL_ARGHELP, - NULL, 0, - cacheinfo_callback}, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .long_name = "cacheinfo", + .argh = N_(",,"), + .help = N_("add the specified entry to the index"), + .flags = PARSE_OPT_NOARG | /* disallow --cacheinfo= form */ + PARSE_OPT_NONEG | PARSE_OPT_LITERAL_ARGHELP, + .ll_callback = cacheinfo_callback, + }, OPT_CALLBACK_F(0, "chmod", &set_executable_bit, "(+|-)x", N_("override the executable bit of the listed files"), PARSE_OPT_NONEG, chmod_callback), - {OPTION_SET_INT, 0, "assume-unchanged", &mark_valid_only, NULL, - N_("mark files as \"not changing\""), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG}, - {OPTION_SET_INT, 0, "no-assume-unchanged", &mark_valid_only, NULL, - N_("clear assumed-unchanged bit"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG}, - {OPTION_SET_INT, 0, "skip-worktree", &mark_skip_worktree_only, NULL, - N_("mark files as \"index-only\""), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG}, - {OPTION_SET_INT, 0, "no-skip-worktree", &mark_skip_worktree_only, NULL, - N_("clear skip-worktree bit"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG}, + { + .type = OPTION_SET_INT, + .long_name = "assume-unchanged", + .value = &mark_valid_only, + .help = N_("mark files as \"not changing\""), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = MARK_FLAG, + }, + { + .type = OPTION_SET_INT, + .long_name = "no-assume-unchanged", + .value = &mark_valid_only, + .help = N_("clear assumed-unchanged bit"), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = UNMARK_FLAG, + }, + { + .type = OPTION_SET_INT, + .long_name = "skip-worktree", + .value = &mark_skip_worktree_only, + .help = N_("mark files as \"index-only\""), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = MARK_FLAG, + }, + { + .type = OPTION_SET_INT, + .long_name = "no-skip-worktree", + .value = &mark_skip_worktree_only, + .help = N_("clear skip-worktree bit"), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = UNMARK_FLAG, + }, OPT_BOOL(0, "ignore-skip-worktree-entries", &ignore_skip_worktree_entries, N_("do not touch index-only entries")), OPT_SET_INT(0, "info-only", &info_only, @@ -995,22 +1017,39 @@ int cmd_update_index(int argc, N_("remove named paths even if present in worktree"), 1), OPT_BOOL('z', NULL, &nul_term_line, N_("with --stdin: input lines are terminated by null bytes")), - {OPTION_LOWLEVEL_CALLBACK, 0, "stdin", &read_from_stdin, NULL, - N_("read list of paths to be updated from standard input"), - PARSE_OPT_NONEG | PARSE_OPT_NOARG, - NULL, 0, stdin_callback}, - {OPTION_LOWLEVEL_CALLBACK, 0, "index-info", &nul_term_line, NULL, - N_("add entries from standard input to the index"), - PARSE_OPT_NONEG | PARSE_OPT_NOARG, - NULL, 0, stdin_cacheinfo_callback}, - {OPTION_LOWLEVEL_CALLBACK, 0, "unresolve", &has_errors, NULL, - N_("repopulate stages #2 and #3 for the listed paths"), - PARSE_OPT_NONEG | PARSE_OPT_NOARG, - NULL, 0, unresolve_callback}, - {OPTION_LOWLEVEL_CALLBACK, 'g', "again", &has_errors, NULL, - N_("only update entries that differ from HEAD"), - PARSE_OPT_NONEG | PARSE_OPT_NOARG, - NULL, 0, reupdate_callback}, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .long_name = "stdin", + .value = &read_from_stdin, + .help = N_("read list of paths to be updated from standard input"), + .flags = PARSE_OPT_NONEG | PARSE_OPT_NOARG, + .ll_callback = stdin_callback, + }, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .long_name = "index-info", + .value = &nul_term_line, + .help = N_("add entries from standard input to the index"), + .flags = PARSE_OPT_NONEG | PARSE_OPT_NOARG, + .ll_callback = stdin_cacheinfo_callback, + }, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .long_name = "unresolve", + .value = &has_errors, + .help = N_("repopulate stages #2 and #3 for the listed paths"), + .flags = PARSE_OPT_NONEG | PARSE_OPT_NOARG, + .ll_callback = unresolve_callback, + }, + { + .type = OPTION_LOWLEVEL_CALLBACK, + .short_name = 'g', + .long_name = "again", + .value = &has_errors, + .help = N_("only update entries that differ from HEAD"), + .flags = PARSE_OPT_NONEG | PARSE_OPT_NOARG, + .ll_callback = reupdate_callback, + }, OPT_BIT(0, "ignore-missing", &refresh_args.flags, N_("ignore files missing from worktree"), REFRESH_IGNORE_MISSING), @@ -1036,12 +1075,22 @@ int cmd_update_index(int argc, N_("write out the index even if is not flagged as changed"), 1), OPT_BOOL(0, "fsmonitor", &fsmonitor, N_("enable or disable file system monitor")), - {OPTION_SET_INT, 0, "fsmonitor-valid", &mark_fsmonitor_only, NULL, - N_("mark files as fsmonitor valid"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, MARK_FLAG}, - {OPTION_SET_INT, 0, "no-fsmonitor-valid", &mark_fsmonitor_only, NULL, - N_("clear fsmonitor valid bit"), - PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, UNMARK_FLAG}, + { + .type = OPTION_SET_INT, + .long_name = "fsmonitor-valid", + .value = &mark_fsmonitor_only, + .help = N_("mark files as fsmonitor valid"), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = MARK_FLAG, + }, + { + .type = OPTION_SET_INT, + .long_name = "no-fsmonitor-valid", + .value = &mark_fsmonitor_only, + .help = N_("clear fsmonitor valid bit"), + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = UNMARK_FLAG, + }, OPT_END() }; diff --git a/builtin/write-tree.c b/builtin/write-tree.c index 43f233e69b08ae..5a8dc377ec07b0 100644 --- a/builtin/write-tree.c +++ b/builtin/write-tree.c @@ -31,10 +31,14 @@ int cmd_write_tree(int argc, WRITE_TREE_MISSING_OK), OPT_STRING(0, "prefix", &tree_prefix, N_("/"), N_("write tree object for a subdirectory ")), - { OPTION_BIT, 0, "ignore-cache-tree", &flags, NULL, - N_("only useful for debugging"), - PARSE_OPT_HIDDEN | PARSE_OPT_NOARG, NULL, - WRITE_TREE_IGNORE_CACHE_TREE }, + { + .type = OPTION_BIT, + .long_name = "ignore-cache-tree", + .value = &flags, + .help = N_("only useful for debugging"), + .flags = PARSE_OPT_HIDDEN | PARSE_OPT_NOARG, + .defval = WRITE_TREE_IGNORE_CACHE_TREE, + }, OPT_END() }; diff --git a/diff.c b/diff.c index 08f5e00a2cc921..f2fcc7f3c22105 100644 --- a/diff.c +++ b/diff.c @@ -5892,10 +5892,15 @@ struct option *add_diff_options(const struct option *opts, OPT_CALLBACK_F(0, "diff-filter", options, N_("[(A|C|D|M|R|T|U|X|B)...[*]]"), N_("select files by diff type"), PARSE_OPT_NONEG, diff_opt_diff_filter), - { OPTION_CALLBACK, 0, "output", options, N_(""), - N_("output to a specific file"), - PARSE_OPT_NONEG, NULL, 0, diff_opt_output }, - + { + .type = OPTION_CALLBACK, + .long_name = "output", + .value = options, + .argh = N_(""), + .help = N_("output to a specific file"), + .flags = PARSE_OPT_NONEG, + .ll_callback = diff_opt_output, + }, OPT_END() }; diff --git a/ref-filter.h b/ref-filter.h index 013d4cfa64b310..c98c4fbd4c150d 100644 --- a/ref-filter.h +++ b/ref-filter.h @@ -114,11 +114,16 @@ struct ref_format { } /* Macros for checking --merged and --no-merged options */ -#define _OPT_MERGED_NO_MERGED(option, filter, h) \ - { OPTION_CALLBACK, 0, option, (filter), N_("commit"), (h), \ - PARSE_OPT_LASTARG_DEFAULT | PARSE_OPT_NONEG, \ - parse_opt_merge_filter, (intptr_t) "HEAD" \ - } +#define _OPT_MERGED_NO_MERGED(option, filter, h) { \ + .type = OPTION_CALLBACK, \ + .long_name = option, \ + .value = (filter), \ + .argh = N_("commit"), \ + .help = (h), \ + .flags = PARSE_OPT_LASTARG_DEFAULT | PARSE_OPT_NONEG, \ + .callback = parse_opt_merge_filter, \ + .defval = (intptr_t) "HEAD", \ +} #define OPT_MERGED(f, h) _OPT_MERGED_NO_MERGED("merged", f, h) #define OPT_NO_MERGED(f, h) _OPT_MERGED_NO_MERGED("no-merged", f, h) diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index bfe45ec68b01f1..997f55fd45b38a 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -124,8 +124,15 @@ int cmd__parse_options(int argc, const char **argv) struct option options[] = { OPT_BOOL(0, "yes", &boolean, "get a boolean"), OPT_BOOL('D', "no-doubt", &boolean, "begins with 'no-'"), - { OPTION_SET_INT, 'B', "no-fear", &boolean, NULL, - "be brave", PARSE_OPT_NOARG | PARSE_OPT_NONEG, NULL, 1 }, + { + .type = OPTION_SET_INT, + .short_name = 'B', + .long_name = "no-fear", + .value = &boolean, + .help = "be brave", + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + .defval = 1, + }, OPT_COUNTUP('b', "boolean", &boolean, "increment by one"), OPT_BIT('4', "or4", &boolean, "bitwise-or boolean with ...0100", 4), @@ -155,12 +162,27 @@ int cmd__parse_options(int argc, const char **argv) OPT_GROUP("Magic arguments"), OPT_NUMBER_CALLBACK(&integer, "set integer to NUM", number_callback), - { OPTION_COUNTUP, '+', NULL, &boolean, NULL, "same as -b", - PARSE_OPT_NOARG | PARSE_OPT_NONEG | PARSE_OPT_NODASH }, - { OPTION_COUNTUP, 0, "ambiguous", &ambiguous, NULL, - "positive ambiguity", PARSE_OPT_NOARG | PARSE_OPT_NONEG }, - { OPTION_COUNTUP, 0, "no-ambiguous", &ambiguous, NULL, - "negative ambiguity", PARSE_OPT_NOARG | PARSE_OPT_NONEG }, + { + .type = OPTION_COUNTUP, + .short_name = '+', + .value = &boolean, + .help = "same as -b", + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG | PARSE_OPT_NODASH, + }, + { + .type = OPTION_COUNTUP, + .long_name = "ambiguous", + .value = &ambiguous, + .help = "positive ambiguity", + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + }, + { + .type = OPTION_COUNTUP, + .long_name = "no-ambiguous", + .value = &ambiguous, + .help = "negative ambiguity", + .flags = PARSE_OPT_NOARG | PARSE_OPT_NONEG, + }, OPT_GROUP("Standard options"), OPT__ABBREV(&abbrev), OPT__VERBOSE(&verbose, "be verbose"), From 8ff1a34bdfef0a0689130508388325af1db38237 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:38 +0200 Subject: [PATCH 27/36] parse-options: support unit factors in `OPT_INTEGER()` There are two main differences between `OPT_INTEGER()` and `OPT_MAGNITUDE()`: - The former parses signed integers whereas the latter parses unsigned integers. - The latter parses unit factors like 'k', 'm' or 'g'. While the first difference makes obvious sense, there isn't really a good reason why signed integers shouldn't support unit factors, too. This inconsistency will also become a bit of a problem with subsequent commits, where we will fix a couple of callsites that pass an unsigned integer to `OPT_INTEGER()`. There are three options: - We could adapt those users to instead pass a signed integer, but this would needlessly extend the range of accepted integer values. - We could convert them to use `OPT_MAGNITUDE()`, as it only accepts unsigned integers. But now we have the inconsistency that we also start to accept unit factors. - We could introduce `OPT_UNSIGNED()` as equivalent to `OPT_INTEGER()` so that it knows to only accept unsigned integers without unit suffix. Introducing a whole new option type feels a bit excessive. There also isn't really a good reason why `OPT_INTEGER()` cannot be extended to also accept unit factors: all valid values passed to such options cannot have a unit factors right now, so there wouldn't be any ambiguity. Refactor `OPT_INTEGER()` to use `git_parse_int()`, which knows to interpret unit factors. This removes the inconsistency between the signed and unsigned options so that we can easily fix up callsites that pass the wrong integer type right now. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Documentation/technical/api-parse-options.adoc | 6 ++++-- parse-options.c | 8 ++++---- t/t0040-parse-options.sh | 4 +++- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Documentation/technical/api-parse-options.adoc b/Documentation/technical/api-parse-options.adoc index 61fa6ee167833a..63acfb419bd23a 100644 --- a/Documentation/technical/api-parse-options.adoc +++ b/Documentation/technical/api-parse-options.adoc @@ -211,8 +211,10 @@ There are some macros to easily define options: Use of `--no-option` will clear the list of preceding values. `OPT_INTEGER(short, long, &int_var, description)`:: - Introduce an option with integer argument. - The integer is put into `int_var`. + Introduce an option with integer argument. The argument must be a + integer and may include a suffix of 'k', 'm' or 'g' to + scale the provided value by 1024, 1024^2 or 1024^3 respectively. + The scaled value is put into `int_var`. `OPT_MAGNITUDE(short, long, &unsigned_long_var, description)`:: Introduce an option with a size argument. The argument must be a diff --git a/parse-options.c b/parse-options.c index 35fbb3b0d633df..b287436e81a3f9 100644 --- a/parse-options.c +++ b/parse-options.c @@ -73,7 +73,7 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, enum opt_parsed flags, const char **argp) { - const char *s, *arg; + const char *arg; const int unset = flags & OPT_UNSET; int err; @@ -185,9 +185,9 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, if (!*arg) return error(_("%s expects a numerical value"), optname(opt, flags)); - *(int *)opt->value = strtol(arg, (char **)&s, 10); - if (*s) - return error(_("%s expects a numerical value"), + if (!git_parse_int(arg, opt->value)) + return error(_("%s expects an integer value" + " with an optional k/m/g suffix"), optname(opt, flags)); return 0; diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index 2fe3522305f64d..0c538c4b4376c7 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -111,7 +111,9 @@ test_expect_success 'OPT_BOOL() no negation #2' 'check_unknown_i18n --no-no-fear test_expect_success 'OPT_BOOL() positivation' 'check boolean: 0 -D --doubt' -test_expect_success 'OPT_INT() negative' 'check integer: -2345 -i -2345' +test_expect_success 'OPT_INTEGER() negative' 'check integer: -2345 -i -2345' +test_expect_success 'OPT_INTEGER() kilo' 'check integer: 239616 -i 234k' +test_expect_success 'OPT_INTEGER() negative kilo' 'check integer: -239616 -i -234k' test_expect_success 'OPT_MAGNITUDE() simple' ' check magnitude: 2345678 -m 2345678 From 785c17df7817df8512d2cb92cfc079ef0b4de27c Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:39 +0200 Subject: [PATCH 28/36] parse-options: rename `OPT_MAGNITUDE()` to `OPT_UNSIGNED()` With the preceding commit, `OPT_INTEGER()` has learned to support unit factors. Consequently, the major differencen between `OPT_INTEGER()` and `OPT_MAGNITUDE()` isn't the support of unit factors anymore, as both of them do support them now. Instead, the difference is that one handles signed and the other handles unsigned integers. Adapt the name of `OPT_MAGNITUDE()` accordingly by renaming it to `OPT_UNSIGNED()`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- .../technical/api-parse-options.adoc | 4 +- builtin/gc.c | 4 +- builtin/multi-pack-index.c | 2 +- builtin/pack-objects.c | 8 +-- builtin/repack.c | 8 +-- parse-options.c | 6 +-- parse-options.h | 6 +-- t/helper/test-parse-options.c | 6 +-- t/t0040-parse-options.sh | 50 +++++++++---------- 9 files changed, 47 insertions(+), 47 deletions(-) diff --git a/Documentation/technical/api-parse-options.adoc b/Documentation/technical/api-parse-options.adoc index 63acfb419bd23a..880eb94642587a 100644 --- a/Documentation/technical/api-parse-options.adoc +++ b/Documentation/technical/api-parse-options.adoc @@ -216,8 +216,8 @@ There are some macros to easily define options: scale the provided value by 1024, 1024^2 or 1024^3 respectively. The scaled value is put into `int_var`. -`OPT_MAGNITUDE(short, long, &unsigned_long_var, description)`:: - Introduce an option with a size argument. The argument must be a +`OPT_UNSIGNED(short, long, &unsigned_long_var, description)`:: + Introduce an option with an unsigned integer argument. The argument must be a non-negative integer and may include a suffix of 'k', 'm' or 'g' to scale the provided value by 1024, 1024^2 or 1024^3 respectively. The scaled value is put into `unsigned_long_var`. diff --git a/builtin/gc.c b/builtin/gc.c index 6707a26bc6e140..b32cf937cdf599 100644 --- a/builtin/gc.c +++ b/builtin/gc.c @@ -709,8 +709,8 @@ struct repository *repo UNUSED) .defval = (intptr_t)prune_expire_arg, }, OPT_BOOL(0, "cruft", &cfg.cruft_packs, N_("pack unreferenced objects separately")), - OPT_MAGNITUDE(0, "max-cruft-size", &cfg.max_cruft_size, - N_("with --cruft, limit the size of new cruft packs")), + OPT_UNSIGNED(0, "max-cruft-size", &cfg.max_cruft_size, + N_("with --cruft, limit the size of new cruft packs")), OPT_BOOL(0, "aggressive", &aggressive, N_("be more thorough (increased runtime)")), OPT_BOOL_F(0, "auto", &opts.auto_flag, N_("enable auto-gc mode"), PARSE_OPT_NOCOMPLETE), diff --git a/builtin/multi-pack-index.c b/builtin/multi-pack-index.c index 2a938466f53aaa..e4820fd721a8d5 100644 --- a/builtin/multi-pack-index.c +++ b/builtin/multi-pack-index.c @@ -245,7 +245,7 @@ static int cmd_multi_pack_index_repack(int argc, const char **argv, { struct option *options; static struct option builtin_multi_pack_index_repack_options[] = { - OPT_MAGNITUDE(0, "batch-size", &opts.batch_size, + OPT_UNSIGNED(0, "batch-size", &opts.batch_size, N_("during repack, collect pack-files of smaller size into a batch that is larger than this size")), OPT_BIT(0, "progress", &opts.flags, N_("force progress reporting"), MIDX_PROGRESS), diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index 79e1e6fb52b9b7..9328812e286ea2 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -4483,16 +4483,16 @@ int cmd_pack_objects(int argc, OPT_CALLBACK_F(0, "index-version", &pack_idx_opts, N_("[,]"), N_("write the pack index file in the specified idx format version"), PARSE_OPT_NONEG, option_parse_index_version), - OPT_MAGNITUDE(0, "max-pack-size", &pack_size_limit, - N_("maximum size of each output pack file")), + OPT_UNSIGNED(0, "max-pack-size", &pack_size_limit, + N_("maximum size of each output pack file")), OPT_BOOL(0, "local", &local, N_("ignore borrowed objects from alternate object store")), OPT_BOOL(0, "incremental", &incremental, N_("ignore packed objects")), OPT_INTEGER(0, "window", &window, N_("limit pack window by objects")), - OPT_MAGNITUDE(0, "window-memory", &window_memory_limit, - N_("limit pack window by memory in addition to object limit")), + OPT_UNSIGNED(0, "window-memory", &window_memory_limit, + N_("limit pack window by memory in addition to object limit")), OPT_INTEGER(0, "depth", &depth, N_("maximum length of delta chain allowed in the resulting pack")), OPT_BOOL(0, "reuse-delta", &reuse_delta, diff --git a/builtin/repack.c b/builtin/repack.c index 75e3752353a27f..8bf9941b2c2ad8 100644 --- a/builtin/repack.c +++ b/builtin/repack.c @@ -1202,8 +1202,8 @@ int cmd_repack(int argc, PACK_CRUFT), OPT_STRING(0, "cruft-expiration", &cruft_expiration, N_("approxidate"), N_("with --cruft, expire objects older than this")), - OPT_MAGNITUDE(0, "max-cruft-size", &cruft_po_args.max_pack_size, - N_("with --cruft, limit the size of new cruft packs")), + OPT_UNSIGNED(0, "max-cruft-size", &cruft_po_args.max_pack_size, + N_("with --cruft, limit the size of new cruft packs")), OPT_BOOL('d', NULL, &delete_redundant, N_("remove redundant packs, and run git-prune-packed")), OPT_BOOL('f', NULL, &po_args.no_reuse_delta, @@ -1233,8 +1233,8 @@ int cmd_repack(int argc, N_("limits the maximum delta depth")), OPT_STRING(0, "threads", &opt_threads, N_("n"), N_("limits the maximum number of threads")), - OPT_MAGNITUDE(0, "max-pack-size", &po_args.max_pack_size, - N_("maximum size of each packfile")), + OPT_UNSIGNED(0, "max-pack-size", &po_args.max_pack_size, + N_("maximum size of each packfile")), OPT_PARSE_LIST_OBJECTS_FILTER(&po_args.filter_options), OPT_BOOL(0, "pack-kept-objects", &pack_kept_objects, N_("repack objects in packs marked with .keep")), diff --git a/parse-options.c b/parse-options.c index b287436e81a3f9..d23e587e98bd97 100644 --- a/parse-options.c +++ b/parse-options.c @@ -191,7 +191,7 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, optname(opt, flags)); return 0; - case OPTION_MAGNITUDE: + case OPTION_UNSIGNED: if (unset) { *(unsigned long *)opt->value = 0; return 0; @@ -656,7 +656,7 @@ static void show_negated_gitcomp(const struct option *opts, int show_all, case OPTION_STRING: case OPTION_FILENAME: case OPTION_INTEGER: - case OPTION_MAGNITUDE: + case OPTION_UNSIGNED: case OPTION_CALLBACK: case OPTION_BIT: case OPTION_NEGBIT: @@ -708,7 +708,7 @@ static int show_gitcomp(const struct option *opts, int show_all) case OPTION_STRING: case OPTION_FILENAME: case OPTION_INTEGER: - case OPTION_MAGNITUDE: + case OPTION_UNSIGNED: case OPTION_CALLBACK: if (opts->flags & PARSE_OPT_NOARG) break; diff --git a/parse-options.h b/parse-options.h index 997ffbee8050d7..14e4df1ee21479 100644 --- a/parse-options.h +++ b/parse-options.h @@ -25,7 +25,7 @@ enum parse_opt_type { /* options with arguments (usually) */ OPTION_STRING, OPTION_INTEGER, - OPTION_MAGNITUDE, + OPTION_UNSIGNED, OPTION_CALLBACK, OPTION_LOWLEVEL_CALLBACK, OPTION_FILENAME @@ -270,8 +270,8 @@ struct option { #define OPT_CMDMODE(s, l, v, h, i) OPT_CMDMODE_F(s, l, v, h, i, 0) #define OPT_INTEGER(s, l, v, h) OPT_INTEGER_F(s, l, v, h, 0) -#define OPT_MAGNITUDE(s, l, v, h) { \ - .type = OPTION_MAGNITUDE, \ +#define OPT_UNSIGNED(s, l, v, h) { \ + .type = OPTION_UNSIGNED, \ .short_name = (s), \ .long_name = (l), \ .value = (v), \ diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index 997f55fd45b38a..fc3e2861c26af3 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -6,7 +6,7 @@ static int boolean = 0; static int integer = 0; -static unsigned long magnitude = 0; +static unsigned long unsigned_integer = 0; static timestamp_t timestamp; static int abbrev = 7; static int verbose = -1; /* unspecified */ @@ -140,7 +140,7 @@ int cmd__parse_options(int argc, const char **argv) OPT_GROUP(""), OPT_INTEGER('i', "integer", &integer, "get a integer"), OPT_INTEGER('j', NULL, &integer, "get a integer, too"), - OPT_MAGNITUDE('m', "magnitude", &magnitude, "get a magnitude"), + OPT_UNSIGNED('u', "unsigned", &unsigned_integer, "get an unsigned integer"), OPT_SET_INT(0, "set23", &integer, "set integer to 23", 23), OPT_CMDMODE(0, "mode1", &integer, "set integer to 1 (cmdmode option)", 1), OPT_CMDMODE(0, "mode2", &integer, "set integer to 2 (cmdmode option)", 2), @@ -210,7 +210,7 @@ int cmd__parse_options(int argc, const char **argv) } show(&expect, &ret, "boolean: %d", boolean); show(&expect, &ret, "integer: %d", integer); - show(&expect, &ret, "magnitude: %lu", magnitude); + show(&expect, &ret, "unsigned: %lu", unsigned_integer); show(&expect, &ret, "timestamp: %"PRItime, timestamp); show(&expect, &ret, "string: %s", string ? string : "(not set)"); show(&expect, &ret, "abbrev: %d", abbrev); diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index 0c538c4b4376c7..65a11c8dbc8c86 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -23,7 +23,7 @@ usage: test-tool parse-options -i, --[no-]integer get a integer -j get a integer, too - -m, --magnitude get a magnitude + -u, --unsigned get an unsigned integer --[no-]set23 set integer to 23 --mode1 set integer to 1 (cmdmode option) --mode2 set integer to 2 (cmdmode option) @@ -115,30 +115,30 @@ test_expect_success 'OPT_INTEGER() negative' 'check integer: -2345 -i -2345' test_expect_success 'OPT_INTEGER() kilo' 'check integer: 239616 -i 234k' test_expect_success 'OPT_INTEGER() negative kilo' 'check integer: -239616 -i -234k' -test_expect_success 'OPT_MAGNITUDE() simple' ' - check magnitude: 2345678 -m 2345678 +test_expect_success 'OPT_UNSIGNED() simple' ' + check unsigned: 2345678 -u 2345678 ' -test_expect_success 'OPT_MAGNITUDE() kilo' ' - check magnitude: 239616 -m 234k +test_expect_success 'OPT_UNSIGNED() kilo' ' + check unsigned: 239616 -u 234k ' -test_expect_success 'OPT_MAGNITUDE() mega' ' - check magnitude: 104857600 -m 100m +test_expect_success 'OPT_UNSIGNED() mega' ' + check unsigned: 104857600 -u 100m ' -test_expect_success 'OPT_MAGNITUDE() giga' ' - check magnitude: 1073741824 -m 1g +test_expect_success 'OPT_UNSIGNED() giga' ' + check unsigned: 1073741824 -u 1g ' -test_expect_success 'OPT_MAGNITUDE() 3giga' ' - check magnitude: 3221225472 -m 3g +test_expect_success 'OPT_UNSIGNED() 3giga' ' + check unsigned: 3221225472 -u 3g ' cat >expect <<\EOF boolean: 2 integer: 1729 -magnitude: 16384 +unsigned: 16384 timestamp: 0 string: 123 abbrev: 7 @@ -149,7 +149,7 @@ file: prefix/my.file EOF test_expect_success 'short options' ' - test-tool parse-options -s123 -b -i 1729 -m 16k -b -vv -n -F my.file \ + test-tool parse-options -s123 -b -i 1729 -u 16k -b -vv -n -F my.file \ >output 2>output.err && test_cmp expect output && test_must_be_empty output.err @@ -158,7 +158,7 @@ test_expect_success 'short options' ' cat >expect <<\EOF boolean: 2 integer: 1729 -magnitude: 16384 +unsigned: 16384 timestamp: 0 string: 321 abbrev: 10 @@ -169,7 +169,7 @@ file: prefix/fi.le EOF test_expect_success 'long options' ' - test-tool parse-options --boolean --integer 1729 --magnitude 16k \ + test-tool parse-options --boolean --integer 1729 --unsigned 16k \ --boolean --string2=321 --verbose --verbose --no-dry-run \ --abbrev=10 --file fi.le --obsolete \ >output 2>output.err && @@ -181,7 +181,7 @@ test_expect_success 'abbreviate to something longer than SHA1 length' ' cat >expect <<-EOF && boolean: 0 integer: 0 - magnitude: 0 + unsigned: 0 timestamp: 0 string: (not set) abbrev: 100 @@ -255,7 +255,7 @@ test_expect_success 'superfluous value provided: cmdmode' ' cat >expect <<\EOF boolean: 1 integer: 13 -magnitude: 0 +unsigned: 0 timestamp: 0 string: 123 abbrev: 7 @@ -278,7 +278,7 @@ test_expect_success 'intermingled arguments' ' cat >expect <<\EOF boolean: 0 integer: 2 -magnitude: 0 +unsigned: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -345,7 +345,7 @@ cat >expect <<\EOF Callback: "four", 0 boolean: 5 integer: 4 -magnitude: 0 +unsigned: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -370,7 +370,7 @@ test_expect_success 'OPT_CALLBACK() and callback errors work' ' cat >expect <<\EOF boolean: 1 integer: 23 -magnitude: 0 +unsigned: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -449,7 +449,7 @@ test_expect_success 'OPT_NUMBER_CALLBACK() works' ' cat >expect <<\EOF boolean: 0 integer: 0 -magnitude: 0 +unsigned: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -773,14 +773,14 @@ test_expect_success 'subcommands are incompatible with KEEP_DASHDASH unless in c grep ^BUG err ' -test_expect_success 'negative magnitude' ' - test_must_fail test-tool parse-options --magnitude -1 >out 2>err && +test_expect_success 'negative unsigned' ' + test_must_fail test-tool parse-options --unsigned -1 >out 2>err && grep "non-negative integer" err && test_must_be_empty out ' -test_expect_success 'magnitude with units but no numbers' ' - test_must_fail test-tool parse-options --magnitude m >out 2>err && +test_expect_success 'unsigned with units but no numbers' ' + test_must_fail test-tool parse-options --unsigned m >out 2>err && grep "non-negative integer" err && test_must_be_empty out ' From 09705696f763bac370ac74926bef137eb712c0c8 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:40 +0200 Subject: [PATCH 29/36] parse-options: introduce precision handling for `OPTION_INTEGER` The `OPTION_INTEGER` option type accepts a signed integer. The type of the underlying integer is a simple `int`, which restricts the range of values accepted by such options. But there is a catch: because the caller provides a pointer to the value via the `.value` field, which is a simple void pointer. This has two consequences: - There is no check whether the passed value is sufficiently long to store the entire range of `int`. This can lead to integer wraparound in the best case and out-of-bounds writes in the worst case. - Even when a caller knows that they want to store a value larger than `INT_MAX` they don't have a way to do so. In practice this doesn't tend to be a huge issue because users typically don't end up passing huge values to most commands. But the parsing logic is demonstrably broken, and it is too easy to get the calling convention wrong. Improve the situation by introducing a new `precision` field into the structure. This field gets assigned automatically by `OPT_INTEGER_F()` and tracks the size of the passed value. Like this it becomes possible for the caller to pass arbitrarily-sized integers and the underlying logic knows to handle it correctly by doing range checks. Furthermore, convert the code to use `strtoimax()` intstead of `strtol()` so that we can also parse values larger than `LONG_MAX`. Note that we do not yet assert signedness of the passed variable, which is another source of bugs. This will be handled in a subsequent commit. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- builtin/fmt-merge-msg.c | 2 ++ builtin/merge.c | 1 + builtin/show-branch.c | 1 + builtin/tag.c | 1 + parse-options.c | 52 ++++++++++++++++++++++++++--------- parse-options.h | 6 ++++ t/helper/test-parse-options.c | 3 ++ t/t0040-parse-options.sh | 23 +++++++++++++++- 8 files changed, 75 insertions(+), 14 deletions(-) diff --git a/builtin/fmt-merge-msg.c b/builtin/fmt-merge-msg.c index 240cdb474bc49b..3b6aac2cf7faab 100644 --- a/builtin/fmt-merge-msg.c +++ b/builtin/fmt-merge-msg.c @@ -24,6 +24,7 @@ int cmd_fmt_merge_msg(int argc, .type = OPTION_INTEGER, .long_name = "log", .value = &shortlog_len, + .precision = sizeof(shortlog_len), .argh = N_("n"), .help = N_("populate log with at most entries from shortlog"), .flags = PARSE_OPT_OPTARG, @@ -33,6 +34,7 @@ int cmd_fmt_merge_msg(int argc, .type = OPTION_INTEGER, .long_name = "summary", .value = &shortlog_len, + .precision = sizeof(shortlog_len), .argh = N_("n"), .help = N_("alias for --log (deprecated)"), .flags = PARSE_OPT_OPTARG | PARSE_OPT_HIDDEN, diff --git a/builtin/merge.c b/builtin/merge.c index 21787d45165e6d..9ab10c7db0a637 100644 --- a/builtin/merge.c +++ b/builtin/merge.c @@ -254,6 +254,7 @@ static struct option builtin_merge_options[] = { .type = OPTION_INTEGER, .long_name = "log", .value = &shortlog_len, + .precision = sizeof(shortlog_len), .argh = N_("n"), .help = N_("add (at most ) entries from shortlog to merge commit message"), .flags = PARSE_OPT_OPTARG, diff --git a/builtin/show-branch.c b/builtin/show-branch.c index dab37019d29210..b549d8c3f5b860 100644 --- a/builtin/show-branch.c +++ b/builtin/show-branch.c @@ -671,6 +671,7 @@ int cmd_show_branch(int ac, .type = OPTION_INTEGER, .long_name = "more", .value = &extra, + .precision = sizeof(extra), .argh = N_("n"), .help = N_("show more commits after the common ancestor"), .flags = PARSE_OPT_OPTARG, diff --git a/builtin/tag.c b/builtin/tag.c index b266f12bb48d12..7597d93c71bd95 100644 --- a/builtin/tag.c +++ b/builtin/tag.c @@ -483,6 +483,7 @@ int cmd_tag(int argc, .type = OPTION_INTEGER, .short_name = 'n', .value = &filter.lines, + .precision = sizeof(filter.lines), .argh = N_("n"), .help = N_("print lines of each tag message"), .flags = PARSE_OPT_OPTARG, diff --git a/parse-options.c b/parse-options.c index d23e587e98bd97..768718a397242e 100644 --- a/parse-options.c +++ b/parse-options.c @@ -172,25 +172,51 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, return (*opt->ll_callback)(p, opt, p_arg, p_unset); } case OPTION_INTEGER: + { + intmax_t upper_bound = INTMAX_MAX >> (bitsizeof(intmax_t) - CHAR_BIT * opt->precision); + intmax_t lower_bound = -upper_bound - 1; + intmax_t value; + if (unset) { - *(int *)opt->value = 0; - return 0; - } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { - *(int *)opt->value = opt->defval; - return 0; - } - if (get_arg(p, opt, flags, &arg)) + value = 0; + } else if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + value = opt->defval; + } else if (get_arg(p, opt, flags, &arg)) { return -1; - if (!*arg) + } else if (!*arg) { return error(_("%s expects a numerical value"), optname(opt, flags)); - if (!git_parse_int(arg, opt->value)) - return error(_("%s expects an integer value" - " with an optional k/m/g suffix"), + } else if (!git_parse_signed(arg, &value, upper_bound)) { + if (errno == ERANGE) + return error(_("value %s for %s not in range [%"PRIdMAX",%"PRIdMAX"]"), + arg, optname(opt, flags), lower_bound, upper_bound); + + return error(_("%s expects an integer value with an optional k/m/g suffix"), optname(opt, flags)); - return 0; + } + + if (value < lower_bound) + return error(_("value %s for %s not in range [%"PRIdMAX",%"PRIdMAX"]"), + arg, optname(opt, flags), lower_bound, upper_bound); + switch (opt->precision) { + case 1: + *(int8_t *)opt->value = value; + return 0; + case 2: + *(int16_t *)opt->value = value; + return 0; + case 4: + *(int32_t *)opt->value = value; + return 0; + case 8: + *(int64_t *)opt->value = value; + return 0; + default: + BUG("invalid precision for option %s", + optname(opt, flags)); + } + } case OPTION_UNSIGNED: if (unset) { *(unsigned long *)opt->value = 0; diff --git a/parse-options.h b/parse-options.h index 14e4df1ee21479..4c430c7273c13e 100644 --- a/parse-options.h +++ b/parse-options.h @@ -92,6 +92,10 @@ typedef int parse_opt_subcommand_fn(int argc, const char **argv, * `value`:: * stores pointers to the values to be filled. * + * `precision`:: + * precision of the integer pointed to by `value` in number of bytes. Should + * typically be its `sizeof()`. + * * `argh`:: * token to explain the kind of argument this option wants. Does not * begin in capital letter, and does not end with a full stop. @@ -151,6 +155,7 @@ struct option { int short_name; const char *long_name; void *value; + size_t precision; const char *argh; const char *help; @@ -214,6 +219,7 @@ struct option { .short_name = (s), \ .long_name = (l), \ .value = (v), \ + .precision = sizeof(*v), \ .argh = N_("n"), \ .help = (h), \ .flags = (f), \ diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index fc3e2861c26af3..3689aee831521f 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -120,6 +120,7 @@ int cmd__parse_options(int argc, const char **argv) }; struct string_list expect = STRING_LIST_INIT_NODUP; struct string_list list = STRING_LIST_INIT_NODUP; + int16_t i16 = 0; struct option options[] = { OPT_BOOL(0, "yes", &boolean, "get a boolean"), @@ -139,6 +140,7 @@ int cmd__parse_options(int argc, const char **argv) OPT_NEGBIT(0, "neg-or4", &boolean, "same as --no-or4", 4), OPT_GROUP(""), OPT_INTEGER('i', "integer", &integer, "get a integer"), + OPT_INTEGER(0, "i16", &i16, "get a 16 bit integer"), OPT_INTEGER('j', NULL, &integer, "get a integer, too"), OPT_UNSIGNED('u', "unsigned", &unsigned_integer, "get an unsigned integer"), OPT_SET_INT(0, "set23", &integer, "set integer to 23", 23), @@ -210,6 +212,7 @@ int cmd__parse_options(int argc, const char **argv) } show(&expect, &ret, "boolean: %d", boolean); show(&expect, &ret, "integer: %d", integer); + show(&expect, &ret, "i16: %"PRIdMAX, (intmax_t) i16); show(&expect, &ret, "unsigned: %lu", unsigned_integer); show(&expect, &ret, "timestamp: %"PRItime, timestamp); show(&expect, &ret, "string: %s", string ? string : "(not set)"); diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index 65a11c8dbc8c86..be785547eaddea 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -22,6 +22,7 @@ usage: test-tool parse-options -i, --[no-]integer get a integer + --[no-]i16 get a 16 bit integer -j get a integer, too -u, --unsigned get an unsigned integer --[no-]set23 set integer to 23 @@ -138,6 +139,7 @@ test_expect_success 'OPT_UNSIGNED() 3giga' ' cat >expect <<\EOF boolean: 2 integer: 1729 +i16: 0 unsigned: 16384 timestamp: 0 string: 123 @@ -158,6 +160,7 @@ test_expect_success 'short options' ' cat >expect <<\EOF boolean: 2 integer: 1729 +i16: 9000 unsigned: 16384 timestamp: 0 string: 321 @@ -169,7 +172,7 @@ file: prefix/fi.le EOF test_expect_success 'long options' ' - test-tool parse-options --boolean --integer 1729 --unsigned 16k \ + test-tool parse-options --boolean --integer 1729 --i16 9000 --unsigned 16k \ --boolean --string2=321 --verbose --verbose --no-dry-run \ --abbrev=10 --file fi.le --obsolete \ >output 2>output.err && @@ -181,6 +184,7 @@ test_expect_success 'abbreviate to something longer than SHA1 length' ' cat >expect <<-EOF && boolean: 0 integer: 0 + i16: 0 unsigned: 0 timestamp: 0 string: (not set) @@ -255,6 +259,7 @@ test_expect_success 'superfluous value provided: cmdmode' ' cat >expect <<\EOF boolean: 1 integer: 13 +i16: 0 unsigned: 0 timestamp: 0 string: 123 @@ -278,6 +283,7 @@ test_expect_success 'intermingled arguments' ' cat >expect <<\EOF boolean: 0 integer: 2 +i16: 0 unsigned: 0 timestamp: 0 string: (not set) @@ -345,6 +351,7 @@ cat >expect <<\EOF Callback: "four", 0 boolean: 5 integer: 4 +i16: 0 unsigned: 0 timestamp: 0 string: (not set) @@ -370,6 +377,7 @@ test_expect_success 'OPT_CALLBACK() and callback errors work' ' cat >expect <<\EOF boolean: 1 integer: 23 +i16: 0 unsigned: 0 timestamp: 0 string: (not set) @@ -449,6 +457,7 @@ test_expect_success 'OPT_NUMBER_CALLBACK() works' ' cat >expect <<\EOF boolean: 0 integer: 0 +i16: 0 unsigned: 0 timestamp: 0 string: (not set) @@ -785,4 +794,16 @@ test_expect_success 'unsigned with units but no numbers' ' test_must_be_empty out ' +test_expect_success 'i16 limits range' ' + test-tool parse-options --i16 32767 >out && + test_grep "i16: 32767" out && + test_must_fail test-tool parse-options --i16 32768 2>err && + test_grep "value 32768 for option .i16. not in range \[-32768,32767\]" err && + + test-tool parse-options --i16 -32768 >out && + test_grep "i16: -32768" out && + test_must_fail test-tool parse-options --i16 -32769 2>err && + test_grep "value -32769 for option .i16. not in range \[-32768,32767\]" err +' + test_done From bc288c59298f199348418ca08322046c67c9a0a2 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:41 +0200 Subject: [PATCH 30/36] parse-options: introduce precision handling for `OPTION_UNSIGNED` This commit is the equivalent to the preceding commit, but instead of introducing precision handling for `OPTION_INTEGER` we introduce it for `OPTION_UNSIGNED`. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- parse-options.c | 48 +++++++++++++++++++++++++++-------- parse-options.h | 1 + parse.c | 2 +- parse.h | 1 + t/helper/test-parse-options.c | 3 +++ t/t0040-parse-options.sh | 18 ++++++++++++- 6 files changed, 60 insertions(+), 13 deletions(-) diff --git a/parse-options.c b/parse-options.c index 768718a397242e..a9a39ecaef6c36 100644 --- a/parse-options.c +++ b/parse-options.c @@ -197,7 +197,7 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, if (value < lower_bound) return error(_("value %s for %s not in range [%"PRIdMAX",%"PRIdMAX"]"), - arg, optname(opt, flags), lower_bound, upper_bound); + arg, optname(opt, flags), (intmax_t)lower_bound, (intmax_t)upper_bound); switch (opt->precision) { case 1: @@ -218,21 +218,47 @@ static enum parse_opt_result do_get_value(struct parse_opt_ctx_t *p, } } case OPTION_UNSIGNED: + { + uintmax_t upper_bound = UINTMAX_MAX >> (bitsizeof(uintmax_t) - CHAR_BIT * opt->precision); + uintmax_t value; + if (unset) { - *(unsigned long *)opt->value = 0; - return 0; - } - if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { - *(unsigned long *)opt->value = opt->defval; - return 0; - } - if (get_arg(p, opt, flags, &arg)) + value = 0; + } else if (opt->flags & PARSE_OPT_OPTARG && !p->opt) { + value = opt->defval; + } else if (get_arg(p, opt, flags, &arg)) { return -1; - if (!git_parse_ulong(arg, opt->value)) + } else if (!*arg) { + return error(_("%s expects a numerical value"), + optname(opt, flags)); + } else if (!git_parse_unsigned(arg, &value, upper_bound)) { + if (errno == ERANGE) + return error(_("value %s for %s not in range [%"PRIdMAX",%"PRIdMAX"]"), + arg, optname(opt, flags), (uintmax_t) 0, upper_bound); + return error(_("%s expects a non-negative integer value" " with an optional k/m/g suffix"), optname(opt, flags)); - return 0; + } + + switch (opt->precision) { + case 1: + *(uint8_t *)opt->value = value; + return 0; + case 2: + *(uint16_t *)opt->value = value; + return 0; + case 4: + *(uint32_t *)opt->value = value; + return 0; + case 8: + *(uint64_t *)opt->value = value; + return 0; + default: + BUG("invalid precision for option %s", + optname(opt, flags)); + } + } default: BUG("opt->type %d should not happen", opt->type); diff --git a/parse-options.h b/parse-options.h index 4c430c7273c13e..dc460a26ff1e4c 100644 --- a/parse-options.h +++ b/parse-options.h @@ -281,6 +281,7 @@ struct option { .short_name = (s), \ .long_name = (l), \ .value = (v), \ + .precision = sizeof(*v), \ .argh = N_("n"), \ .help = (h), \ .flags = PARSE_OPT_NONEG, \ diff --git a/parse.c b/parse.c index 3c47448ca675fb..48313571aab129 100644 --- a/parse.c +++ b/parse.c @@ -51,7 +51,7 @@ int git_parse_signed(const char *value, intmax_t *ret, intmax_t max) return 0; } -static int git_parse_unsigned(const char *value, uintmax_t *ret, uintmax_t max) +int git_parse_unsigned(const char *value, uintmax_t *ret, uintmax_t max) { if (value && *value) { char *end; diff --git a/parse.h b/parse.h index 6bb9a54d9ac8f7..ea32de9a91fbfb 100644 --- a/parse.h +++ b/parse.h @@ -2,6 +2,7 @@ #define PARSE_H int git_parse_signed(const char *value, intmax_t *ret, intmax_t max); +int git_parse_unsigned(const char *value, uintmax_t *ret, uintmax_t max); int git_parse_ssize_t(const char *, ssize_t *); int git_parse_ulong(const char *, unsigned long *); int git_parse_int(const char *value, int *ret); diff --git a/t/helper/test-parse-options.c b/t/helper/test-parse-options.c index 3689aee831521f..f2663dd0c07279 100644 --- a/t/helper/test-parse-options.c +++ b/t/helper/test-parse-options.c @@ -120,6 +120,7 @@ int cmd__parse_options(int argc, const char **argv) }; struct string_list expect = STRING_LIST_INIT_NODUP; struct string_list list = STRING_LIST_INIT_NODUP; + uint16_t u16 = 0; int16_t i16 = 0; struct option options[] = { @@ -143,6 +144,7 @@ int cmd__parse_options(int argc, const char **argv) OPT_INTEGER(0, "i16", &i16, "get a 16 bit integer"), OPT_INTEGER('j', NULL, &integer, "get a integer, too"), OPT_UNSIGNED('u', "unsigned", &unsigned_integer, "get an unsigned integer"), + OPT_UNSIGNED(0, "u16", &u16, "get a 16 bit unsigned integer"), OPT_SET_INT(0, "set23", &integer, "set integer to 23", 23), OPT_CMDMODE(0, "mode1", &integer, "set integer to 1 (cmdmode option)", 1), OPT_CMDMODE(0, "mode2", &integer, "set integer to 2 (cmdmode option)", 2), @@ -214,6 +216,7 @@ int cmd__parse_options(int argc, const char **argv) show(&expect, &ret, "integer: %d", integer); show(&expect, &ret, "i16: %"PRIdMAX, (intmax_t) i16); show(&expect, &ret, "unsigned: %lu", unsigned_integer); + show(&expect, &ret, "u16: %"PRIuMAX, (uintmax_t) u16); show(&expect, &ret, "timestamp: %"PRItime, timestamp); show(&expect, &ret, "string: %s", string ? string : "(not set)"); show(&expect, &ret, "abbrev: %d", abbrev); diff --git a/t/t0040-parse-options.sh b/t/t0040-parse-options.sh index be785547eaddea..ca55ea8228c378 100755 --- a/t/t0040-parse-options.sh +++ b/t/t0040-parse-options.sh @@ -25,6 +25,7 @@ usage: test-tool parse-options --[no-]i16 get a 16 bit integer -j get a integer, too -u, --unsigned get an unsigned integer + --u16 get a 16 bit unsigned integer --[no-]set23 set integer to 23 --mode1 set integer to 1 (cmdmode option) --mode2 set integer to 2 (cmdmode option) @@ -141,6 +142,7 @@ boolean: 2 integer: 1729 i16: 0 unsigned: 16384 +u16: 0 timestamp: 0 string: 123 abbrev: 7 @@ -162,6 +164,7 @@ boolean: 2 integer: 1729 i16: 9000 unsigned: 16384 +u16: 32768 timestamp: 0 string: 321 abbrev: 10 @@ -173,7 +176,7 @@ EOF test_expect_success 'long options' ' test-tool parse-options --boolean --integer 1729 --i16 9000 --unsigned 16k \ - --boolean --string2=321 --verbose --verbose --no-dry-run \ + --u16 32k --boolean --string2=321 --verbose --verbose --no-dry-run \ --abbrev=10 --file fi.le --obsolete \ >output 2>output.err && test_must_be_empty output.err && @@ -186,6 +189,7 @@ test_expect_success 'abbreviate to something longer than SHA1 length' ' integer: 0 i16: 0 unsigned: 0 + u16: 0 timestamp: 0 string: (not set) abbrev: 100 @@ -261,6 +265,7 @@ boolean: 1 integer: 13 i16: 0 unsigned: 0 +u16: 0 timestamp: 0 string: 123 abbrev: 7 @@ -285,6 +290,7 @@ boolean: 0 integer: 2 i16: 0 unsigned: 0 +u16: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -353,6 +359,7 @@ boolean: 5 integer: 4 i16: 0 unsigned: 0 +u16: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -379,6 +386,7 @@ boolean: 1 integer: 23 i16: 0 unsigned: 0 +u16: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -459,6 +467,7 @@ boolean: 0 integer: 0 i16: 0 unsigned: 0 +u16: 0 timestamp: 0 string: (not set) abbrev: 7 @@ -806,4 +815,11 @@ test_expect_success 'i16 limits range' ' test_grep "value -32769 for option .i16. not in range \[-32768,32767\]" err ' +test_expect_success 'u16 limits range' ' + test-tool parse-options --u16 65535 >out && + test_grep "u16: 65535" out && + test_must_fail test-tool parse-options --u16 65536 2>err && + test_grep "value 65536 for option .u16. not in range \[0,65535\]" err +' + test_done From 791aeddfa2fdb9e830e24c50c97bb5e8bf3613e6 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Thu, 17 Apr 2025 12:49:42 +0200 Subject: [PATCH 31/36] parse-options: detect mismatches in integer signedness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It was reported that "t5620-backfill.sh" fails on s390x and sparc64 in a test that exercises the "--min-batch-size" command line option. The symptom was that the option didn't seem to have an effect: we didn't fetch objects with a batch size of 20, but instead fetched all objects at once. As it turns out, the root cause is that `--min-batch-size` uses `OPT_INTEGER()` to parse the command line option. While this macro expects the caller to pass a pointer to an integer, we instead pass a pointer to a `size_t`. This coincidentally works on most platforms, but it breaks apart on the mentioned platforms because they are big endian. This issue isn't specific to git-backfill(1): there are a couple of other places where we have the same type confusion going on. This indicates that the issue really is the interface that the parse-options subsystem provides -- it is simply too easy to get this wrong as there isn't any kind of compiler warning, and things just work on the most common systems. Address the systemic issue by introducing two new build asserts `BARF_UNLESS_SIGNED()` and `BARF_UNLESS_UNSIGNED()`. As the names already hint at, those macros will cause a compiler error when passed a value that is not signed or unsigned, respectively. Adapt `OPT_INTEGER()`, `OPT_UNSIGNED()` as well as `OPT_MAGNITUDE()` to use those asserts. This uncovers a small set of sites where we indeed have the same bug as in git-backfill(1). Adapt all of them to use the correct option. Reported-by: Todd Zullinger Reported-by: John Paul Adrian Glaubitz Helped-by: SZEDER Gábor Helped-by: Jeff King Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- apply.c | 4 ++-- builtin/backfill.c | 4 ++-- builtin/column.c | 2 +- builtin/grep.c | 4 ++-- git-compat-util.h | 7 +++++++ parse-options.h | 4 ++-- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/apply.c b/apply.c index f274a3794877dc..a850c7d75fe25e 100644 --- a/apply.c +++ b/apply.c @@ -5123,8 +5123,8 @@ int apply_parse_options(int argc, const char **argv, /* Think twice before adding "--nul" synonym to this */ OPT_SET_INT('z', NULL, &state->line_termination, N_("paths are separated with NUL character"), '\0'), - OPT_INTEGER('C', NULL, &state->p_context, - N_("ensure at least lines of context match")), + OPT_UNSIGNED('C', NULL, &state->p_context, + N_("ensure at least lines of context match")), OPT_CALLBACK(0, "whitespace", state, N_("action"), N_("detect new or modified lines that have whitespace errors"), apply_option_parse_whitespace), diff --git a/builtin/backfill.c b/builtin/backfill.c index 33e1ea2f84ff6b..d95d7a2d4d698b 100644 --- a/builtin/backfill.c +++ b/builtin/backfill.c @@ -123,8 +123,8 @@ int cmd_backfill(int argc, const char **argv, const char *prefix, struct reposit .sparse = 0, }; struct option options[] = { - OPT_INTEGER(0, "min-batch-size", &ctx.min_batch_size, - N_("Minimum number of objects to request at a time")), + OPT_UNSIGNED(0, "min-batch-size", &ctx.min_batch_size, + N_("Minimum number of objects to request at a time")), OPT_BOOL(0, "sparse", &ctx.sparse, N_("Restrict the missing objects to the current sparse-checkout")), OPT_END(), diff --git a/builtin/column.c b/builtin/column.c index 50314cc2559e55..ce6443d5fac84d 100644 --- a/builtin/column.c +++ b/builtin/column.c @@ -31,7 +31,7 @@ int cmd_column(int argc, struct option options[] = { OPT_STRING(0, "command", &real_command, N_("name"), N_("lookup config vars")), OPT_COLUMN(0, "mode", &colopts, N_("layout to use")), - OPT_INTEGER(0, "raw-mode", &colopts, N_("layout to use")), + OPT_UNSIGNED(0, "raw-mode", &colopts, N_("layout to use")), OPT_INTEGER(0, "width", &copts.width, N_("maximum width")), OPT_STRING(0, "indent", &copts.indent, N_("string"), N_("padding space on left border")), OPT_STRING(0, "nl", &copts.nl, N_("string"), N_("padding space on right border")), diff --git a/builtin/grep.c b/builtin/grep.c index c4869733e1bab6..f23a6f1dc865ca 100644 --- a/builtin/grep.c +++ b/builtin/grep.c @@ -983,9 +983,9 @@ int cmd_grep(int argc, OPT_CALLBACK('C', "context", &opt, N_("n"), N_("show context lines before and after matches"), context_callback), - OPT_INTEGER('B', "before-context", &opt.pre_context, + OPT_UNSIGNED('B', "before-context", &opt.pre_context, N_("show context lines before matches")), - OPT_INTEGER('A', "after-context", &opt.post_context, + OPT_UNSIGNED('A', "after-context", &opt.post_context, N_("show context lines after matches")), OPT_INTEGER(0, "threads", &num_threads, N_("use worker threads")), diff --git a/git-compat-util.h b/git-compat-util.h index cf733b38acdea0..1218fcf81a4b56 100644 --- a/git-compat-util.h +++ b/git-compat-util.h @@ -110,12 +110,19 @@ DISABLE_WARNING(-Wsign-compare) # define BARF_UNLESS_COPYABLE(dst, src) \ BUILD_ASSERT_OR_ZERO(__builtin_types_compatible_p(__typeof__(*(dst)), \ __typeof__(*(src)))) + +# define BARF_UNLESS_SIGNED(var) BUILD_ASSERT_OR_ZERO(((__typeof__(var)) -1) < 0) +# define BARF_UNLESS_UNSIGNED(var) BUILD_ASSERT_OR_ZERO(((__typeof__(var)) -1) > 0) #else # define BARF_UNLESS_AN_ARRAY(arr) 0 # define BARF_UNLESS_COPYABLE(dst, src) \ BUILD_ASSERT_OR_ZERO(0 ? ((*(dst) = *(src)), 0) : \ sizeof(*(dst)) == sizeof(*(src))) + +# define BARF_UNLESS_SIGNED(var) 0 +# define BARF_UNLESS_UNSIGNED(var) 0 #endif + /* * ARRAY_SIZE - get the number of elements in a visible array * @x: the array whose size you want. diff --git a/parse-options.h b/parse-options.h index dc460a26ff1e4c..91c3e3c29b3dda 100644 --- a/parse-options.h +++ b/parse-options.h @@ -218,7 +218,7 @@ struct option { .type = OPTION_INTEGER, \ .short_name = (s), \ .long_name = (l), \ - .value = (v), \ + .value = (v) + BARF_UNLESS_SIGNED(*(v)), \ .precision = sizeof(*v), \ .argh = N_("n"), \ .help = (h), \ @@ -280,7 +280,7 @@ struct option { .type = OPTION_UNSIGNED, \ .short_name = (s), \ .long_name = (l), \ - .value = (v), \ + .value = (v) + BARF_UNLESS_UNSIGNED(*(v)), \ .precision = sizeof(*v), \ .argh = N_("n"), \ .help = (h), \ From cdda67de0316ec29dfc1e290bb7f2154b7b95ee8 Mon Sep 17 00:00:00 2001 From: Ramsay Jones Date: Thu, 17 Apr 2025 00:18:34 +0100 Subject: [PATCH 32/36] config.mak.uname: set CSPRNG_METHOD to getrandom on Linux Commit 05cd988dce ("wrapper: add a helper to generate numbers from a CSPRNG", 2022-01-17) added a csprng_bytes() function which used one of several interfaces to provide a source of cryptographically secure pseudorandom numbers. The CSPRNG_METHOD make variable was provided to determine the choice of available 'backends' for the source of random bytes. Commit 05cd988dce did not set CSPRNG_METHOD in the Linux section of the config.mak.uname file, so it defaults to using '/dev/urandom' as the source of random bytes. The 'backend' values which could be used on Linux are 'arc4random', 'getrandom' or 'getentropy' ('openssl' is an option, but seems to be discouraged). The arc4random routines (arc4random_buf() is the one actually used) were added to glibc in version 2.36, while both getrandom() and getentropy() were included in 2.25. So, some of the more up-to-date distributions of Linux (eg Debian 12, Ubuntu 24.04) would be able to use the 'arc4random' setting. All currently supported distributions have glibc 2.25 or later (RHEL 8 has v2.28) and, therefore, have support for the 'getrandom' and 'getentropy' settings. The arc4random routines on the *BSDs (along with cygwin) implement the ChaCha20 stream cipher algorithm (see RFC8439) in userspace, rather than as a system call, and are thus somewhat faster (having avoided a context switch to the kernel). In contrast, on Linux all three functions are simple wrappers around the same kernel CSPRNG syscall. If the meson build system is used on a newer platform, then they will be configured to use 'arc4random', whereas the make build will currently default to using '/dev/urandom' on Linux. Since there is no advantage, in terms of performance, to the 'arc4random' setting, the 'getrandom' setting should be preferred from an availability perspective. (Also, the current uses of csprng_bytes() are not in any hot path). In order to set an appropriate default, set the CSPRNG_METHOD build variable to 'getrandom' in the Linux section of the 'config.mak.uname' file. Signed-off-by: Ramsay Jones Signed-off-by: Junio C Hamano --- config.mak.uname | 1 + 1 file changed, 1 insertion(+) diff --git a/config.mak.uname b/config.mak.uname index 330741eb5a1cdb..db22a8fb3113a3 100644 --- a/config.mak.uname +++ b/config.mak.uname @@ -50,6 +50,7 @@ ifeq ($(uname_S),Linux) HAVE_ALLOCA_H = YesPlease # override in config.mak if you have glibc >= 2.38 NO_STRLCPY = YesPlease + CSPRNG_METHOD = getrandom HAVE_PATHS_H = YesPlease LIBC_CONTAINS_LIBINTL = YesPlease HAVE_DEV_TTY = YesPlease From ee40e26e6999349b09e260c3bc8ab1f5397d8b1b Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Fri, 18 Apr 2025 10:54:59 -0700 Subject: [PATCH 33/36] t9811: fix misconversion of tests The previous commit started to insist TAG_F1_ONLY to be missing, which was not in the original. Let's not be overly eager in the conversion. Also, the other hunk in the commit introduced a shell syntax error, causing the test to fail. Fix it. Signed-off-by: Junio C Hamano --- t/t9811-git-p4-label-import.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/t/t9811-git-p4-label-import.sh b/t/t9811-git-p4-label-import.sh index 39856629c00657..7614dfbd953efe 100755 --- a/t/t9811-git-p4-label-import.sh +++ b/t/t9811-git-p4-label-import.sh @@ -97,7 +97,6 @@ test_expect_success 'two labels on the same changelist' ' git show-ref --verify refs/tags/TAG_F1_1 && git show-ref --verify refs/tags/TAG_F1_2 && - test_must_fail git show-ref --verify refs/tags/TAG_F1_ONLY && cd main && @@ -207,7 +206,7 @@ test_expect_success 'use git config to enable import/export of tags' ' git tag CFG_A_GIT_TAG && git p4 rebase --verbose && git p4 submit --verbose && - git show-ref --verify refs/tags/TAG_F1_1 && + git show-ref --verify refs/tags/TAG_F1_1 ) && ( cd "$cli" && From b0026daf1e60ebff7f1d4a840c158b3a05ecffcd Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 24 Apr 2025 16:10:47 -0700 Subject: [PATCH 34/36] ci: skip unavailable external software The ci/install-dependencies.sh script used in a very early phase of our CI jobs downloads Perforce, Git-LFS, and JGit, used for running the test scripts. The test framework is prepared to properly skip the tests that depend on these external software, but the CI script is unnecessarily strict (due to its use of "set -e" in ci/lib.sh) and fails the entire CI run before even starting to test the rest of the system. Notice a failure to download to any of these external software, but keep going. We need to be careful about cleaning after a failed wget, as a later part of the script that does: if type jgit >/dev/null 2>&1 then echo "$(tput setaf 6)JGit Version$(tput sgr0)" jgit version else echo >&2 "WARNING: JGit wasn't installed, see above for clues why" fi will (surprise!) succeed running "type jgit", and then fail with "jgit version", taking the whole thing down due to "set -e". Signed-off-by: Junio C Hamano --- ci/install-dependencies.sh | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/ci/install-dependencies.sh b/ci/install-dependencies.sh index 0df74610d063fb..e51304c3b0eed2 100755 --- a/ci/install-dependencies.sh +++ b/ci/install-dependencies.sh @@ -66,16 +66,29 @@ ubuntu-*|i386/ubuntu-*|debian-*) mkdir --parents "$CUSTOM_PATH" wget --quiet --directory-prefix="$CUSTOM_PATH" \ - "$P4WHENCE/bin.linux26x86_64/p4d" "$P4WHENCE/bin.linux26x86_64/p4" - chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" - - wget --quiet "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" + "$P4WHENCE/bin.linux26x86_64/p4d" \ + "$P4WHENCE/bin.linux26x86_64/p4" && + chmod a+x "$CUSTOM_PATH/p4d" "$CUSTOM_PATH/p4" || { + rm -f "$CUSTOM_PATH/p4" + rm -f "$CUSTOM_PATH/p4d" + echo >&2 "P4 download (optional) failed" + } + + wget --quiet \ + "$LFSWHENCE/git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" && tar -xzf "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" \ - -C "$CUSTOM_PATH" --strip-components=1 "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" - rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" - - wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" - chmod a+x "$CUSTOM_PATH/jgit" + -C "$CUSTOM_PATH" --strip-components=1 \ + "git-lfs-$LINUX_GIT_LFS_VERSION/git-lfs" && + rm "git-lfs-linux-amd64-$LINUX_GIT_LFS_VERSION.tar.gz" || { + rm -f "$CUSTOM_PATH/git-lfs" + echo >&2 "LFS download (optional) failed" + } + + wget --quiet "$JGITWHENCE" --output-document="$CUSTOM_PATH/jgit" && + chmod a+x "$CUSTOM_PATH/jgit" || { + rm -f "$CUSTOM_PATH/jgit" + echo >&2 "JGit download (optional) failed" + } ;; esac ;; From d50a5e8939abfc07c2ff97ae72e9330939b36ee0 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 24 Apr 2025 17:14:14 -0700 Subject: [PATCH 35/36] CI updates Ever since we issued 2.49, external forces broke our CI jobs in various ways, and we had to adjust our code to work them around. Backmerge them from the 'master' front to make it easier to test real changes to the maintenance track. Signed-off-by: Junio C Hamano From f65182a99e545d2f2bc22e6c1c2da192133b16a3 Mon Sep 17 00:00:00 2001 From: Junio C Hamano Date: Thu, 24 Apr 2025 17:27:03 -0700 Subject: [PATCH 36/36] The ninth batch Signed-off-by: Junio C Hamano --- Documentation/RelNotes/2.50.0.adoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/Documentation/RelNotes/2.50.0.adoc b/Documentation/RelNotes/2.50.0.adoc index e4b731e61d2c1a..3ab82ac3a2f153 100644 --- a/Documentation/RelNotes/2.50.0.adoc +++ b/Documentation/RelNotes/2.50.0.adoc @@ -112,6 +112,9 @@ Performance, Internal Implementation, Development Support etc. * Optimize the code to dedup references recorded in a bundle file. + * Update parse-options API to catch mistakes to pass address of an + integral variable of a wrong type/size. + Fixes since v2.49 ----------------- @@ -210,6 +213,17 @@ Fixes since v2.49 any common ancestor, now behaves as expected. (merge e7ef4be7c2 mh/left-right-limited later to maint). + * Document the convention to disable hooks altogether by setting the + hooksPath configuration variable to /dev/nulll + (merge 1b2eee94f1 ds/doc-disable-hooks later to maint). + + * Make sure outage of third-party sites that supply P4, Git-LFS, and + JGit we use for testing would not prevent our CI jobs from running + at all. + + * Various build tweaks, including CSPRNG selection on some platforms. + (merge cdda67de03 rj/build-tweaks later to maint). + * Other code cleanup, docfix, build fix, etc. (merge 227c4f33a0 ja/doc-block-delimiter-markup-fix later to maint). (merge 2bfd3b3685 ab/decorate-code-cleanup later to maint).