From 2b3d8794aa648bacd6de5d03ae89c8fe63ce57b1 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 27 Jan 2026 12:03:15 +0100 Subject: [PATCH 1/4] crt/cmds/release: add --local-run option avoid accessing remotes if the flag is set and the subcommand doesn't require remote access. affected subcommands: * start: * skip adding remote URLs. * skip checking if the release already exists on the remote. * create a local release branch from base_ref (which must exist locally). * do not push the created branch or tag to the remote. * list: * skip adding remote urls. * skip fetching from remotes. other subcommands will ignore the flag. Signed-off-by: Uwe Schwaeke --- crt/src/crt/cmds/__init__.py | 2 + crt/src/crt/cmds/crt.py | 11 +++ crt/src/crt/cmds/release.py | 140 ++++++++++++++++++++++---------- crt/src/crt/crtlib/git_utils.py | 12 +++ 4 files changed, 122 insertions(+), 43 deletions(-) diff --git a/crt/src/crt/cmds/__init__.py b/crt/src/crt/cmds/__init__.py index fee0dff..564932b 100644 --- a/crt/src/crt/cmds/__init__.py +++ b/crt/src/crt/cmds/__init__.py @@ -33,10 +33,12 @@ class Ctx: github_token: str | None patches_repo_path: Path | None + run_locally: bool def __init__(self) -> None: self.github_token = None self.patches_repo_path = None + self.run_locally = False pass_ctx = click.make_pass_decorator(Ctx, ensure=True) diff --git a/crt/src/crt/cmds/crt.py b/crt/src/crt/cmds/crt.py index ae943a2..0ab17fe 100644 --- a/crt/src/crt/cmds/crt.py +++ b/crt/src/crt/cmds/crt.py @@ -78,6 +78,15 @@ required=True, help="Path to CES patches git repository.", ) +@click.option( + "-l", + "--local-run", + "run_locally", + is_flag=True, + default=False, + required=False, + help="Run without accessing remotes.", +) @pass_ctx def cmd_crt( ctx: Ctx, @@ -85,6 +94,7 @@ def cmd_crt( verbose: bool, github_token: str | None, patches_repo_path: Path, + run_locally: bool, ) -> None: if verbose: set_verbose_logging() @@ -97,6 +107,7 @@ def cmd_crt( ctx.github_token = github_token ctx.patches_repo_path = patches_repo_path + ctx.run_locally = run_locally if debug or verbose: table = Table(show_header=False, show_lines=False, box=None) diff --git a/crt/src/crt/cmds/release.py b/crt/src/crt/cmds/release.py index df67262..6282534 100644 --- a/crt/src/crt/cmds/release.py +++ b/crt/src/crt/cmds/release.py @@ -33,9 +33,11 @@ git_branch_from, git_cleanup_repo, git_fetch_ref, + git_get_local_head, git_get_remote_ref, git_prepare_remote, git_push, + git_remote, git_reset_head, git_tag, ) @@ -44,8 +46,11 @@ from crt.crtlib.release import load_release, release_exists, store_release from . import ( + Ctx, console, + pass_ctx, perror, + pinfo, psuccess, pwarn, with_gh_token, @@ -65,6 +70,8 @@ def _prepare_release_repo( src_repo: str, dst_repo: str, token: str, + *, + run_locally: bool = False, ) -> None: try: git_cleanup_repo(ceph_repo_path) @@ -73,6 +80,10 @@ def _prepare_release_repo( perror(f"failed to cleanup ceph repo at '{ceph_repo_path}': {e}") raise _ExitError(errno.ENOTRECOVERABLE) from e + if run_locally: + # return because only thing we do next is to set the remotes and fetch from it. + return + try: _ = git_prepare_remote( ceph_repo_path, f"github.com/{src_repo}", src_repo, token @@ -92,27 +103,45 @@ def _prepare_release_branches( src_ref: str, dst_repo: str, dst_branch: str, + *, + run_locally: bool = False, ) -> None: try: - if git_get_remote_ref(ceph_repo_path, dst_branch, dst_repo): - perror(f"destination branch '{dst_branch}' already exists in '{dst_repo}'") - sys.exit(errno.EEXIST) + if run_locally: + if git_get_local_head(ceph_repo_path, dst_branch): + perror(f"destination branch '{dst_branch}' already exists locally") + sys.exit(errno.EEXIST) + else: + if git_get_remote_ref(ceph_repo_path, dst_branch, dst_repo): + perror( + f"destination branch '{dst_branch}' already exists in '{dst_repo}'" + ) + sys.exit(errno.EEXIST) except GitError as e: perror(f"failed to check for existing branch '{dst_branch}': {e}") raise _ExitError(errno.ENOTRECOVERABLE) from e is_tag = False - try: - _ = git_fetch_ref(ceph_repo_path, src_ref, dst_branch, src_repo) - except GitIsTagError: - logger.debug(f"source ref '{src_ref}' is a tag, fetching as branch") - is_tag = True - except GitFetchHeadNotFoundError: - perror(f"source ref '{src_ref}' not found in '{src_repo}'") - raise _ExitError(errno.ENOENT) from None - except GitError as e: - perror(f"failed to fetch source ref '{src_ref}': {e}") - raise _ExitError(errno.ENOTRECOVERABLE) from e + if run_locally: + try: + git_branch_from(ceph_repo_path, src_ref, dst_branch) + except GitError as e: + perror(f"failed to create branch from source ref '{src_ref}': {e}") + raise _ExitError(errno.ENOTRECOVERABLE) from e + else: + return + else: + try: + _ = git_fetch_ref(ceph_repo_path, src_ref, dst_branch, src_repo) + except GitIsTagError: + logger.debug(f"source ref '{src_ref}' is a tag, fetching as branch") + is_tag = True + except GitFetchHeadNotFoundError: + perror(f"source ref '{src_ref}' not found in '{src_repo}'") + raise _ExitError(errno.ENOENT) from None + except GitError as e: + perror(f"failed to fetch source ref '{src_ref}': {e}") + raise _ExitError(errno.ENOTRECOVERABLE) from e if is_tag: try: @@ -190,7 +219,9 @@ def cmd_release(): @click.argument("release_name", type=str, required=True, metavar="NAME") @with_patches_repo_path @with_gh_token +@pass_ctx def cmd_release_start( + ctx: Ctx, gh_token: str, patches_repo_path: Path, ceph_repo_path: Path, @@ -276,6 +307,7 @@ def cmd_release_start( base_ref_repo, dst_repo, gh_token, + run_locally=ctx.run_locally, ) except _ExitError as e: progress.stop_error() @@ -288,7 +320,9 @@ def cmd_release_start( progress.done_task() progress.new_task("prepare release branches") - if git_get_remote_ref(ceph_repo_path, f"release/{release_name}", dst_repo): + if not ctx.run_locally and git_get_remote_ref( + ceph_repo_path, f"release/{release_name}", dst_repo + ): progress.stop_error() perror(f"release '{release_name}' already marked released in '{dst_repo}'") sys.exit(errno.EEXIST) @@ -300,6 +334,7 @@ def cmd_release_start( base_ref, dst_repo, release_base_branch, + run_locally=ctx.run_locally, ) except _ExitError as e: progress.stop_error() @@ -309,16 +344,19 @@ def cmd_release_start( perror(f"failed to prepare release branches: {e}") sys.exit(errno.ENOTRECOVERABLE) - try: - _ = git_push(ceph_repo_path, release_base_branch, dst_repo) - except GitError as e: - progress.stop_error() - perror(f"failed to push release branch '{release_base_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - except Exception as e: - progress.stop_error() - perror(f"unexpected error pushing release branch '{release_base_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) + if not ctx.run_locally: + try: + _ = git_push(ceph_repo_path, release_base_branch, dst_repo) + except GitError as e: + progress.stop_error() + perror(f"failed to push release branch '{release_base_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + except Exception as e: + progress.stop_error() + perror( + f"unexpected error pushing release branch '{release_base_branch}': {e}" + ) + sys.exit(errno.ENOTRECOVERABLE) try: git_tag( @@ -326,7 +364,7 @@ def cmd_release_start( release_base_tag, release_base_branch, msg=f"Base release for {release_name}", - push_to=dst_repo, + push_to=dst_repo if not ctx.run_locally else None, ) except GitError as e: progress.stop_error() @@ -364,6 +402,7 @@ def cmd_release_start( summary_table.add_row("From Base Reference", f"{base_ref} from {base_ref_repo}") summary_table.add_row("Release base branch", release_base_branch) summary_table.add_row("Release base tag", release_base_tag) + summary_table.add_row("Is local", str(ctx.run_locally)) console.print(Padding(summary_table, (1, 0, 1, 0))) @@ -399,25 +438,45 @@ def cmd_release_start( ) @with_patches_repo_path @with_gh_token +@pass_ctx def cmd_release_list( - gh_token: str, patches_repo_path: Path, ceph_repo_path: Path, dst_repo: str + ctx: Ctx, + gh_token: str, + patches_repo_path: Path, + ceph_repo_path: Path, + dst_repo: str, ) -> None: progress = CRTProgress(console) progress.start() - progress.new_task("prepare remote") + table = Table(show_header=True, show_lines=True, box=rich.box.HORIZONTALS) + table.add_column("Name", justify="left", style="bold cyan", no_wrap=True) + table.add_column("Base", justify="left", style="magenta", no_wrap=True) + table.add_column("Status", justify="left", no_wrap=True) - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{dst_repo}", dst_repo, gh_token - ) - except GitError as e: - perror(f"unable to prepare remote repository '{dst_repo}': {e}") - progress.stop_error() - sys.exit(errno.ENOTRECOVERABLE) + if ctx.run_locally: + progress.new_task("get remote") + if not (remote := git_remote(ceph_repo_path, dst_repo)): + pinfo(f"remote {dst_repo} doesn't exist locally") + console.print(Padding(table, (1, 0, 1, 0))) + progress.done_task() + progress.stop() + return + progress.done_task() + progress.stop() + else: + progress.new_task("prepare remote") + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{dst_repo}", dst_repo, gh_token + ) + except GitError as e: + perror(f"unable to prepare remote repository '{dst_repo}': {e}") + progress.stop_error() + sys.exit(errno.ENOTRECOVERABLE) - progress.done_task() - progress.stop() + progress.done_task() + progress.stop() remote_releases: list[str] = [] remote_base_releases: list[str] = [] @@ -461,11 +520,6 @@ def cmd_release_list( if r not in remote_releases: not_released.append(r) - table = Table(show_header=True, show_lines=True, box=rich.box.HORIZONTALS) - table.add_column("Name", justify="left", style="bold cyan", no_wrap=True) - table.add_column("Base", justify="left", style="magenta", no_wrap=True) - table.add_column("Status", justify="left", no_wrap=True) - for r in remote_releases: rel = releases_meta.get(r) table.add_row( diff --git a/crt/src/crt/crtlib/git_utils.py b/crt/src/crt/crtlib/git_utils.py index e96ac3b..bca604a 100644 --- a/crt/src/crt/crtlib/git_utils.py +++ b/crt/src/crt/crtlib/git_utils.py @@ -382,6 +382,18 @@ def git_prepare_remote( return remote +def git_remote(repo_path: Path, remote_name: str) -> git.Remote | None: + logger.info(f"get remote '{remote_name}'") + + repo = git.Repo(repo_path) + try: + return repo.remote(remote_name) + except ValueError: + logger.debug(f"remote '{remote_name}' doesn't exist") + + return None + + def _get_remote_ref_name( remote_name: str, remote_ref: str, *, ref_name: str | None = None ) -> tuple[str, str] | None: From 9f3a0476083d0a2e9a9d27c22ffb1402355f1259 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 27 Jan 2026 14:23:26 +0100 Subject: [PATCH 2/4] crt/cmds/manifest: add --local-run option don't access remotes if the flag is set and the subcommand doesn't require remote access. affected subcommands: * patchset add: * don't add remote urls and don't fetch patch branch (assume branch exists locally) * validate: * don't add remote urls and don't fetch from remote other subcommands will ignore the flag Signed-off-by: Uwe Schwaeke --- crt/src/crt/cmds/manifest.py | 93 +++++++++++++++++++++++++++++++-- crt/src/crt/crtlib/apply.py | 11 ++-- crt/src/crt/crtlib/git_utils.py | 37 +++++++++---- crt/src/crt/crtlib/manifest.py | 87 ++++++++++++++++++------------ 4 files changed, 179 insertions(+), 49 deletions(-) diff --git a/crt/src/crt/cmds/manifest.py b/crt/src/crt/cmds/manifest.py index ef25021..e8729e3 100644 --- a/crt/src/crt/cmds/manifest.py +++ b/crt/src/crt/cmds/manifest.py @@ -38,7 +38,15 @@ NoSuchManifestError, ) from crt.crtlib.errors.patchset import NoSuchPatchSetError, PatchSetError -from crt.crtlib.errors.release import NoSuchReleaseError +from crt.crtlib.errors.release import NoSuchReleaseError, ReleaseError +from crt.crtlib.git_utils import ( + GitError, + git_get_remote_ref, + git_prepare_remote, + git_push, + git_remote, + git_tag_exists_in_remote, +) from crt.crtlib.github import gh_get_pr from crt.crtlib.manifest import ( ManifestExecuteResult, @@ -646,7 +654,12 @@ def _check_repo(repo_path: Path, what: str) -> None: progress.new_task("applying patch set to manifest") try: _, added, skipped = patches_apply_to_manifest( - manifest, patchset, ceph_repo_path, patches_repo_path, ctx.github_token + manifest, + patchset, + ceph_repo_path, + patches_repo_path, + ctx.github_token, + run_locally=ctx.run_locally, ) except (ApplyError, Exception) as e: perror(f"unable to apply to manifest: {e}") @@ -683,6 +696,7 @@ def _manifest_execute( ceph_repo_path: Path, patches_repo_path: Path, no_cleanup: bool = True, + run_locally: bool = False, progress: CRTProgress, ) -> tuple[ManifestExecuteResult, RenderableType]: """ @@ -694,7 +708,12 @@ def _manifest_execute( try: res = manifest_execute( - manifest, ceph_repo_path, patches_repo_path, token, no_cleanup=no_cleanup + manifest, + ceph_repo_path, + patches_repo_path, + token, + no_cleanup=no_cleanup, + run_locally=run_locally, ) except ApplyConflictError as e: progress.stop_error() @@ -870,6 +889,7 @@ def cmd_manifest_validate( ceph_repo_path=ceph_repo_path, patches_repo_path=patches_repo_path, no_cleanup=no_cleanup, + run_locally=ctx.run_locally, progress=progress, ) progress.stop() @@ -904,6 +924,15 @@ def cmd_manifest_validate( default="release-dev", help="Prefix to use for published branch.", ) +@click.option( + "-r", + "--release", + "release_name", + type=str, + required=True, + metavar="RELEASE_NAME", + help="Release associated with the manifest.", +) @with_patches_repo_path @pass_ctx def cmd_manifest_publish( @@ -912,6 +941,7 @@ def cmd_manifest_publish( ceph_repo_path: Path, release_branch_prefix: str, manifest_name_or_uuid: str, + release_name: str, ) -> None: """ Publish a manifest. @@ -945,6 +975,27 @@ def cmd_manifest_publish( progress = CRTProgress(console) progress.start() + try: + _prepare_release_repo( + ceph_repo_path, + patches_repo_path, + manifest, + release_name, + ctx.github_token, + ) + except NoSuchReleaseError: + msg = f"release {release_name} does not exist" + logger.error(msg) + sys.exit(errno.ENOENT) + except ReleaseError as e: + msg = f"can't load release {release_name}: '{e}'" + logger.error(msg) + sys.exit(errno.EBADMSG) + except GitError as e: + msg = f"can't publish manifest {manifest.name}: '{e}'" + logger.error(msg) + sys.exit(e.ec if e.ec else errno.ENOTRECOVERABLE) + execute_res, execute_summary = _manifest_execute( manifest, token=ctx.github_token, @@ -1084,3 +1135,39 @@ def cmd_manifest_update(patches_repo_path: Path, manifest_name_or_uuid: str) -> sys.exit(errno.ENOTRECOVERABLE) psuccess(f"updated manifest '{manifest_name_or_uuid}' on-disk representation") + + +def _prepare_release_repo( + ceph_repo_path: Path, + ces_patch_path: Path, + manifest: ReleaseManifest, + release_name: str, + token: str, +) -> None: + try: + release_branch_name = manifest.base_ref + release_tag_name = release_branch_name.replace("/", "-") + remote_name = manifest.dst_repo + if ( + git_remote(ceph_repo_path, remote_name) + and git_get_remote_ref(ceph_repo_path, release_branch_name, remote_name) + and git_tag_exists_in_remote(ceph_repo_path, remote_name, release_tag_name) + ): + pinfo("release repo already prepared") + return + + release = load_release(ces_patch_path, release_name) + release_repo_name = release.base_repo + + git_prepare_remote( + ceph_repo_path, f"github.com/{release_repo_name}", release_repo_name, token + ) + if remote_name != release_repo_name: + git_prepare_remote( + ceph_repo_path, f"github.com/{remote_name}", remote_name, token + ) + git_push(ceph_repo_path, release_branch_name, remote_name) + git_push(ceph_repo_path, release_tag_name, remote_name) + except GitError as e: + perror(f"unable to prepare release repository: {e}") + sys.exit(e.ec if e.ec else errno.ENOTRECOVERABLE) diff --git a/crt/src/crt/crtlib/apply.py b/crt/src/crt/crtlib/apply.py index 26f2253..65a21d3 100644 --- a/crt/src/crt/crtlib/apply.py +++ b/crt/src/crt/crtlib/apply.py @@ -146,6 +146,7 @@ def apply_manifest( token: str, *, no_cleanup: bool = False, + run_locally: bool = False, ) -> tuple[bool, list[ManifestPatchEntry], list[ManifestPatchEntry]]: ceph_repo = git.Repo(ceph_repo_path) @@ -191,9 +192,10 @@ def _apply_patches( try: _prepare_repo(ceph_repo_path) repo_name = f"{manifest.base_ref_org}/{manifest.base_ref_repo}" - _ = git_prepare_remote( - ceph_repo_path, f"github.com/{repo_name}", repo_name, token - ) + if not run_locally: + _ = git_prepare_remote( + ceph_repo_path, f"github.com/{repo_name}", repo_name, token + ) except ApplyError as e: logger.error(e) raise e from None @@ -232,6 +234,8 @@ def patches_apply_to_manifest( ceph_repo_path: Path, patches_repo_path: Path, token: str, + *, + run_locally: bool = False, ) -> tuple[bool, list[ManifestPatchEntry], list[ManifestPatchEntry]]: manifest = orig_manifest.model_copy(deep=True) if not manifest.add_patches(patch): @@ -247,4 +251,5 @@ def patches_apply_to_manifest( target_branch, token, no_cleanup=False, + run_locally=run_locally, ) diff --git a/crt/src/crt/crtlib/git_utils.py b/crt/src/crt/crtlib/git_utils.py index bca604a..cdccabc 100644 --- a/crt/src/crt/crtlib/git_utils.py +++ b/crt/src/crt/crtlib/git_utils.py @@ -670,17 +670,21 @@ def git_branch_delete(repo_path: Path, branch: str) -> None: def git_push( repo_path: Path, - branch: str, + ref: str, remote_name: str, *, - branch_to: str | None = None, + ref_to: str | None = None, ) -> tuple[bool, list[str], list[str]]: - dst_branch = branch_to if branch_to else branch + """Pushes either a local head of branch or a local tag to the remote.""" + dst_ref = ref_to if ref_to else ref - head = git_get_local_head(repo_path, branch) - if not head: - logger.error(f"unable to find branch '{branch}' to push") - raise GitHeadNotFoundError(branch) + if _get_tag(repo_path, ref): + ref = f"refs/tags/{ref}" + dst_ref = f"refs/tags/{dst_ref}" + elif not git_get_local_head(repo_path, ref): + # ref is neither a local branch nor tag + logger.error(f"unable to find ref '{ref}' to push") + raise GitHeadNotFoundError(ref) repo = git.Repo(repo_path) try: @@ -690,12 +694,12 @@ def git_push( raise GitMissingRemoteError(remote_name) from None try: - info = remote.push(f"{branch}:{dst_branch}") + info = remote.push(f"{ref}:{dst_ref}") except git.CommandError as e: - msg = f"unable to push '{branch}' to '{dst_branch}': {e}" + msg = f"unable to push '{ref}' to '{dst_ref}': {e}" logger.error(msg) logger.error(e.stderr) - raise GitPushError(branch, dst_branch, remote_name) from None + raise GitPushError(ref, dst_ref, remote_name) from None updated: list[str] = [] rejected: list[str] = [] @@ -807,6 +811,19 @@ def git_format_patch(repo_path: Path, rev: SHA, *, base_rev: SHA | None = None) return res +def git_tag_exists_in_remote(repo_path: Path, remote_name: str, tag_name: str) -> bool: + try: + repo = git.Repo(repo_path) + raw_tag: str = repo.git.ls_remote( + "--tags", remote_name, f"refs/tags/{tag_name}" + ) + return bool(raw_tag.strip()) + except git.CommandError as e: + msg = f"unable to execute git ls-remote --tags {remote_name} refs/tags/{tag_name}: {e}" + logger.error(msg) + raise GitError(msg) from None + + if __name__ == "__main__": if len(sys.argv) < 2: print("error: missing repo path argument") diff --git a/crt/src/crt/crtlib/manifest.py b/crt/src/crt/crtlib/manifest.py index fbc6607..32b837e 100644 --- a/crt/src/crt/crtlib/manifest.py +++ b/crt/src/crt/crtlib/manifest.py @@ -32,11 +32,13 @@ ) from crt.crtlib.errors.stages import MissingStagePatchError from crt.crtlib.git_utils import ( + GitCreateHeadExistsError, GitError, GitFetchError, GitFetchHeadNotFoundError, GitIsTagError, GitPushError, + git_branch_from, git_checkout_ref, git_cleanup_repo, git_fetch_ref, @@ -75,6 +77,8 @@ def _prepare_repo( base_remote_name: str, push_remote_name: str, token: str, + *, + run_locally: bool = False, ) -> None: try: git_cleanup_repo(repo_path) @@ -83,50 +87,57 @@ def _prepare_repo( logger.error(msg) raise ManifestError(uuid=manifest_uuid, msg=msg) from None - try: - base_remote_uri = f"github.com/{base_remote_name}" - _ = git_prepare_remote(repo_path, base_remote_uri, base_remote_name, token) - push_remote_uri = f"github.com/{push_remote_name}" - _ = git_prepare_remote(repo_path, push_remote_uri, push_remote_name, token) - except GitError as e: - raise ManifestError(uuid=manifest_uuid, msg=str(e)) from None - - # fetch from base repository, if it exists. - try: - _ = git_fetch_ref(repo_path, target_branch, target_branch, push_remote_name) - except GitIsTagError as e: - msg = f"unexpected tag for branch '{target_branch}': {e}" - logger.error(msg) - raise ManifestError(uuid=manifest_uuid, msg=msg) from None - except GitFetchHeadNotFoundError: - # does not exist in the provided remote. - logger.debug( - f"branch '{target_branch}' does not exist in remote '{push_remote_name}'" - ) - except GitFetchError as e: - msg = f"unable to fetch '{target_branch}' from '{push_remote_name}': {e}" - logger.error(msg) - raise ManifestError(uuid=manifest_uuid, msg=msg) from None - except GitError as e: - msg = ( - f"unexpected error fetching branch '{target_branch}' " - + f"from '{push_remote_name}': {e}" - ) - logger.error(msg) - raise ManifestError(uuid=manifest_uuid, msg=msg) from None + if not run_locally: + try: + base_remote_uri = f"github.com/{base_remote_name}" + _ = git_prepare_remote(repo_path, base_remote_uri, base_remote_name, token) + push_remote_uri = f"github.com/{push_remote_name}" + _ = git_prepare_remote(repo_path, push_remote_uri, push_remote_name, token) + except GitError as e: + raise ManifestError(uuid=manifest_uuid, msg=str(e)) from None + + # fetch from base repository, if it exists. + try: + _ = git_fetch_ref(repo_path, target_branch, target_branch, push_remote_name) + except GitIsTagError as e: + msg = f"unexpected tag for branch '{target_branch}': {e}" + logger.error(msg) + raise ManifestError(uuid=manifest_uuid, msg=msg) from None + except GitFetchHeadNotFoundError: + # does not exist in the provided remote. + logger.debug( + f"branch '{target_branch}' does not exist in remote '{push_remote_name}'" + ) + except GitFetchError as e: + msg = f"unable to fetch '{target_branch}' from '{push_remote_name}': {e}" + logger.error(msg) + raise ManifestError(uuid=manifest_uuid, msg=msg) from None + except GitError as e: + msg = ( + f"unexpected error fetching branch '{target_branch}' " + + f"from '{push_remote_name}': {e}" + ) + logger.error(msg) + raise ManifestError(uuid=manifest_uuid, msg=msg) from None # we either fetched and thus we have an up-to-date local branch, or we didn't find # a corresponding reference in the remote and we need to either: # 1. checkout a new copy of the base ref to the target branch # 2. use an existing local target branch try: + if run_locally: + try: + git_branch_from(repo_path, base_ref, target_branch) + except GitCreateHeadExistsError: + msg = f"branch {target_branch} exists" + logger.info(msg) _ = git_checkout_ref( repo_path, base_ref, to_branch=target_branch, remote_name=base_remote_name, update_from_remote=False, - fetch_if_not_exists=True, + fetch_if_not_exists=not run_locally, ) git_cleanup_repo(repo_path) @@ -148,6 +159,7 @@ def manifest_execute( token: str, *, no_cleanup: bool = True, + run_locally: bool = False, ) -> ManifestExecuteResult: """ Execute a manifest against its base ref. @@ -183,6 +195,7 @@ def manifest_execute( base_remote_name, manifest.dst_repo, token, + run_locally=run_locally, ) except ManifestError as e: logger.error(f"unable to prepare repository to execute manifest: {e}") @@ -197,6 +210,7 @@ def manifest_execute( target_branch, token, no_cleanup=no_cleanup, + run_locally=run_locally, ) pass except ApplyError as e: @@ -329,7 +343,7 @@ def manifest_publish_branch( repo_path, our_branch, dst_repo, - branch_to=dst_branch, + ref_to=dst_branch, ) except GitPushError as e: msg = f"unable to push '{our_branch}': {e}" @@ -490,6 +504,13 @@ def store_manifest(patches_repo_path: Path, manifest: ReleaseManifest) -> None: logger.error(msg) raise ManifestError(uuid=manifest.release_uuid, msg=msg) from None + try: + manifest_name_path.parent.mkdir(parents=True, exist_ok=True) + except Exception as e: + msg = f"error creating folder '{manifest_name_path.parent}': {e}" + logger.error(msg) + raise ManifestError(uuid=manifest.release_uuid, msg=msg) from None + if not manifest_name_path.exists(): try: manifest_name_path.symlink_to(Path("..").joinpath(manifest_uuid_path.name)) From 0e71f05354b1d6bb7e448cfa874dbd0264fc19b2 Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 27 Jan 2026 14:56:42 +0100 Subject: [PATCH 3/4] crt/cmds/patch: add --local-run option don't access remotes if the flag is set and the subcommand doesn't require remote access. affected subcommands: * add: * don't add remote url and don't fetch Signed-off-by: Uwe Schwaeke --- crt/src/crt/cmds/patch.py | 24 +- crt/src/crt/cmds/patchset.py | 42 +- crt/src/crt/cmds/patchset.py.orig | 1093 +++++++++++++++++++++++++++++ 3 files changed, 1142 insertions(+), 17 deletions(-) create mode 100644 crt/src/crt/cmds/patchset.py.orig diff --git a/crt/src/crt/cmds/patch.py b/crt/src/crt/cmds/patch.py index 0930e3f..4a45818 100644 --- a/crt/src/crt/cmds/patch.py +++ b/crt/src/crt/cmds/patch.py @@ -161,17 +161,18 @@ def _check_repo(repo_path: Path, what: str) -> None: else: src_ceph_repo_path = ceph_repo_path - # update remote repo, maybe patches are not yet in the current repo state - try: - _ = git_prepare_remote( - src_ceph_repo_path, - f"github.com/{src_gh_repo}", - src_gh_repo, - ctx.github_token, - ) - except Exception as e: - perror(f"unable to update remote '{src_gh_repo}': {e}") - sys.exit(errno.ENOTRECOVERABLE) + if not ctx.run_locally: + # update remote repo, maybe patches are not yet in the current repo state + try: + _ = git_prepare_remote( + src_ceph_repo_path, + f"github.com/{src_gh_repo}", + src_gh_repo, + ctx.github_token, + ) + except Exception as e: + perror(f"unable to update remote '{src_gh_repo}': {e}") + sys.exit(errno.ENOTRECOVERABLE) try: shas = [git_revparse(src_ceph_repo_path, sha) for sha in patch_sha] @@ -230,6 +231,7 @@ def _check_repo(repo_path: Path, what: str) -> None: ceph_repo_path, patches_repo_path, ctx.github_token, + run_locally=ctx.run_locally, ) except (ApplyError, Exception) as e: perror(f"unable to apply patch sha '{sha}' to manifest: {e}") diff --git a/crt/src/crt/cmds/patchset.py b/crt/src/crt/cmds/patchset.py index 1a065c5..e6b0023 100644 --- a/crt/src/crt/cmds/patchset.py +++ b/crt/src/crt/cmds/patchset.py @@ -44,7 +44,9 @@ ) from crt.crtlib.git_utils import ( SHA, + GitError, git_branch_delete, + git_branch_from, git_get_patch_sha_title, git_patches_in_interval, git_prepare_remote, @@ -564,12 +566,40 @@ def cmd_patchset_add( seq = dt.now(datetime.UTC).strftime("%Y%m%d%H%M%S") dst_branch = f"patchset/branch/{gh_repo.replace('/', '--')}--{patches_branch}-{seq}" - try: - _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") - except Exception as e: - progress.stop_error() - perror(f"unable to fetch branch '{patches_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) + + if not ctx.run_locally: + # ensure we have the specified branch in the ceph repo, so we can actually + # obtain the right shas + progress.new_task("prepare remote") + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + ) + except Exception as e: + progress.stop_error() + perror(f"unable to update remote '{gh_repo}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + progress.done_task() + + progress.new_task("fetch patches") + + try: + _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") + except Exception as e: + progress.stop_error() + perror(f"unable to fetch branch '{patches_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + else: + progress.new_task("create branch patches") + try: + git_branch_from(ceph_repo_path, patches_branch, dst_branch) + except GitError as e: + progress.stop_error() + perror( + f"unable to create branch '{dst_branch}' from '{patches_branch}': {e}" + ) + sys.exit(errno.ENOTRECOVERABLE) def _cleanup() -> None: try: diff --git a/crt/src/crt/cmds/patchset.py.orig b/crt/src/crt/cmds/patchset.py.orig new file mode 100644 index 0000000..e783556 --- /dev/null +++ b/crt/src/crt/cmds/patchset.py.orig @@ -0,0 +1,1093 @@ +# CBS Release Tool - patchset commands +# Copyright (C) 2025 Clyso GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +import datetime +import errno +import re +import sys +import uuid +from datetime import datetime as dt +from pathlib import Path +from typing import cast + +import click +import pydantic +import rich.box +from rich.console import Group, RenderableType +from rich.padding import Padding +from rich.table import Table + +from crt.cmds import ( + Ctx, + console, + pass_ctx, + perror, + pinfo, + psuccess, + pwarn, + with_patches_repo_path, +) +from crt.cmds import logger as parent_logger +from crt.cmds._common import CRTProgress +from crt.crtlib.errors.patchset import ( + MalformedPatchSetError, +) +from crt.crtlib.git_utils import ( + SHA, + git_branch_delete, + git_get_patch_sha_title, + git_patches_in_interval, + git_prepare_remote, +) +from crt.crtlib.models.common import ( + AuthorData, + ManifestPatchEntry, + ManifestPatchSetEntryType, +) +from crt.crtlib.models.discriminator import ManifestPatchEntryWrapper +from crt.crtlib.models.patch import PatchMeta +from crt.crtlib.models.patchset import ( + CustomPatchMeta, + CustomPatchSet, + GitHubPullRequest, + PatchSetBase, +) +from crt.crtlib.patchset import ( + fetch_custom_patchset_patches, + get_patchset_meta_path, + load_patchset, + write_patchset, +) + +logger = parent_logger.getChild("patchset") + + +def _is_valid_sha(sha: str) -> bool: + return re.match(r"^[\da-f]{4}[\da-f]{0,36}$", sha) is not None + + +def _gen_rich_patch_meta_info_header(table: Table, patch_meta: PatchMeta) -> Table: + table.add_row("sha", f"[orchid2]{patch_meta.sha}[/orchid2]") + table.add_row("title", patch_meta.info.title) + table.add_row( + "author", f"{patch_meta.info.author.user} <{patch_meta.info.author.email}>" + ) + table.add_row("date", f"{patch_meta.info.date}") + table.add_row("src version", patch_meta.src_version or "n/a") + table.add_row("fixes", "\n".join(patch_meta.info.fixes) or "n/a") + return table + + +def _gen_rich_patchset_base_info_header(table: Table, patchset: PatchSetBase) -> Table: + table.add_row("title", patchset.title) + table.add_row("author", f"{patchset.author.user} <{patchset.author.email}>") + table.add_row("created", f"{patchset.creation_date}") + table.add_row("related to", "\n".join(patchset.related_to) or "n/a") + + if isinstance(patchset, GitHubPullRequest): + table.add_row("repo", f"{patchset.org_name}/{patchset.repo_name}") + table.add_row("pr id", str(patchset.pull_request_id)) + table.add_row("updated on", str(patchset.updated_date) or "n/a") + table.add_row("merged", "Yes" if patchset.merged else "No") + if patchset.merged: + table.add_row("merged on", str(patchset.merge_date) or "n/a") + table.add_row("target branch", patchset.target_branch) + table.add_row("patches", str(len(patchset.patches))) + + elif isinstance(patchset, CustomPatchSet): + desc = patchset.description_text + table.add_row("description", desc or "n/a") + table.add_row("release", patchset.release_name or "n/a") + table.add_row( + "patches", str(sum(len(meta.patches) for meta in patchset.patches_meta)) + ) + table.add_row( + "published", + "[green]Yes[/green]" if patchset.is_published else "[red]No[/red]", + ) + + return table + + +def _gen_rich_patchset_info_header(table: Table, patchset: ManifestPatchEntry) -> Table: + if isinstance(patchset, PatchMeta): + return _gen_rich_patch_meta_info_header(table, patchset) + elif isinstance(patchset, PatchSetBase): + return _gen_rich_patchset_base_info_header(table, patchset) + else: + perror(f"unknown patch set type: {type(patchset)}") + sys.exit(errno.ENOTRECOVERABLE) + + +def _gen_rich_patchset_info(patchset: ManifestPatchEntry) -> RenderableType: + header_table = Table(show_header=False, box=None, expand=False) + header_table.add_column(justify="left", style="bold cyan", no_wrap=False) + header_table.add_column(justify="left", style="orange3", no_wrap=False) + + header_table.add_row("uuid", f"[gold1]{patchset.entry_uuid}[/gold1]") + header_table.add_row("type", patchset.entry_type.value) + header_table = _gen_rich_patchset_info_header(header_table, patchset) + + patch_lst_table = Table( + show_header=False, + show_lines=False, + box=rich.box.HORIZONTALS, + ) + patch_lst_table.add_column(justify="left", style="bold gold1", no_wrap=True) + patch_lst_table.add_column(justify="left", style="white", no_wrap=False) + + if isinstance(patchset, GitHubPullRequest): + for patch in patchset.patches: + patch_lst_table.add_row(patch.sha, patch.title) + elif isinstance(patchset, CustomPatchSet): + for entry in patchset.patches_meta: + for sha, title in entry.patches: + patch_lst_table.add_row(sha, title) + + return Group(header_table, patch_lst_table) + + +def _gen_rich_patchset_list() -> Table: + table = Table( + show_header=False, + show_lines=True, + box=rich.box.HORIZONTALS, + ) + # uuids + table.add_column(justify="left", style="bold gold1", no_wrap=True) + # type + table.add_column(justify="left", style="bold cyan", no_wrap=True) + # freeform entry + table.add_column(justify="left", style="white", no_wrap=False) + return table + + +def _gen_rich_patch_meta(patch_meta: PatchMeta) -> RenderableType: + version = patch_meta.src_version if patch_meta.src_version else "n/a" + + # mimic what we would do with 'Columns', except that those will always take all + # the available width within the console. And we want it to just fit to contents. + t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + t1.add_column(justify="left") + t1.add_column(justify="left") + t1.add_row( + f"[orchid2]{patch_meta.sha}[/orchid2]", + f"[italic]version:[/italic] [orange3]{version}[/orange3]", + ) + + t2 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + t2.add_column(justify="left") + t2.add_column(justify="left") + t2.add_row( + f"[cyan]{patch_meta.info.date}[/cyan]", + f"[cyan]{patch_meta.info.author.user} " + + f"<{patch_meta.info.author.email}>[/cyan]", + ) + + entries: list[RenderableType] = [ + f"[bold magenta]{patch_meta.info.title}[/bold magenta]", + t1, + t2, + ] + group = Group(*entries) + return group + + +def _gen_rich_patchset_gh(patchset: GitHubPullRequest) -> RenderableType: + is_merged = "[green]merged[/green]" if patchset.merged else "[red]not merged[/red]" + repo_pr_id = f"{patchset.org_name}/{patchset.repo_name} #{patchset.pull_request_id}" + t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + t1.add_row( + f"[orchid2]{repo_pr_id}[/orchid2] ({is_merged})", + f"[italic]version:[/italic] [orange3]{patchset.target_branch}[/orange3]", + ) + + updated = patchset.updated_date if patchset.updated_date else "n/a" + merged = patchset.merge_date if patchset.merge_date else "n/a" + t2 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + t2.add_row( + f"[italic]updated:[/italic] [cyan]{updated}[/cyan]", + f"[italic]merged:[/italic] [cyan]{merged}[/cyan]", + ) + + return Group(t1, t2) + + +def _gen_rich_patchset_custom(patchset: CustomPatchSet) -> RenderableType: + release = patchset.release_name if patchset.release_name else "n/a" + is_published = "[green]Yes[/green]" if patchset.is_published else "[red]No[/red]" + t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + n_patches = sum(len(meta.patches) for meta in patchset.patches_meta) + t1.add_row( + f"[italic]release:[/italic] [orange3]{release}[/orange3]", + f"[italic]published:[/italic] {is_published}", + f"[italic]patches:[/italic] [orange3]{n_patches}[/orange3]", + ) + return t1 + + +def _gen_rich_patchset_base(patchset: PatchSetBase) -> RenderableType: + # mimic what we would do with 'Columns', except that those will always take all + # the available width within the console. And we want it to just fit to contents. + + t = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) + t.add_row( + f"[cyan]{patchset.creation_date}[/cyan]", + f"[cyan]{patchset.author.user} " + f"<{patchset.author.email}>[/cyan]", + ) + + rdr: RenderableType + if isinstance(patchset, GitHubPullRequest): + rdr = _gen_rich_patchset_gh(patchset) + elif isinstance(patchset, CustomPatchSet): + rdr = _gen_rich_patchset_custom(patchset) + else: + perror(f"unknown base patch set type: {type(patchset)}") + sys.exit(errno.ENOTRECOVERABLE) + + entries = [ + f"[bold magenta]{patchset.title}[/bold magenta]", + Group(t, rdr), + ] + + return Group(*entries) + + +def _add_rich_patchset_entry(table: Table, patchset: ManifestPatchEntry) -> None: + rdr: RenderableType + if isinstance(patchset, PatchMeta): + rdr = _gen_rich_patch_meta(patchset) + elif isinstance(patchset, PatchSetBase): + rdr = _gen_rich_patchset_base(patchset) + else: + perror(f"unknown patch set type: {type(patchset)}") + sys.exit(errno.ENOTRECOVERABLE) + + row: tuple[str, str, RenderableType] = ( + str(patchset.entry_uuid), + patchset.entry_type.value, + rdr, + ) + table.add_row(*row) + pass + + +@click.group("patchset", help="Handle patch sets.") +def cmd_patchset() -> None: + pass + + +@cmd_patchset.command("create", help="Create a patch set from individual patches.") +@click.option( + "--author", + "author_name", + required=True, + type=str, + metavar="NAME", + help="Author's name.", +) +@click.option( + "--email", + "author_email", + required=True, + type=str, + metavar="EMAIL", + help="Author's email.", +) +@click.option( + "--title", + "-T", + "patchset_title", + required=False, + default="", + type=str, + metavar="TEXT", + help="Title for this patch set.", +) +@click.option( + "--desc", + "-D", + "patchset_desc", + required=False, + type=str, + metavar="TEXT", + help="Short description of this patch set.", +) +@click.option( + "-r", + "--release-name", + "release_name", + required=False, + type=str, + metavar="NAME", + help="Release associated with this patch set.", +) +@with_patches_repo_path +def cmd_patchset_create( + patches_repo_path: Path, + author_name: str, + author_email: str, + patchset_title: str | None, + patchset_desc: str | None, + release_name: str | None, +) -> None: + print("prompt?") + if not patchset_title: + patchset_title = cast( + str | None, click.prompt("Patch set title", type=str, prompt_suffix=" > ") + ) + if not patchset_title or not patchset_title.strip(): + perror("must specify a patch set title") + sys.exit(errno.EINVAL) + + if patchset_desc and not patchset_desc.strip(): + perror("patch set description is empty") + sys.exit(errno.EINVAL) + + author_info = f"{author_name} <{author_email}>" + if not patchset_desc and click.confirm("Add a description?", default=False): + try: + desc_msg = ( + click.edit( + f"{patchset_title}\n\n\n\n" + + f"Signed-off-by: {author_info}" + ), + ) + except Exception as e: + perror(f"unable to open editor: {e}") + sys.exit(errno.ENOTRECOVERABLE) + + print(desc_msg) + if not desc_msg or not desc_msg[0]: + perror("must specify a patch set description") + sys.exit(errno.EINVAL) + + patchset_desc = desc_msg[0].strip() + + if not patchset_desc: + patchset_desc = f"{patchset_title}\n\nSigned-off-by: {author_info}" + + patchset = CustomPatchSet( + author=AuthorData(user=author_name, email=author_email), + creation_date=dt.now(datetime.UTC), + title=patchset_title, + related_to=[], + patches=[], + description=patchset_desc, + release_name=release_name, + ) + + patchset_meta_path = get_patchset_meta_path(patches_repo_path, patchset.entry_uuid) + assert not patchset_meta_path.exists() + + try: + write_patchset(patches_repo_path, patchset) + except Exception as e: + perror(f"unable to write patch set meta file: {e}") + sys.exit(errno.ENOTRECOVERABLE) + + psuccess(f"successfully created patch set '{patchset.entry_uuid}'") + + +@cmd_patchset.command("list", help="List patch sets in the patches repository.") +@click.option( + "-t", + "--type", + "patchset_types", + type=str, + multiple=True, + required=False, + metavar="TYPE", + help="Filter by patch set type.", +) +@with_patches_repo_path +def cmd_patchset_list(patches_repo_path: Path, patchset_types: list[str]) -> None: + meta_path = patches_repo_path / "ceph" / "patches" / "meta" + + avail_types = [m.value for m in ManifestPatchSetEntryType] + if patchset_types and any(t not in avail_types for t in patchset_types): + perror(f"unknown patch set type(s), available: {', '.join(avail_types)}") + sys.exit(errno.EINVAL) + + table = _gen_rich_patchset_list() + for patchset_path in meta_path.glob("*.json"): + try: + patchset_uuid = uuid.UUID(patchset_path.stem) + except Exception: + pwarn(f"malformed patch set uuid in '{patchset_path}', skip") + continue + + try: + patchset = load_patchset(patches_repo_path, patchset_uuid) + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + if patchset_types and patchset.entry_type.value not in patchset_types: + continue + + _add_rich_patchset_entry(table, patchset) + + if len(table.rows) > 0: + console.print(table) + else: + pwarn("no entries found") + + +@cmd_patchset.command("info", help="Obtain info on a given patch set.") +@click.option( + "-u", + "--patchset-uuid", + "patchset_uuid", + type=uuid.UUID, + required=True, + help="Patch set UUID.", +) +@with_patches_repo_path +def cmd_patchset_info(patches_repo_path: Path, patchset_uuid: uuid.UUID) -> None: + try: + patchset = load_patchset(patches_repo_path, patchset_uuid) + except MalformedPatchSetError as e: + perror(f"malformed patch set '{patchset_uuid}': {e}") + sys.exit(errno.EINVAL) + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + console.print(_gen_rich_patchset_info(patchset)) + + +@cmd_patchset.command("add", help="Add one or more patches to a patch set.") +@click.option( + "-c", + "--ceph-repo", + "ceph_repo_path", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, resolve_path=True, path_type=Path + ), + required=True, + envvar="CRT_CEPH_REPO_PATH", + help="Path to ceph git repository where operations will be performed.", +) +@click.option( + "-u", + "--patchset-uuid", + "patchset_uuid", + type=uuid.UUID, + required=True, + help="Patch set UUID.", +) +@click.option( + "--gh-repo", + "gh_repo", + type=str, + required=False, + default="ceph/ceph", + metavar="OWNER/REPO", + help="GitHub repository to obtain the patch(es) from.", +) +@click.option( + "-b", + "--branch", + "patches_branch", + type=str, + required=True, + metavar="NAME", + help="Branch on which to find patches.", +) +@click.argument( + "patch_sha", + metavar="SHA|SHA1..SHA2 [...]", + type=str, + required=True, + nargs=-1, +) +@with_patches_repo_path +@pass_ctx +def cmd_patchset_add( + ctx: Ctx, + patches_repo_path: Path, + ceph_repo_path: Path, + patchset_uuid: uuid.UUID, + gh_repo: str, + patches_branch: str, + patch_sha: list[str], +) -> None: + if not ctx.github_token: + perror("github token not specified") + sys.exit(errno.EINVAL) + + try: + patchset = load_patchset(patches_repo_path, patchset_uuid) + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + if not isinstance(patchset, CustomPatchSet): + perror(f"patch set '{patchset_uuid}' is not a custom patch set") + sys.exit(errno.EINVAL) + + if patchset.is_published: + perror(f"patch set '{patchset_uuid}' is already published") + sys.exit(errno.EINVAL) + + existing_shas = [p[0] for meta in patchset.patches_meta for p in meta.patches] + + progress = CRTProgress(console) + progress.start() + progress.new_task(f"add patches from '{gh_repo}' branch '{patches_branch}'") + + # ensure we have the specified branch in the ceph repo, so we can actually obtain + # the right shas + progress.new_task("prepare remote") + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + ) + except Exception as e: + progress.stop_error() + perror(f"unable to update remote '{gh_repo}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + progress.done_task() + progress.new_task("fetch patches") + + seq = dt.now(datetime.UTC).strftime("%Y%m%d%H%M%S") + dst_branch = f"patchset/branch/{gh_repo.replace('/', '--')}--{patches_branch}-{seq}" +<<<<<<< HEAD + try: + _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") + except Exception as e: + progress.stop_error() + perror(f"unable to fetch branch '{patches_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) +||||||| parent of 716fbc5 ( fixup f95accd9) + + if not ctx.run_locally: + # ensure we have the specified branch in the ceph repo, so we can actually + # obtain the right shas + progress.new_task("prepare remote") + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + ) + except Exception as e: + progress.stop_error() + perror(f"unable to update remote '{gh_repo}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + progress.done_task() + + progress.new_task("fetch patches") + + try: + _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") + except Exception as e: + progress.stop_error() + perror(f"unable to fetch branch '{patches_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + else: + progress.new_task("create branch patches") + try: + git_branch_from(ceph_repo_path, patches_branch, dst_branch) + except GitError as e: + progress.stop_error() + perror(f"unable to create branch '{dst_branch} from {patches_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) +======= + + if not ctx.run_locally: + # ensure we have the specified branch in the ceph repo, so we can actually + # obtain the right shas + progress.new_task("prepare remote") + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token + ) + except Exception as e: + progress.stop_error() + perror(f"unable to update remote '{gh_repo}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + progress.done_task() + + progress.new_task("fetch patches") + + try: + _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") + except Exception as e: + progress.stop_error() + perror(f"unable to fetch branch '{patches_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + else: + progress.new_task("create branch patches") + try: + git_branch_from(ceph_repo_path, patches_branch, dst_branch) + except GitError as e: + progress.stop_error() + perror( + f"unable to create branch '{dst_branch}' from '{patches_branch}': {e}" + ) + sys.exit(errno.ENOTRECOVERABLE) +>>>>>>> 716fbc5 ( fixup f95accd9) + + def _cleanup() -> None: + try: + git_branch_delete(ceph_repo_path, dst_branch) + except Exception as e: + progress.stop_error() + perror(f"unable to delete temporary branch '{dst_branch}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + patches_meta_lst: list[CustomPatchMeta] = [] + skipped_patches: list[tuple[SHA, str]] = [] + for sha_entry in patch_sha: + sha_begin: SHA + sha_end: SHA | None = None + if ".." in sha_entry: + interval = sha_entry.split("..", 1) + if len(interval) != 2 or not interval[0] or not interval[1]: + perror(f"malformed patch interval '{sha_entry}'") + sys.exit(errno.EINVAL) + sha_begin, sha_end = interval[0], interval[1] + else: + sha_begin = sha_entry + + if not _is_valid_sha(sha_begin) or (sha_end and not _is_valid_sha(sha_end)): + _cleanup() + progress.stop_error() + perror(f"malformed patch sha '{sha_entry}'") + sys.exit(errno.EINVAL) + + patches_lst: list[tuple[SHA, str]] = [] + if sha_end: + try: + for sha, title in reversed( + git_patches_in_interval(ceph_repo_path, sha_begin, sha_end) + ): + if sha not in existing_shas: + patches_lst.append((sha, title)) + else: + skipped_patches.append((sha, title)) + except Exception as e: + _cleanup() + progress.stop_error() + perror(f"unable to obtain patches in interval '{sha_entry}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + else: + try: + sha, title = git_get_patch_sha_title(ceph_repo_path, sha_begin) + if sha not in existing_shas: + patches_lst.append((sha, title)) + else: + skipped_patches.append((sha, title)) + except Exception as e: + _cleanup() + progress.stop_error() + perror(f"unable to obtain patch info for sha '{sha_entry}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + if patches_lst: + patches_meta_lst.append( + CustomPatchMeta( + repo=gh_repo, + branch=patches_branch, + sha=sha_begin, + sha_end=sha_end, + patches=patches_lst, + ) + ) + + progress.done_task() + progress.done_task() + progress.stop() + + if patchset.patches_meta: + existing_patches_table = Table( + title="Existing patches", + title_style="bold magenta", + show_header=False, + show_lines=False, + box=rich.box.HORIZONTALS, + ) + existing_patches_table.add_column( + justify="left", style="bold cyan", no_wrap=True + ) + existing_patches_table.add_column(justify="left", style="white", no_wrap=False) + + for existing_entry in patchset.patches_meta: + for sha, title in existing_entry.patches: + existing_patches_table.add_row(sha, title) + + console.print(Padding(existing_patches_table, (1, 0, 0, 0))) + + if skipped_patches: + skipped_patches_table = Table( + title="Skipped patches", + title_style="bold orange3", + show_header=False, + show_lines=False, + box=rich.box.HORIZONTALS, + ) + skipped_patches_table.add_column( + justify="left", style="bold hot_pink", no_wrap=True + ) + skipped_patches_table.add_column(justify="left", style="white", no_wrap=False) + + for sha, title in skipped_patches: + skipped_patches_table.add_row(sha, title) + + console.print(Padding(skipped_patches_table, (1, 0, 0, 0))) + + if not patches_meta_lst: + console.print( + Padding("[bold yellow]No new patches to add[/bold yellow]", (1, 0, 1, 2)) + ) + return + else: + patch_lst_table = Table( + title="Patches to add", + title_style="bold green", + show_header=False, + show_lines=False, + box=rich.box.HORIZONTALS, + ) + patch_lst_table.add_column(justify="left", style="bold gold1", no_wrap=True) + patch_lst_table.add_column(justify="left", style="white", no_wrap=False) + + for entry in patches_meta_lst: + for sha, title in entry.patches: + patch_lst_table.add_row(sha, title) + + console.print(Padding(patch_lst_table, (1, 0, 1, 0))) + if not click.confirm("Add above patches to patch set?", prompt_suffix=" "): + pwarn("Aborted") + sys.exit(0) + + patchset.patches_meta.extend(patches_meta_lst) + + try: + write_patchset(patches_repo_path, patchset) + except Exception as e: + _cleanup() + perror(f"unable to write patch set meta file: {e}") + sys.exit(errno.ENOTRECOVERABLE) + + n_patches = sum(len(meta.patches) for meta in patches_meta_lst) + psuccess(f"wrote patch set '{patchset.entry_uuid}', {n_patches} new patches") + _cleanup() + + +@cmd_patchset.command("publish", help="Publish a patch set to the patches repository.") +@click.option( + "-c", + "--ceph-repo", + "ceph_repo_path", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, resolve_path=True, path_type=Path + ), + envvar="CRT_CEPH_REPO_PATH", + required=True, + help="Path to ceph git repository where operations will be performed.", +) +@click.option( + "-u", + "--patchset-uuid", + "patchset_uuid", + type=uuid.UUID, + required=True, + help="Patch set UUID.", +) +@with_patches_repo_path +@pass_ctx +def cmd_patchset_publish( + ctx: Ctx, + patches_repo_path: Path, + ceph_repo_path: Path, + patchset_uuid: uuid.UUID, +) -> None: + if not ctx.github_token: + perror("missing GitHub token") + sys.exit(errno.EINVAL) + + try: + patchset = load_patchset(patches_repo_path, patchset_uuid) + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + if not isinstance(patchset, CustomPatchSet): + perror(f"patch set '{patchset_uuid}' is not a custom patch set") + sys.exit(errno.EINVAL) + + if patchset.is_published: + perror(f"patch set '{patchset_uuid}' is already published") + sys.exit(errno.EINVAL) + + if not patchset.patches_meta or all( + len(meta.patches) == 0 for meta in patchset.patches_meta + ): + perror(f"patch set '{patchset_uuid}' has no patches") + sys.exit(errno.EINVAL) + + console.print(_gen_rich_patchset_info(patchset)) + if not click.confirm("Publish above patch set?", prompt_suffix=" "): + pwarn("Aborted") + sys.exit(0) + + progress = CRTProgress(console) + progress.start() + progress.new_task("fetch patches") + + try: + patches = fetch_custom_patchset_patches( + ceph_repo_path, patches_repo_path, patchset, ctx.github_token + ) + except Exception as e: + progress.stop_error() + perror(f"unable to fetch patches for patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + progress.done_task() + + patchset.patches = patches + patchset.is_published = True + + try: + write_patchset(patches_repo_path, patchset) + except Exception as e: + perror(f"unable to write patch set meta file: {e}") + sys.exit(errno.ENOTRECOVERABLE) + + psuccess(f"published patch set '{patchset.entry_uuid}'") + + +@cmd_patchset.command("remove", help="Remove an unpublished patch set.") +@click.option( + "-u", + "--patchset-uuid", + "patchset_uuid", + type=uuid.UUID, + required=True, + help="Patch set UUID.", +) +@with_patches_repo_path +def cmd_patchset_remove(patches_repo_path: Path, patchset_uuid: uuid.UUID) -> None: + patchset_meta_path = get_patchset_meta_path(patches_repo_path, patchset_uuid) + if not patchset_meta_path.exists(): + perror(f"patch set '{patchset_uuid}' does not exist") + sys.exit(errno.ENOENT) + + try: + patchset = load_patchset(patches_repo_path, patchset_uuid) + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + if not isinstance(patchset, CustomPatchSet): + perror(f"patch set '{patchset_uuid}' is not a custom patch set") + sys.exit(errno.EINVAL) + + if patchset.is_published: + perror(f"patch set '{patchset_uuid}' is already published") + sys.exit(errno.EINVAL) + + console.print(_gen_rich_patchset_info(patchset)) + if not click.confirm("Remove above patch set?", prompt_suffix=" "): + pwarn("Aborted") + sys.exit(0) + + try: + patchset_meta_path.unlink() + except Exception as e: + perror(f"unable to remove patch set meta file: {e}") + sys.exit(errno.ENOTRECOVERABLE) + + psuccess(f"removed patch set '{patchset.entry_uuid}'") + + +@cmd_patchset.group("advanced", help="Advanced patch set operations.") +def cmd_patchset_advanced() -> None: + pass + + +@cmd_patchset_advanced.command( + "migrate-store-format", help="Migrate patch sets' store format" +) +@with_patches_repo_path +def cmd_patchset_migrate_store_format(patches_repo_path: Path) -> None: + if not patches_repo_path.exists(): + perror(f"patches repository does not exist at '{patches_repo_path}'") + sys.exit(errno.ENOENT) + + if not patches_repo_path.joinpath(".git").exists(): + perror("provided path for patches repository is not a git repository") + sys.exit(errno.EINVAL) + + patches_path = patches_repo_path / "ceph" / "patches" + if not patches_path.exists(): + pinfo(f"patches path does not exist at '{patches_path}', nothing to do") + return + + n_patchsets = 0 + candidate_dirs: list[Path] = [] + for p in patches_path.iterdir(): + if p.is_dir() and p.name != "meta": + candidate_dirs.append(p) + + print(candidate_dirs) + for d in candidate_dirs: + for p in list(d.walk()): + for sub in p[1]: + sub_path = Path(p[0]) / sub + if not sub_path.is_dir(): + continue + + if not re.match(r"^[\w\d_.-]+$", sub): + # not a valid repo name + continue + + repo_name = f"{d.name}/{sub}" + + for pr in sub_path.iterdir(): + if pr.is_dir(): + continue + + if not re.match(r"^\d+$", pr.name): + # not a valid pr id + pwarn(f"skip invalid pr id '{pr.name}' in '{repo_name}'") + continue + + try: + patchset_uuid = uuid.UUID(pr.read_text()) + except Exception: + pwarn( + f"malformed patch set uuid in '{pr}' in '{repo_name}', skip" + ) + continue + + pinfo(f"pr id '{pr.name}' uuid '{patchset_uuid}' in '{repo_name}'") + latest_patchset_path = patches_path / f"{patchset_uuid}.patch" + latest_meta_path = patches_path / "meta" / f"{patchset_uuid}.json" + + if ( + not latest_patchset_path.exists() + and not latest_meta_path.exists() + ): + pwarn( + f"missing patch file '{latest_patchset_path}', " + + "skip migration" + ) + continue + + try: + patchset_meta = ManifestPatchEntryWrapper.model_validate_json( + latest_meta_path.read_text() + ) + except pydantic.ValidationError as e: + perror(f"malformed meta file '{latest_meta_path}': {e}") + continue + + if not isinstance(patchset_meta.contents, GitHubPullRequest): + perror( + f"found meta for patchset uuid '{patchset_uuid}' " + + "is not a gh pr" + ) + continue + + patchset = patchset_meta.contents + if not patchset.patches: + perror( + f"found empty patch set for uuid '{patchset_uuid}' " + + f"pr id '{pr.name}' repo '{repo_name}'" + ) + continue + + head_patch_sha = next(reversed(patchset.patches)).sha + pinfo( + f"pr id '{pr.name}' repo '{repo_name}' " + + f"head patch sha '{head_patch_sha}'" + ) + head_path_sha_path = pr / head_patch_sha + latest_path = pr / "latest" + + try: + pr.unlink() + pr.mkdir() + _ = head_path_sha_path.write_text(str(patchset_uuid)) + latest_path.symlink_to(head_patch_sha) + except Exception as e: + perror(f"unable to migrate pr id '{pr.name}': {e}") + continue + + psuccess( + f"successfully migrated pr id '{pr.name}' repo '{repo_name}'" + ) + n_patchsets += 1 + + psuccess(f"successfully migrated {n_patchsets} patch sets") + + +@cmd_patchset_advanced.command("migrate-single-patches", help="Migrate single patches.") +@with_patches_repo_path +def cmd_patchset_migrate_single_patches(patches_repo_path: Path) -> None: + meta_path = patches_repo_path / "ceph" / "patches" / "meta" + + n_patchsets = 0 + for patchset_path in meta_path.glob("*.json"): + try: + patchset_uuid = uuid.UUID(patchset_path.stem) + except Exception: + pwarn(f"malformed patch set uuid in '{patchset_path}', skip") + continue + + try: + _ = load_patchset(patches_repo_path, patchset_uuid) + continue + except MalformedPatchSetError: + # possibly a single patch, check and migrate it if so. + pinfo(f"possible single patch '{patchset_uuid}', check") + except Exception as e: + perror(f"unable to load patch set '{patchset_uuid}': {e}") + continue + + try: + single_patch = PatchMeta.model_validate_json(patchset_path.read_text()) + except Exception: + perror(f"unable to parse single patch meta '{patchset_path}'") + continue + + pinfo(f"found single patch at '{single_patch.entry_uuid}', migrate") + # backup existing meta file + bak_path = patchset_path.with_suffix(".json.bak") + try: + _ = bak_path.write_text(patchset_path.read_text()) + except Exception as e: + perror(f"unable to backup single patch meta '{patchset_path}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + try: + _ = patchset_path.write_text( + ManifestPatchEntryWrapper(contents=single_patch).model_dump_json( + indent=2 + ) + ) + except Exception as e: + perror(f"unable to migrate single patch meta '{patchset_path}': {e}") + sys.exit(errno.ENOTRECOVERABLE) + + psuccess(f"successfully migrated single patch '{patchset_uuid}'") + bak_path.unlink() + n_patchsets += 1 + + pinfo(f"migrated {n_patchsets} single patches") From d64c2176553babe2633615e1dc59b5d32d66b6ca Mon Sep 17 00:00:00 2001 From: Uwe Schwaeke Date: Tue, 27 Jan 2026 15:01:45 +0100 Subject: [PATCH 4/4] crt/cmds/patchset: add --local-run option don't access remotes if the flag is set and the subcommand doesn't require remote access. affected subcommands: * add: * don't add remote url and don't fetch * create branch from an existing local branch * publish: * don't add remote url and don't fetch * create branch from an existing local branch other subcommands will ignore the flag Signed-off-by: Uwe Schwaeke --- crt/src/crt/cmds/patchset.py | 21 +- crt/src/crt/cmds/patchset.py.orig | 1093 ----------------------------- crt/src/crt/crtlib/patchset.py | 38 +- 3 files changed, 31 insertions(+), 1121 deletions(-) delete mode 100644 crt/src/crt/cmds/patchset.py.orig diff --git a/crt/src/crt/cmds/patchset.py b/crt/src/crt/cmds/patchset.py index e6b0023..d3d8822 100644 --- a/crt/src/crt/cmds/patchset.py +++ b/crt/src/crt/cmds/patchset.py @@ -549,21 +549,6 @@ def cmd_patchset_add( progress.start() progress.new_task(f"add patches from '{gh_repo}' branch '{patches_branch}'") - # ensure we have the specified branch in the ceph repo, so we can actually obtain - # the right shas - progress.new_task("prepare remote") - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token - ) - except Exception as e: - progress.stop_error() - perror(f"unable to update remote '{gh_repo}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - progress.done_task() - progress.new_task("fetch patches") - seq = dt.now(datetime.UTC).strftime("%Y%m%d%H%M%S") dst_branch = f"patchset/branch/{gh_repo.replace('/', '--')}--{patches_branch}-{seq}" @@ -811,7 +796,11 @@ def cmd_patchset_publish( try: patches = fetch_custom_patchset_patches( - ceph_repo_path, patches_repo_path, patchset, ctx.github_token + ceph_repo_path, + patches_repo_path, + patchset, + ctx.github_token, + run_locally=ctx.run_locally, ) except Exception as e: progress.stop_error() diff --git a/crt/src/crt/cmds/patchset.py.orig b/crt/src/crt/cmds/patchset.py.orig deleted file mode 100644 index e783556..0000000 --- a/crt/src/crt/cmds/patchset.py.orig +++ /dev/null @@ -1,1093 +0,0 @@ -# CBS Release Tool - patchset commands -# Copyright (C) 2025 Clyso GmbH -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -import datetime -import errno -import re -import sys -import uuid -from datetime import datetime as dt -from pathlib import Path -from typing import cast - -import click -import pydantic -import rich.box -from rich.console import Group, RenderableType -from rich.padding import Padding -from rich.table import Table - -from crt.cmds import ( - Ctx, - console, - pass_ctx, - perror, - pinfo, - psuccess, - pwarn, - with_patches_repo_path, -) -from crt.cmds import logger as parent_logger -from crt.cmds._common import CRTProgress -from crt.crtlib.errors.patchset import ( - MalformedPatchSetError, -) -from crt.crtlib.git_utils import ( - SHA, - git_branch_delete, - git_get_patch_sha_title, - git_patches_in_interval, - git_prepare_remote, -) -from crt.crtlib.models.common import ( - AuthorData, - ManifestPatchEntry, - ManifestPatchSetEntryType, -) -from crt.crtlib.models.discriminator import ManifestPatchEntryWrapper -from crt.crtlib.models.patch import PatchMeta -from crt.crtlib.models.patchset import ( - CustomPatchMeta, - CustomPatchSet, - GitHubPullRequest, - PatchSetBase, -) -from crt.crtlib.patchset import ( - fetch_custom_patchset_patches, - get_patchset_meta_path, - load_patchset, - write_patchset, -) - -logger = parent_logger.getChild("patchset") - - -def _is_valid_sha(sha: str) -> bool: - return re.match(r"^[\da-f]{4}[\da-f]{0,36}$", sha) is not None - - -def _gen_rich_patch_meta_info_header(table: Table, patch_meta: PatchMeta) -> Table: - table.add_row("sha", f"[orchid2]{patch_meta.sha}[/orchid2]") - table.add_row("title", patch_meta.info.title) - table.add_row( - "author", f"{patch_meta.info.author.user} <{patch_meta.info.author.email}>" - ) - table.add_row("date", f"{patch_meta.info.date}") - table.add_row("src version", patch_meta.src_version or "n/a") - table.add_row("fixes", "\n".join(patch_meta.info.fixes) or "n/a") - return table - - -def _gen_rich_patchset_base_info_header(table: Table, patchset: PatchSetBase) -> Table: - table.add_row("title", patchset.title) - table.add_row("author", f"{patchset.author.user} <{patchset.author.email}>") - table.add_row("created", f"{patchset.creation_date}") - table.add_row("related to", "\n".join(patchset.related_to) or "n/a") - - if isinstance(patchset, GitHubPullRequest): - table.add_row("repo", f"{patchset.org_name}/{patchset.repo_name}") - table.add_row("pr id", str(patchset.pull_request_id)) - table.add_row("updated on", str(patchset.updated_date) or "n/a") - table.add_row("merged", "Yes" if patchset.merged else "No") - if patchset.merged: - table.add_row("merged on", str(patchset.merge_date) or "n/a") - table.add_row("target branch", patchset.target_branch) - table.add_row("patches", str(len(patchset.patches))) - - elif isinstance(patchset, CustomPatchSet): - desc = patchset.description_text - table.add_row("description", desc or "n/a") - table.add_row("release", patchset.release_name or "n/a") - table.add_row( - "patches", str(sum(len(meta.patches) for meta in patchset.patches_meta)) - ) - table.add_row( - "published", - "[green]Yes[/green]" if patchset.is_published else "[red]No[/red]", - ) - - return table - - -def _gen_rich_patchset_info_header(table: Table, patchset: ManifestPatchEntry) -> Table: - if isinstance(patchset, PatchMeta): - return _gen_rich_patch_meta_info_header(table, patchset) - elif isinstance(patchset, PatchSetBase): - return _gen_rich_patchset_base_info_header(table, patchset) - else: - perror(f"unknown patch set type: {type(patchset)}") - sys.exit(errno.ENOTRECOVERABLE) - - -def _gen_rich_patchset_info(patchset: ManifestPatchEntry) -> RenderableType: - header_table = Table(show_header=False, box=None, expand=False) - header_table.add_column(justify="left", style="bold cyan", no_wrap=False) - header_table.add_column(justify="left", style="orange3", no_wrap=False) - - header_table.add_row("uuid", f"[gold1]{patchset.entry_uuid}[/gold1]") - header_table.add_row("type", patchset.entry_type.value) - header_table = _gen_rich_patchset_info_header(header_table, patchset) - - patch_lst_table = Table( - show_header=False, - show_lines=False, - box=rich.box.HORIZONTALS, - ) - patch_lst_table.add_column(justify="left", style="bold gold1", no_wrap=True) - patch_lst_table.add_column(justify="left", style="white", no_wrap=False) - - if isinstance(patchset, GitHubPullRequest): - for patch in patchset.patches: - patch_lst_table.add_row(patch.sha, patch.title) - elif isinstance(patchset, CustomPatchSet): - for entry in patchset.patches_meta: - for sha, title in entry.patches: - patch_lst_table.add_row(sha, title) - - return Group(header_table, patch_lst_table) - - -def _gen_rich_patchset_list() -> Table: - table = Table( - show_header=False, - show_lines=True, - box=rich.box.HORIZONTALS, - ) - # uuids - table.add_column(justify="left", style="bold gold1", no_wrap=True) - # type - table.add_column(justify="left", style="bold cyan", no_wrap=True) - # freeform entry - table.add_column(justify="left", style="white", no_wrap=False) - return table - - -def _gen_rich_patch_meta(patch_meta: PatchMeta) -> RenderableType: - version = patch_meta.src_version if patch_meta.src_version else "n/a" - - # mimic what we would do with 'Columns', except that those will always take all - # the available width within the console. And we want it to just fit to contents. - t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - t1.add_column(justify="left") - t1.add_column(justify="left") - t1.add_row( - f"[orchid2]{patch_meta.sha}[/orchid2]", - f"[italic]version:[/italic] [orange3]{version}[/orange3]", - ) - - t2 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - t2.add_column(justify="left") - t2.add_column(justify="left") - t2.add_row( - f"[cyan]{patch_meta.info.date}[/cyan]", - f"[cyan]{patch_meta.info.author.user} " - + f"<{patch_meta.info.author.email}>[/cyan]", - ) - - entries: list[RenderableType] = [ - f"[bold magenta]{patch_meta.info.title}[/bold magenta]", - t1, - t2, - ] - group = Group(*entries) - return group - - -def _gen_rich_patchset_gh(patchset: GitHubPullRequest) -> RenderableType: - is_merged = "[green]merged[/green]" if patchset.merged else "[red]not merged[/red]" - repo_pr_id = f"{patchset.org_name}/{patchset.repo_name} #{patchset.pull_request_id}" - t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - t1.add_row( - f"[orchid2]{repo_pr_id}[/orchid2] ({is_merged})", - f"[italic]version:[/italic] [orange3]{patchset.target_branch}[/orange3]", - ) - - updated = patchset.updated_date if patchset.updated_date else "n/a" - merged = patchset.merge_date if patchset.merge_date else "n/a" - t2 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - t2.add_row( - f"[italic]updated:[/italic] [cyan]{updated}[/cyan]", - f"[italic]merged:[/italic] [cyan]{merged}[/cyan]", - ) - - return Group(t1, t2) - - -def _gen_rich_patchset_custom(patchset: CustomPatchSet) -> RenderableType: - release = patchset.release_name if patchset.release_name else "n/a" - is_published = "[green]Yes[/green]" if patchset.is_published else "[red]No[/red]" - t1 = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - n_patches = sum(len(meta.patches) for meta in patchset.patches_meta) - t1.add_row( - f"[italic]release:[/italic] [orange3]{release}[/orange3]", - f"[italic]published:[/italic] {is_published}", - f"[italic]patches:[/italic] [orange3]{n_patches}[/orange3]", - ) - return t1 - - -def _gen_rich_patchset_base(patchset: PatchSetBase) -> RenderableType: - # mimic what we would do with 'Columns', except that those will always take all - # the available width within the console. And we want it to just fit to contents. - - t = Table(show_header=False, box=None, expand=False, padding=(0, 2, 0, 0)) - t.add_row( - f"[cyan]{patchset.creation_date}[/cyan]", - f"[cyan]{patchset.author.user} " + f"<{patchset.author.email}>[/cyan]", - ) - - rdr: RenderableType - if isinstance(patchset, GitHubPullRequest): - rdr = _gen_rich_patchset_gh(patchset) - elif isinstance(patchset, CustomPatchSet): - rdr = _gen_rich_patchset_custom(patchset) - else: - perror(f"unknown base patch set type: {type(patchset)}") - sys.exit(errno.ENOTRECOVERABLE) - - entries = [ - f"[bold magenta]{patchset.title}[/bold magenta]", - Group(t, rdr), - ] - - return Group(*entries) - - -def _add_rich_patchset_entry(table: Table, patchset: ManifestPatchEntry) -> None: - rdr: RenderableType - if isinstance(patchset, PatchMeta): - rdr = _gen_rich_patch_meta(patchset) - elif isinstance(patchset, PatchSetBase): - rdr = _gen_rich_patchset_base(patchset) - else: - perror(f"unknown patch set type: {type(patchset)}") - sys.exit(errno.ENOTRECOVERABLE) - - row: tuple[str, str, RenderableType] = ( - str(patchset.entry_uuid), - patchset.entry_type.value, - rdr, - ) - table.add_row(*row) - pass - - -@click.group("patchset", help="Handle patch sets.") -def cmd_patchset() -> None: - pass - - -@cmd_patchset.command("create", help="Create a patch set from individual patches.") -@click.option( - "--author", - "author_name", - required=True, - type=str, - metavar="NAME", - help="Author's name.", -) -@click.option( - "--email", - "author_email", - required=True, - type=str, - metavar="EMAIL", - help="Author's email.", -) -@click.option( - "--title", - "-T", - "patchset_title", - required=False, - default="", - type=str, - metavar="TEXT", - help="Title for this patch set.", -) -@click.option( - "--desc", - "-D", - "patchset_desc", - required=False, - type=str, - metavar="TEXT", - help="Short description of this patch set.", -) -@click.option( - "-r", - "--release-name", - "release_name", - required=False, - type=str, - metavar="NAME", - help="Release associated with this patch set.", -) -@with_patches_repo_path -def cmd_patchset_create( - patches_repo_path: Path, - author_name: str, - author_email: str, - patchset_title: str | None, - patchset_desc: str | None, - release_name: str | None, -) -> None: - print("prompt?") - if not patchset_title: - patchset_title = cast( - str | None, click.prompt("Patch set title", type=str, prompt_suffix=" > ") - ) - if not patchset_title or not patchset_title.strip(): - perror("must specify a patch set title") - sys.exit(errno.EINVAL) - - if patchset_desc and not patchset_desc.strip(): - perror("patch set description is empty") - sys.exit(errno.EINVAL) - - author_info = f"{author_name} <{author_email}>" - if not patchset_desc and click.confirm("Add a description?", default=False): - try: - desc_msg = ( - click.edit( - f"{patchset_title}\n\n\n\n" - + f"Signed-off-by: {author_info}" - ), - ) - except Exception as e: - perror(f"unable to open editor: {e}") - sys.exit(errno.ENOTRECOVERABLE) - - print(desc_msg) - if not desc_msg or not desc_msg[0]: - perror("must specify a patch set description") - sys.exit(errno.EINVAL) - - patchset_desc = desc_msg[0].strip() - - if not patchset_desc: - patchset_desc = f"{patchset_title}\n\nSigned-off-by: {author_info}" - - patchset = CustomPatchSet( - author=AuthorData(user=author_name, email=author_email), - creation_date=dt.now(datetime.UTC), - title=patchset_title, - related_to=[], - patches=[], - description=patchset_desc, - release_name=release_name, - ) - - patchset_meta_path = get_patchset_meta_path(patches_repo_path, patchset.entry_uuid) - assert not patchset_meta_path.exists() - - try: - write_patchset(patches_repo_path, patchset) - except Exception as e: - perror(f"unable to write patch set meta file: {e}") - sys.exit(errno.ENOTRECOVERABLE) - - psuccess(f"successfully created patch set '{patchset.entry_uuid}'") - - -@cmd_patchset.command("list", help="List patch sets in the patches repository.") -@click.option( - "-t", - "--type", - "patchset_types", - type=str, - multiple=True, - required=False, - metavar="TYPE", - help="Filter by patch set type.", -) -@with_patches_repo_path -def cmd_patchset_list(patches_repo_path: Path, patchset_types: list[str]) -> None: - meta_path = patches_repo_path / "ceph" / "patches" / "meta" - - avail_types = [m.value for m in ManifestPatchSetEntryType] - if patchset_types and any(t not in avail_types for t in patchset_types): - perror(f"unknown patch set type(s), available: {', '.join(avail_types)}") - sys.exit(errno.EINVAL) - - table = _gen_rich_patchset_list() - for patchset_path in meta_path.glob("*.json"): - try: - patchset_uuid = uuid.UUID(patchset_path.stem) - except Exception: - pwarn(f"malformed patch set uuid in '{patchset_path}', skip") - continue - - try: - patchset = load_patchset(patches_repo_path, patchset_uuid) - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - if patchset_types and patchset.entry_type.value not in patchset_types: - continue - - _add_rich_patchset_entry(table, patchset) - - if len(table.rows) > 0: - console.print(table) - else: - pwarn("no entries found") - - -@cmd_patchset.command("info", help="Obtain info on a given patch set.") -@click.option( - "-u", - "--patchset-uuid", - "patchset_uuid", - type=uuid.UUID, - required=True, - help="Patch set UUID.", -) -@with_patches_repo_path -def cmd_patchset_info(patches_repo_path: Path, patchset_uuid: uuid.UUID) -> None: - try: - patchset = load_patchset(patches_repo_path, patchset_uuid) - except MalformedPatchSetError as e: - perror(f"malformed patch set '{patchset_uuid}': {e}") - sys.exit(errno.EINVAL) - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - console.print(_gen_rich_patchset_info(patchset)) - - -@cmd_patchset.command("add", help="Add one or more patches to a patch set.") -@click.option( - "-c", - "--ceph-repo", - "ceph_repo_path", - type=click.Path( - exists=True, file_okay=False, dir_okay=True, resolve_path=True, path_type=Path - ), - required=True, - envvar="CRT_CEPH_REPO_PATH", - help="Path to ceph git repository where operations will be performed.", -) -@click.option( - "-u", - "--patchset-uuid", - "patchset_uuid", - type=uuid.UUID, - required=True, - help="Patch set UUID.", -) -@click.option( - "--gh-repo", - "gh_repo", - type=str, - required=False, - default="ceph/ceph", - metavar="OWNER/REPO", - help="GitHub repository to obtain the patch(es) from.", -) -@click.option( - "-b", - "--branch", - "patches_branch", - type=str, - required=True, - metavar="NAME", - help="Branch on which to find patches.", -) -@click.argument( - "patch_sha", - metavar="SHA|SHA1..SHA2 [...]", - type=str, - required=True, - nargs=-1, -) -@with_patches_repo_path -@pass_ctx -def cmd_patchset_add( - ctx: Ctx, - patches_repo_path: Path, - ceph_repo_path: Path, - patchset_uuid: uuid.UUID, - gh_repo: str, - patches_branch: str, - patch_sha: list[str], -) -> None: - if not ctx.github_token: - perror("github token not specified") - sys.exit(errno.EINVAL) - - try: - patchset = load_patchset(patches_repo_path, patchset_uuid) - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - if not isinstance(patchset, CustomPatchSet): - perror(f"patch set '{patchset_uuid}' is not a custom patch set") - sys.exit(errno.EINVAL) - - if patchset.is_published: - perror(f"patch set '{patchset_uuid}' is already published") - sys.exit(errno.EINVAL) - - existing_shas = [p[0] for meta in patchset.patches_meta for p in meta.patches] - - progress = CRTProgress(console) - progress.start() - progress.new_task(f"add patches from '{gh_repo}' branch '{patches_branch}'") - - # ensure we have the specified branch in the ceph repo, so we can actually obtain - # the right shas - progress.new_task("prepare remote") - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token - ) - except Exception as e: - progress.stop_error() - perror(f"unable to update remote '{gh_repo}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - progress.done_task() - progress.new_task("fetch patches") - - seq = dt.now(datetime.UTC).strftime("%Y%m%d%H%M%S") - dst_branch = f"patchset/branch/{gh_repo.replace('/', '--')}--{patches_branch}-{seq}" -<<<<<<< HEAD - try: - _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") - except Exception as e: - progress.stop_error() - perror(f"unable to fetch branch '{patches_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) -||||||| parent of 716fbc5 ( fixup f95accd9) - - if not ctx.run_locally: - # ensure we have the specified branch in the ceph repo, so we can actually - # obtain the right shas - progress.new_task("prepare remote") - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token - ) - except Exception as e: - progress.stop_error() - perror(f"unable to update remote '{gh_repo}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - progress.done_task() - - progress.new_task("fetch patches") - - try: - _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") - except Exception as e: - progress.stop_error() - perror(f"unable to fetch branch '{patches_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - else: - progress.new_task("create branch patches") - try: - git_branch_from(ceph_repo_path, patches_branch, dst_branch) - except GitError as e: - progress.stop_error() - perror(f"unable to create branch '{dst_branch} from {patches_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) -======= - - if not ctx.run_locally: - # ensure we have the specified branch in the ceph repo, so we can actually - # obtain the right shas - progress.new_task("prepare remote") - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{gh_repo}", gh_repo, ctx.github_token - ) - except Exception as e: - progress.stop_error() - perror(f"unable to update remote '{gh_repo}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - progress.done_task() - - progress.new_task("fetch patches") - - try: - _ = remote.fetch(refspec=f"{patches_branch}:{dst_branch}") - except Exception as e: - progress.stop_error() - perror(f"unable to fetch branch '{patches_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - else: - progress.new_task("create branch patches") - try: - git_branch_from(ceph_repo_path, patches_branch, dst_branch) - except GitError as e: - progress.stop_error() - perror( - f"unable to create branch '{dst_branch}' from '{patches_branch}': {e}" - ) - sys.exit(errno.ENOTRECOVERABLE) ->>>>>>> 716fbc5 ( fixup f95accd9) - - def _cleanup() -> None: - try: - git_branch_delete(ceph_repo_path, dst_branch) - except Exception as e: - progress.stop_error() - perror(f"unable to delete temporary branch '{dst_branch}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - patches_meta_lst: list[CustomPatchMeta] = [] - skipped_patches: list[tuple[SHA, str]] = [] - for sha_entry in patch_sha: - sha_begin: SHA - sha_end: SHA | None = None - if ".." in sha_entry: - interval = sha_entry.split("..", 1) - if len(interval) != 2 or not interval[0] or not interval[1]: - perror(f"malformed patch interval '{sha_entry}'") - sys.exit(errno.EINVAL) - sha_begin, sha_end = interval[0], interval[1] - else: - sha_begin = sha_entry - - if not _is_valid_sha(sha_begin) or (sha_end and not _is_valid_sha(sha_end)): - _cleanup() - progress.stop_error() - perror(f"malformed patch sha '{sha_entry}'") - sys.exit(errno.EINVAL) - - patches_lst: list[tuple[SHA, str]] = [] - if sha_end: - try: - for sha, title in reversed( - git_patches_in_interval(ceph_repo_path, sha_begin, sha_end) - ): - if sha not in existing_shas: - patches_lst.append((sha, title)) - else: - skipped_patches.append((sha, title)) - except Exception as e: - _cleanup() - progress.stop_error() - perror(f"unable to obtain patches in interval '{sha_entry}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - else: - try: - sha, title = git_get_patch_sha_title(ceph_repo_path, sha_begin) - if sha not in existing_shas: - patches_lst.append((sha, title)) - else: - skipped_patches.append((sha, title)) - except Exception as e: - _cleanup() - progress.stop_error() - perror(f"unable to obtain patch info for sha '{sha_entry}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - if patches_lst: - patches_meta_lst.append( - CustomPatchMeta( - repo=gh_repo, - branch=patches_branch, - sha=sha_begin, - sha_end=sha_end, - patches=patches_lst, - ) - ) - - progress.done_task() - progress.done_task() - progress.stop() - - if patchset.patches_meta: - existing_patches_table = Table( - title="Existing patches", - title_style="bold magenta", - show_header=False, - show_lines=False, - box=rich.box.HORIZONTALS, - ) - existing_patches_table.add_column( - justify="left", style="bold cyan", no_wrap=True - ) - existing_patches_table.add_column(justify="left", style="white", no_wrap=False) - - for existing_entry in patchset.patches_meta: - for sha, title in existing_entry.patches: - existing_patches_table.add_row(sha, title) - - console.print(Padding(existing_patches_table, (1, 0, 0, 0))) - - if skipped_patches: - skipped_patches_table = Table( - title="Skipped patches", - title_style="bold orange3", - show_header=False, - show_lines=False, - box=rich.box.HORIZONTALS, - ) - skipped_patches_table.add_column( - justify="left", style="bold hot_pink", no_wrap=True - ) - skipped_patches_table.add_column(justify="left", style="white", no_wrap=False) - - for sha, title in skipped_patches: - skipped_patches_table.add_row(sha, title) - - console.print(Padding(skipped_patches_table, (1, 0, 0, 0))) - - if not patches_meta_lst: - console.print( - Padding("[bold yellow]No new patches to add[/bold yellow]", (1, 0, 1, 2)) - ) - return - else: - patch_lst_table = Table( - title="Patches to add", - title_style="bold green", - show_header=False, - show_lines=False, - box=rich.box.HORIZONTALS, - ) - patch_lst_table.add_column(justify="left", style="bold gold1", no_wrap=True) - patch_lst_table.add_column(justify="left", style="white", no_wrap=False) - - for entry in patches_meta_lst: - for sha, title in entry.patches: - patch_lst_table.add_row(sha, title) - - console.print(Padding(patch_lst_table, (1, 0, 1, 0))) - if not click.confirm("Add above patches to patch set?", prompt_suffix=" "): - pwarn("Aborted") - sys.exit(0) - - patchset.patches_meta.extend(patches_meta_lst) - - try: - write_patchset(patches_repo_path, patchset) - except Exception as e: - _cleanup() - perror(f"unable to write patch set meta file: {e}") - sys.exit(errno.ENOTRECOVERABLE) - - n_patches = sum(len(meta.patches) for meta in patches_meta_lst) - psuccess(f"wrote patch set '{patchset.entry_uuid}', {n_patches} new patches") - _cleanup() - - -@cmd_patchset.command("publish", help="Publish a patch set to the patches repository.") -@click.option( - "-c", - "--ceph-repo", - "ceph_repo_path", - type=click.Path( - exists=True, file_okay=False, dir_okay=True, resolve_path=True, path_type=Path - ), - envvar="CRT_CEPH_REPO_PATH", - required=True, - help="Path to ceph git repository where operations will be performed.", -) -@click.option( - "-u", - "--patchset-uuid", - "patchset_uuid", - type=uuid.UUID, - required=True, - help="Patch set UUID.", -) -@with_patches_repo_path -@pass_ctx -def cmd_patchset_publish( - ctx: Ctx, - patches_repo_path: Path, - ceph_repo_path: Path, - patchset_uuid: uuid.UUID, -) -> None: - if not ctx.github_token: - perror("missing GitHub token") - sys.exit(errno.EINVAL) - - try: - patchset = load_patchset(patches_repo_path, patchset_uuid) - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - if not isinstance(patchset, CustomPatchSet): - perror(f"patch set '{patchset_uuid}' is not a custom patch set") - sys.exit(errno.EINVAL) - - if patchset.is_published: - perror(f"patch set '{patchset_uuid}' is already published") - sys.exit(errno.EINVAL) - - if not patchset.patches_meta or all( - len(meta.patches) == 0 for meta in patchset.patches_meta - ): - perror(f"patch set '{patchset_uuid}' has no patches") - sys.exit(errno.EINVAL) - - console.print(_gen_rich_patchset_info(patchset)) - if not click.confirm("Publish above patch set?", prompt_suffix=" "): - pwarn("Aborted") - sys.exit(0) - - progress = CRTProgress(console) - progress.start() - progress.new_task("fetch patches") - - try: - patches = fetch_custom_patchset_patches( - ceph_repo_path, patches_repo_path, patchset, ctx.github_token - ) - except Exception as e: - progress.stop_error() - perror(f"unable to fetch patches for patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - progress.done_task() - - patchset.patches = patches - patchset.is_published = True - - try: - write_patchset(patches_repo_path, patchset) - except Exception as e: - perror(f"unable to write patch set meta file: {e}") - sys.exit(errno.ENOTRECOVERABLE) - - psuccess(f"published patch set '{patchset.entry_uuid}'") - - -@cmd_patchset.command("remove", help="Remove an unpublished patch set.") -@click.option( - "-u", - "--patchset-uuid", - "patchset_uuid", - type=uuid.UUID, - required=True, - help="Patch set UUID.", -) -@with_patches_repo_path -def cmd_patchset_remove(patches_repo_path: Path, patchset_uuid: uuid.UUID) -> None: - patchset_meta_path = get_patchset_meta_path(patches_repo_path, patchset_uuid) - if not patchset_meta_path.exists(): - perror(f"patch set '{patchset_uuid}' does not exist") - sys.exit(errno.ENOENT) - - try: - patchset = load_patchset(patches_repo_path, patchset_uuid) - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - if not isinstance(patchset, CustomPatchSet): - perror(f"patch set '{patchset_uuid}' is not a custom patch set") - sys.exit(errno.EINVAL) - - if patchset.is_published: - perror(f"patch set '{patchset_uuid}' is already published") - sys.exit(errno.EINVAL) - - console.print(_gen_rich_patchset_info(patchset)) - if not click.confirm("Remove above patch set?", prompt_suffix=" "): - pwarn("Aborted") - sys.exit(0) - - try: - patchset_meta_path.unlink() - except Exception as e: - perror(f"unable to remove patch set meta file: {e}") - sys.exit(errno.ENOTRECOVERABLE) - - psuccess(f"removed patch set '{patchset.entry_uuid}'") - - -@cmd_patchset.group("advanced", help="Advanced patch set operations.") -def cmd_patchset_advanced() -> None: - pass - - -@cmd_patchset_advanced.command( - "migrate-store-format", help="Migrate patch sets' store format" -) -@with_patches_repo_path -def cmd_patchset_migrate_store_format(patches_repo_path: Path) -> None: - if not patches_repo_path.exists(): - perror(f"patches repository does not exist at '{patches_repo_path}'") - sys.exit(errno.ENOENT) - - if not patches_repo_path.joinpath(".git").exists(): - perror("provided path for patches repository is not a git repository") - sys.exit(errno.EINVAL) - - patches_path = patches_repo_path / "ceph" / "patches" - if not patches_path.exists(): - pinfo(f"patches path does not exist at '{patches_path}', nothing to do") - return - - n_patchsets = 0 - candidate_dirs: list[Path] = [] - for p in patches_path.iterdir(): - if p.is_dir() and p.name != "meta": - candidate_dirs.append(p) - - print(candidate_dirs) - for d in candidate_dirs: - for p in list(d.walk()): - for sub in p[1]: - sub_path = Path(p[0]) / sub - if not sub_path.is_dir(): - continue - - if not re.match(r"^[\w\d_.-]+$", sub): - # not a valid repo name - continue - - repo_name = f"{d.name}/{sub}" - - for pr in sub_path.iterdir(): - if pr.is_dir(): - continue - - if not re.match(r"^\d+$", pr.name): - # not a valid pr id - pwarn(f"skip invalid pr id '{pr.name}' in '{repo_name}'") - continue - - try: - patchset_uuid = uuid.UUID(pr.read_text()) - except Exception: - pwarn( - f"malformed patch set uuid in '{pr}' in '{repo_name}', skip" - ) - continue - - pinfo(f"pr id '{pr.name}' uuid '{patchset_uuid}' in '{repo_name}'") - latest_patchset_path = patches_path / f"{patchset_uuid}.patch" - latest_meta_path = patches_path / "meta" / f"{patchset_uuid}.json" - - if ( - not latest_patchset_path.exists() - and not latest_meta_path.exists() - ): - pwarn( - f"missing patch file '{latest_patchset_path}', " - + "skip migration" - ) - continue - - try: - patchset_meta = ManifestPatchEntryWrapper.model_validate_json( - latest_meta_path.read_text() - ) - except pydantic.ValidationError as e: - perror(f"malformed meta file '{latest_meta_path}': {e}") - continue - - if not isinstance(patchset_meta.contents, GitHubPullRequest): - perror( - f"found meta for patchset uuid '{patchset_uuid}' " - + "is not a gh pr" - ) - continue - - patchset = patchset_meta.contents - if not patchset.patches: - perror( - f"found empty patch set for uuid '{patchset_uuid}' " - + f"pr id '{pr.name}' repo '{repo_name}'" - ) - continue - - head_patch_sha = next(reversed(patchset.patches)).sha - pinfo( - f"pr id '{pr.name}' repo '{repo_name}' " - + f"head patch sha '{head_patch_sha}'" - ) - head_path_sha_path = pr / head_patch_sha - latest_path = pr / "latest" - - try: - pr.unlink() - pr.mkdir() - _ = head_path_sha_path.write_text(str(patchset_uuid)) - latest_path.symlink_to(head_patch_sha) - except Exception as e: - perror(f"unable to migrate pr id '{pr.name}': {e}") - continue - - psuccess( - f"successfully migrated pr id '{pr.name}' repo '{repo_name}'" - ) - n_patchsets += 1 - - psuccess(f"successfully migrated {n_patchsets} patch sets") - - -@cmd_patchset_advanced.command("migrate-single-patches", help="Migrate single patches.") -@with_patches_repo_path -def cmd_patchset_migrate_single_patches(patches_repo_path: Path) -> None: - meta_path = patches_repo_path / "ceph" / "patches" / "meta" - - n_patchsets = 0 - for patchset_path in meta_path.glob("*.json"): - try: - patchset_uuid = uuid.UUID(patchset_path.stem) - except Exception: - pwarn(f"malformed patch set uuid in '{patchset_path}', skip") - continue - - try: - _ = load_patchset(patches_repo_path, patchset_uuid) - continue - except MalformedPatchSetError: - # possibly a single patch, check and migrate it if so. - pinfo(f"possible single patch '{patchset_uuid}', check") - except Exception as e: - perror(f"unable to load patch set '{patchset_uuid}': {e}") - continue - - try: - single_patch = PatchMeta.model_validate_json(patchset_path.read_text()) - except Exception: - perror(f"unable to parse single patch meta '{patchset_path}'") - continue - - pinfo(f"found single patch at '{single_patch.entry_uuid}', migrate") - # backup existing meta file - bak_path = patchset_path.with_suffix(".json.bak") - try: - _ = bak_path.write_text(patchset_path.read_text()) - except Exception as e: - perror(f"unable to backup single patch meta '{patchset_path}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - try: - _ = patchset_path.write_text( - ManifestPatchEntryWrapper(contents=single_patch).model_dump_json( - indent=2 - ) - ) - except Exception as e: - perror(f"unable to migrate single patch meta '{patchset_path}': {e}") - sys.exit(errno.ENOTRECOVERABLE) - - psuccess(f"successfully migrated single patch '{patchset_uuid}'") - bak_path.unlink() - n_patchsets += 1 - - pinfo(f"migrated {n_patchsets} single patches") diff --git a/crt/src/crt/crtlib/patchset.py b/crt/src/crt/crtlib/patchset.py index 122bf7a..e37ae9a 100644 --- a/crt/src/crt/crtlib/patchset.py +++ b/crt/src/crt/crtlib/patchset.py @@ -30,6 +30,7 @@ GitError, GitPatchDiffError, git_branch_delete, + git_branch_from, git_check_patches_diff, git_format_patch, git_prepare_remote, @@ -349,6 +350,8 @@ def fetch_custom_patchset_patches( patches_repo_path: Path, patchset: CustomPatchSet, token: str, + *, + run_locally: bool = False, ) -> list[Patch]: """Fetch and store a custom patch set's patches into the patches repository.""" if patchset.is_published: @@ -378,18 +381,29 @@ def fetch_custom_patchset_patches( if dst_branch in fetched_branches: continue - try: - remote = git_prepare_remote( - ceph_repo_path, f"github.com/{meta.repo}", meta.repo, token - ) - _ = remote.fetch(refspec=f"{meta.branch}:{dst_branch}") - except Exception as e: - msg = ( - f"error fetching patchset branch '{meta.branch}' " - + f"from '{meta.repo}': {e}" - ) - logger.error(msg) - raise PatchSetError(msg=msg) from None + if run_locally: + try: + git_branch_from(ceph_repo_path, meta.branch, dst_branch) + except GitError as e: + msg = ( + f"error creating patchset branch '{dst_branch}' " + f"from local branch '{meta.branch}' (repo '{meta.repo}'): {e}" + ) + logger.error(msg) + raise PatchSetError(msg=msg) from None + else: + try: + remote = git_prepare_remote( + ceph_repo_path, f"github.com/{meta.repo}", meta.repo, token + ) + _ = remote.fetch(refspec=f"{meta.branch}:{dst_branch}") + except Exception as e: + msg = ( + f"error fetching patchset branch '{meta.branch}' " + + f"from '{meta.repo}': {e}" + ) + logger.error(msg) + raise PatchSetError(msg=msg) from None fetched_branches.add(dst_branch)