diff --git a/ci.py b/ci.py index 713294e9..9dcbde04 100644 --- a/ci.py +++ b/ci.py @@ -60,9 +60,7 @@ from threading import Lock, Thread from typing import Any, Callable, Protocol, cast -PROJECT_ROOT = Path(__file__).resolve().parent - -from simpler.task_interface import ( # noqa: E402 # type: ignore[import-not-found] +from simpler.task_interface import ( # type: ignore[import-not-found] ChipCallable, # pyright: ignore[reportAttributeAccessIssue] ChipCallConfig, # pyright: ignore[reportAttributeAccessIssue] ChipStorageTaskArgs, # pyright: ignore[reportAttributeAccessIssue] @@ -72,6 +70,10 @@ scalar_to_uint64, ) +from simpler_setup.log_config import DEFAULT_LOG_LEVEL, LOG_LEVEL_CHOICES, configure_logging + +PROJECT_ROOT = Path(__file__).resolve().parent + logger = logging.getLogger("ci") # --------------------------------------------------------------------------- @@ -264,16 +266,17 @@ def discover_tasks(platform: str, runtime_filter: str | None = None) -> list[Tas def ensure_pto_isa(commit: str | None, clone_protocol: str) -> str: - from simpler_setup.code_runner import _ensure_pto_isa_root # noqa: PLC0415 - - root = _ensure_pto_isa_root(verbose=True, commit=commit, clone_protocol=clone_protocol) - if root is None: - raise OSError( - "PTO_ISA_ROOT could not be resolved.\n" - "Set it manually or let auto-clone run:\n" - " export PTO_ISA_ROOT=$(pwd)/examples/scripts/_deps/pto-isa" - ) - return root + from simpler_setup.pto_isa import ensure_pto_isa_root # noqa: PLC0415 + + # update_if_exists=True: when no commit is pinned, fetch latest origin/HEAD + # so CI runs reproducibly track main rather than whatever local checkout + # happens to be on disk. + return ensure_pto_isa_root( + commit=commit, + clone_protocol=clone_protocol, + update_if_exists=True, + verbose=True, + ) # --------------------------------------------------------------------------- @@ -952,11 +955,11 @@ def print_summary(results: list[TaskResult]) -> int: def reset_pto_isa(commit: str, clone_protocol: str) -> str: """Checkout PTO-ISA at the pinned commit (or re-clone if needed).""" - from simpler_setup.code_runner import _checkout_pto_isa_commit, _get_pto_isa_clone_path # noqa: PLC0415 + from simpler_setup.pto_isa import checkout_pto_isa_commit, get_pto_isa_clone_path # noqa: PLC0415 - clone_path = _get_pto_isa_clone_path() + clone_path = get_pto_isa_clone_path() if clone_path.exists(): - _checkout_pto_isa_commit(clone_path, commit, verbose=True) + checkout_pto_isa_commit(clone_path, commit, verbose=True) return str(clone_path.resolve()) return ensure_pto_isa(commit, clone_protocol) @@ -1177,6 +1180,9 @@ def parse_args() -> argparse.Namespace: parser.add_argument("-t", "--timeout", type=int, default=600) parser.add_argument("--clone-protocol", choices=["ssh", "https"], default="ssh") parser.add_argument("--all", dest="run_all_cases", action="store_true", help="Run all cases, not just DEFAULT_CASE") + parser.add_argument( + "--log-level", choices=LOG_LEVEL_CHOICES, default=DEFAULT_LOG_LEVEL, help="Root logger level (default: info)" + ) parser.add_argument("--device-worker", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--result-json", default=None, help=argparse.SUPPRESS) parser.add_argument("--task-list-json", default=None, help=argparse.SUPPRESS) @@ -1276,9 +1282,8 @@ def _run_single_platform(platform: str, args: argparse.Namespace) -> list[TaskRe def main() -> int: - logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s", force=True) - args = parse_args() + configure_logging(args.log_level) args.devices = parse_device_range(args.device_range) valid_platforms = _discover_valid_platforms() diff --git a/examples/scripts/run_example.py b/examples/scripts/run_example.py index e2b29382..83075a23 100644 --- a/examples/scripts/run_example.py +++ b/examples/scripts/run_example.py @@ -40,6 +40,9 @@ import time from pathlib import Path +from simpler_setup.code_runner import create_code_runner +from simpler_setup.log_config import DEFAULT_LOG_LEVEL, LOG_LEVEL_CHOICES, configure_logging + project_root = Path(__file__).parent.parent.parent logger = logging.getLogger(__name__) @@ -127,23 +130,11 @@ def compute_golden(tensors: dict, params: dict) -> None: help="Platform name: 'a2a3'/'a5' for hardware, 'a2a3sim'/'a5sim' for simulation (default: a2a3)", ) - parser.add_argument( - "-v", - "--verbose", - action="store_true", - help="Enable verbose output (equivalent to --log-level debug)", - ) - - parser.add_argument( - "--silent", - action="store_true", - help="Silent mode - only show errors (equivalent to --log-level error)", - ) - parser.add_argument( "--log-level", - choices=["error", "warn", "info", "debug"], - help="Set log level explicitly (overrides --verbose and --silent)", + choices=LOG_LEVEL_CHOICES, + default=DEFAULT_LOG_LEVEL, + help=f"Root logger level (default: {DEFAULT_LOG_LEVEL})", ) parser.add_argument( @@ -206,31 +197,7 @@ def compute_golden(tensors: dict, params: dict) -> None: if args.all and args.case: parser.error("--all and --case are mutually exclusive") - # Determine log level from arguments - log_level_str = None - if args.log_level: - log_level_str = args.log_level - elif args.verbose: - log_level_str = "debug" - elif args.silent: - log_level_str = "error" - else: - log_level_str = "info" - - # Setup logging before any other operations - level_map = { - "error": logging.ERROR, - "warn": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - } - log_level = level_map.get(log_level_str.lower(), logging.INFO) - - # Configure Python logging - logging.basicConfig(level=log_level, format="[%(levelname)s] %(message)s", force=True) - - # Set environment variable for C++ side - os.environ["PTO_LOG_LEVEL"] = log_level_str + configure_logging(args.log_level) # Validate paths kernels_path = Path(args.kernels) @@ -249,10 +216,7 @@ def compute_golden(tensors: dict, params: dict) -> None: logger.error(f"kernel_config.py not found in {kernels_path}") return 1 - # Import and run try: - from simpler_setup.code_runner import create_code_runner # noqa: PLC0415 - runner = create_code_runner( kernels_dir=str(args.kernels), golden_path=str(args.golden), @@ -310,7 +274,7 @@ def compute_golden(tensors: dict, params: dict) -> None: else: cmd += ["-d", str(args.device)] - if log_level_str == "debug": + if logger.isEnabledFor(logging.DEBUG): cmd.append("-v") result = subprocess.run(cmd, check=True, capture_output=True, text=True) @@ -318,8 +282,7 @@ def compute_golden(tensors: dict, params: dict) -> None: logger.info("Swimlane JSON generation completed") except subprocess.CalledProcessError as e: logger.warning(f"Failed to generate swimlane JSON: {e}") - if log_level_str == "debug": - logger.debug(f"stderr: {e.stderr}") + logger.debug(f"stderr: {e.stderr}") else: logger.warning(f"Swimlane converter script not found: {swimlane_script}") @@ -332,7 +295,7 @@ def compute_golden(tensors: dict, params: dict) -> None: except Exception as e: logger.error(f"TEST FAILED: {e}") - if log_level_str == "debug": + if logger.isEnabledFor(logging.DEBUG): import traceback # noqa: PLC0415 traceback.print_exc() diff --git a/simpler_setup/code_runner.py b/simpler_setup/code_runner.py index f382aa27..d5e96290 100644 --- a/simpler_setup/code_runner.py +++ b/simpler_setup/code_runner.py @@ -53,7 +53,6 @@ def compute_golden(tensors: dict, params: dict) -> None: """ import ctypes -import fcntl import importlib.util import logging import os @@ -78,25 +77,21 @@ def compute_golden(tensors: dict, params: dict) -> None: scalar_to_uint64, ) +from .environment import PROJECT_ROOT +from .log_config import DEFAULT_LOG_LEVEL, configure_logging +from .pto_isa import ensure_pto_isa_root + logger = logging.getLogger(__name__) -def _setup_logging_if_needed() -> None: - """ - Setup logging if not already configured (for direct CodeRunner usage). - Uses PTO_LOG_LEVEL environment variable or defaults to 'info'. +def _maybe_configure_logging(log_level: Optional[str]) -> None: + """Apply log_level if given; fall back to DEFAULT_LOG_LEVEL only when the + caller hasn't configured logging themselves (e.g. notebook / ad-hoc script). """ - # Only setup if logging hasn't been configured yet - if not logging.getLogger().hasHandlers(): - level_str = os.environ.get("PTO_LOG_LEVEL", "info") - level_map = { - "error": logging.ERROR, - "warn": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - } - log_level = level_map.get(level_str.lower(), logging.INFO) - logging.basicConfig(level=log_level, format="[%(levelname)s] %(message)s", force=True) + if log_level is not None: + configure_logging(log_level) + elif not logging.getLogger().hasHandlers(): + configure_logging(DEFAULT_LOG_LEVEL) def _to_torch(tensor) -> torch.Tensor: @@ -128,286 +123,6 @@ def _load_module_from_path(module_path: Path, module_name: str): return module -def _get_project_root() -> Path: - """Get the project root directory (one level above simpler_setup/).""" - return Path(__file__).parent.parent - - -def _get_pto_isa_clone_path() -> Path: - """Get the expected path to pto-isa clone.""" - return _get_project_root() / "examples" / "scripts" / "_deps" / "pto-isa" - - -def _is_pto_isa_cloned() -> bool: - """ - Check if pto-isa is cloned. - - A clone is considered valid if: - 1. The directory exists - 2. It contains the include directory (essential content) - """ - clone_path = _get_pto_isa_clone_path() - if not clone_path.exists(): - return False - - # Check for essential content - include_dir = clone_path / "include" - return include_dir.exists() and include_dir.is_dir() - - -def _is_git_available() -> bool: - """Check if git command is available.""" - try: - import subprocess # noqa: PLC0415 - - result = subprocess.run(["git", "--version"], check=False, capture_output=True, text=True, timeout=5) - return result.returncode == 0 - except (FileNotFoundError, subprocess.TimeoutExpired): - return False - - -_PTO_ISA_HTTPS = "https://github.com/PTO-ISA/pto-isa.git" -_PTO_ISA_SSH = "git@github.com:PTO-ISA/pto-isa.git" - - -def _pto_isa_repo_url(clone_protocol: str = "ssh") -> str: - """Return the pto-isa clone URL for the given protocol.""" - if clone_protocol == "https": - return _PTO_ISA_HTTPS - return _PTO_ISA_SSH - - -def _clone_pto_isa(verbose: bool = False, commit: Optional[str] = None, clone_protocol: str = "ssh") -> bool: - """ - Clone pto-isa repository, optionally at a specific commit. - - Args: - verbose: Print detailed progress information - commit: If provided, checkout this commit after cloning - - Returns: - True if successful, False otherwise - """ - import subprocess # noqa: PLC0415 - - if not _is_git_available(): - if verbose: - logger.warning("git command not available, cannot clone pto-isa") - return False - - clone_path = _get_pto_isa_clone_path() - - # Create parent deps directory if it doesn't exist - deps_dir = clone_path.parent - try: - deps_dir.mkdir(parents=True, exist_ok=True) - except Exception as e: - if verbose: - logger.warning(f"Failed to create deps directory: {e}") - return False - - try: - if verbose: - logger.info(f"Cloning pto-isa to {clone_path}...") - logger.info("This may take a few moments on first run...") - - repo_url = _pto_isa_repo_url(clone_protocol) - result = subprocess.run( - ["git", "clone", repo_url, str(clone_path)], - check=False, - capture_output=True, - text=True, - timeout=300, # 5 minutes timeout - ) - - if result.returncode != 0: - if verbose: - logger.warning(f"Failed to clone pto-isa:\n{result.stderr}") - return False - - # Checkout specific commit if requested - if commit: - result = subprocess.run( - ["git", "checkout", commit], - check=False, - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=30, - ) - if result.returncode != 0: - if verbose: - logger.warning(f"Failed to checkout pto-isa commit {commit}:\n{result.stderr}") - return False - - if verbose: - suffix = f" at commit {commit}" if commit else "" - logger.info(f"pto-isa cloned successfully{suffix}: {clone_path}") - - return True - - except subprocess.TimeoutExpired: - if verbose: - logger.warning("Clone operation timed out") - return False - except Exception as e: - if verbose: - logger.warning(f"Failed to clone pto-isa: {e}") - return False - - -def _checkout_pto_isa_commit(clone_path: Path, commit: str, verbose: bool = False) -> None: - """Checkout the specified commit if the existing clone is at a different revision.""" - import subprocess # noqa: PLC0415 - - try: - result = subprocess.run( - ["git", "rev-parse", "--short", "HEAD"], - check=False, - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=5, - ) - current = result.stdout.strip() if result.returncode == 0 else "" - if current and not commit.startswith(current) and not current.startswith(commit): - if verbose: - logger.info(f"pto-isa at {current}, checking out {commit}...") - subprocess.run( - ["git", "fetch", "origin"], - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=120, - check=True, - ) - subprocess.run( - ["git", "checkout", commit], - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=30, - check=True, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - logger.warning(f"Failed to checkout pto-isa commit {commit}: {e.stderr if hasattr(e, 'stderr') else e}") - except Exception as e: - logger.warning(f"Unexpected error checking out pto-isa commit {commit}: {e}") - - -def _update_pto_isa_to_latest(clone_path: Path, verbose: bool = False) -> None: - """Fetch and reset existing clone to the remote default branch.""" - import subprocess # noqa: PLC0415 - - try: - if verbose: - logger.info("Updating pto-isa to latest...") - subprocess.run( - ["git", "fetch", "origin"], - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=120, - check=True, - ) - # Use origin/HEAD which tracks the remote's default branch - subprocess.run( - ["git", "reset", "--hard", "origin/HEAD"], - capture_output=True, - text=True, - cwd=str(clone_path), - timeout=30, - check=True, - ) - except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: - logger.warning(f"Failed to update pto-isa to latest: {e.stderr if hasattr(e, 'stderr') else e}") - except Exception as e: - logger.warning(f"Unexpected error updating pto-isa: {e}") - - -def _ensure_pto_isa_root( - verbose: bool = False, commit: Optional[str] = None, clone_protocol: str = "ssh" -) -> Optional[str]: - """ - Ensure PTO_ISA_ROOT is available, either from environment or cloned repo. - - This function: - 1. Checks if PTO_ISA_ROOT is already set - 2. If not, tries to clone pto-isa repository - 3. Returns the resolved path - - Uses a file lock to prevent parallel processes from racing on the clone. - - Args: - verbose: Print detailed progress information - commit: If provided, checkout this specific commit - - Returns: - PTO_ISA_ROOT path if successful, None otherwise - """ - # Check if already set in environment - existing_root = os.environ.get("PTO_ISA_ROOT") - if existing_root: - if verbose: - logger.info(f"Using existing PTO_ISA_ROOT: {existing_root}") - return existing_root - - # Try to use cloned repository - clone_path = _get_pto_isa_clone_path() - - # Use a file lock so only one process clones at a time - lock_path = clone_path.parent / ".pto-isa.lock" - lock_path.parent.mkdir(parents=True, exist_ok=True) - with open(lock_path, "w") as lock_fd: - fcntl.flock(lock_fd, fcntl.LOCK_EX) - return _ensure_pto_isa_root_locked(clone_path, verbose=verbose, commit=commit, clone_protocol=clone_protocol) - - -def _ensure_pto_isa_root_locked( - clone_path: Path, - verbose: bool = False, - commit: Optional[str] = None, - clone_protocol: str = "ssh", -) -> Optional[str]: - """Inner logic for _ensure_pto_isa_root, called while holding the file lock.""" - - # Clone if needed - if not _is_pto_isa_cloned(): - if verbose: - logger.info("PTO_ISA_ROOT not set, cloning pto-isa repository...") - if not _clone_pto_isa(verbose=verbose, commit=commit, clone_protocol=clone_protocol): - # Another parallel process may have completed the clone - if not _is_pto_isa_cloned(): - if verbose: - logger.warning("Failed to automatically clone pto-isa.") - logger.warning("You can manually clone it with:") - logger.warning(f" mkdir -p {clone_path.parent}") - logger.warning(f" git clone {_pto_isa_repo_url(clone_protocol)} {clone_path}") - logger.warning("Or set PTO_ISA_ROOT to an existing pto-isa installation:") - logger.warning(" export PTO_ISA_ROOT=/path/to/pto-isa") - return None - if verbose: - logger.info("pto-isa already cloned by another process") - # Recovered from race — apply commit/update below - if commit: - _checkout_pto_isa_commit(clone_path, commit, verbose=verbose) - else: - _update_pto_isa_to_latest(clone_path, verbose=verbose) - elif commit: - _checkout_pto_isa_commit(clone_path, commit, verbose=verbose) - else: - _update_pto_isa_to_latest(clone_path, verbose=verbose) - - # Verify clone has expected content - include_dir = clone_path / "include" - if not include_dir.exists(): - if verbose: - logger.warning(f"pto-isa cloned but missing include directory: {include_dir}") - return None - - return str(clone_path.resolve()) - - def _kernel_config_runtime_env(kernel_config_module, kernels_dir: Path) -> dict[str, str]: """ Optional per-example environment variables for runtime compilation. @@ -485,16 +200,20 @@ def __init__( # noqa: PLR0913 repeat_rounds: Optional[int] = None, clone_protocol: str = "ssh", skip_golden: bool = False, + log_level: Optional[str] = None, ): - # Setup logging if not already configured (e.g., when used directly, not via run_example.py) - _setup_logging_if_needed() + # If caller passed log_level, apply it. Otherwise only set up a sane + # default when no logging has been configured yet (notebook / ad-hoc + # script path). CLI callers (run_example.py) already configured, so + # this becomes a no-op. + _maybe_configure_logging(log_level) self.kernels_dir = Path(kernels_dir).resolve() self.golden_path = Path(golden_path).resolve() self.platform = platform self.enable_profiling = enable_profiling self.skip_golden = skip_golden - self.project_root = _get_project_root() + self.project_root = PROJECT_ROOT # Resolve device ID self.device_id = device_id if device_id is not None else 0 @@ -711,16 +430,14 @@ def run(self) -> None: # noqa: PLR0912, PLR0915 from .kernel_compiler import KernelCompiler # noqa: PLC0415 from .runtime_builder import RuntimeBuilder # noqa: PLC0415 - # Auto-setup PTO_ISA_ROOT if needed (for all platforms, since kernels may use PTO ISA headers) - pto_isa_root = _ensure_pto_isa_root( - verbose=True, commit=self.pto_isa_commit, clone_protocol=self.clone_protocol + # Auto-setup PTO_ISA_ROOT if needed (for all platforms, since kernels may use PTO ISA headers). + # update_if_exists=True mirrors ci.py: when no commit is pinned, fetch origin/HEAD. + pto_isa_root = ensure_pto_isa_root( + commit=self.pto_isa_commit, + clone_protocol=self.clone_protocol, + update_if_exists=True, + verbose=True, ) - if pto_isa_root is None: - raise OSError( - "PTO_ISA_ROOT could not be resolved.\n" - "Please set it to the PTO-ISA root directory, e.g.:\n" - " export PTO_ISA_ROOT=$(pwd)/examples/scripts/_deps/pto-isa" - ) # Step 1: Build runtime, orchestration, and kernels in parallel # (they are independent — all only need kernel_compiler which is ready) @@ -958,6 +675,7 @@ def create_code_runner( # noqa: PLR0913 repeat_rounds=None, clone_protocol="ssh", skip_golden=False, + log_level=None, ): """Factory: creates a CodeRunner based on kernel_config.""" return CodeRunner( @@ -973,4 +691,5 @@ def create_code_runner( # noqa: PLR0913 repeat_rounds=repeat_rounds, clone_protocol=clone_protocol, skip_golden=skip_golden, + log_level=log_level, ) diff --git a/simpler_setup/log_config.py b/simpler_setup/log_config.py new file mode 100644 index 00000000..9ef6d24d --- /dev/null +++ b/simpler_setup/log_config.py @@ -0,0 +1,48 @@ +# Copyright (c) PyPTO Contributors. +# This program is free software, you can redistribute it and/or modify it under the terms and conditions of +# CANN Open Software License Agreement Version 2.0 (the "License"). +# Please refer to the License for details. You may not use this file except in compliance with the License. +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR IMPLIED, +# INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. +# See LICENSE in the root of the software repository for the full text of the License. +# ----------------------------------------------------------------------------------------------------------- +"""Shared CLI log-level helper. + +Used by ci.py, examples/scripts/run_example.py, and SceneTestCase.run_module +(python test_*.py) so every entry point wires `--log-level` the same way and +defaults to INFO. Also propagates via `PTO_LOG_LEVEL` env var so subprocesses +spawned by ci.py's runtime-isolation inherit the level. + +pytest is intentionally not touched — it has its own `--log-cli-level` and +pyproject `log_cli_level` knobs. +""" + +import logging +import os + +LOG_LEVEL_CHOICES = ["error", "warn", "info", "debug"] +DEFAULT_LOG_LEVEL = "info" + +_LEVEL_MAP = { + "error": logging.ERROR, + "warn": logging.WARNING, + "info": logging.INFO, + "debug": logging.DEBUG, +} + + +def configure_logging(log_level: str = DEFAULT_LOG_LEVEL) -> None: + """Configure root logger for a CLI entry point. + + Args: + log_level: one of "error" / "warn" / "info" / "debug" (case-insensitive). + Unknown values fall back to INFO. + """ + log_level = log_level.lower() + level = _LEVEL_MAP.get(log_level, logging.INFO) + logging.basicConfig( + level=level, + format="[%(levelname)s] %(message)s", + force=True, + ) + os.environ["PTO_LOG_LEVEL"] = log_level diff --git a/simpler_setup/pto_isa.py b/simpler_setup/pto_isa.py index ee15b9ef..1020ff90 100644 --- a/simpler_setup/pto_isa.py +++ b/simpler_setup/pto_isa.py @@ -6,37 +6,229 @@ # INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY, OR FITNESS FOR A PARTICULAR PURPOSE. # See LICENSE in the root of the software repository for the full text of the License. # ----------------------------------------------------------------------------------------------------------- -"""PTO-ISA root discovery for kernel compilation.""" +"""PTO-ISA dependency management: resolve or auto-clone the repo. +Single source of truth for locating / cloning / pinning the PTO-ISA repo. +Callers: scene_test (via ensure_pto_isa_root), ci.py (commit pin), code_runner +(wraps with CLI-friendly signature). + +Resolution order for ensure_pto_isa_root(): + 1. PTO_ISA_ROOT environment variable (if set and points to a directory) + 2. PROJECT_ROOT / build / pto-isa (auto-clone if missing) + +Lock file under build/ serializes concurrent clones from parallel processes. +""" + +import fcntl +import logging import os +import subprocess from pathlib import Path +from typing import Optional + +from .environment import PROJECT_ROOT + +logger = logging.getLogger(__name__) + +_PTO_ISA_HTTPS = "https://github.com/PTO-ISA/pto-isa.git" +_PTO_ISA_SSH = "git@github.com:PTO-ISA/pto-isa.git" + + +def get_pto_isa_clone_path() -> Path: + """Default auto-clone target for PTO-ISA, anchored to PROJECT_ROOT. + + Lives under PROJECT_ROOT/build/ so each repo / worktree / venv has its own + isolated clone (no races when multiple worktrees pin different commits). + """ + return PROJECT_ROOT / "build" / "pto-isa" + + +def _is_cloned(path: Path) -> bool: + """Return True if `path` looks like a valid PTO-ISA clone (has include/).""" + return (path / "include").is_dir() + + +def _is_git_available() -> bool: + try: + result = subprocess.run(["git", "--version"], check=False, capture_output=True, timeout=5) + return result.returncode == 0 + except (FileNotFoundError, subprocess.TimeoutExpired): + return False + + +def _repo_url(clone_protocol: str) -> str: + return _PTO_ISA_HTTPS if clone_protocol == "https" else _PTO_ISA_SSH + + +def _run_git( + args: list, cwd: Optional[Path] = None, timeout: int = 30, check: bool = False +) -> subprocess.CompletedProcess: + """Run a git subcommand. + Always captures stdout/stderr as text. `check=False` (default) returns the + CompletedProcess for manual returncode inspection; `check=True` raises + CalledProcessError on non-zero exit. + """ + return subprocess.run( + ["git"] + args, + check=check, + capture_output=True, + text=True, + cwd=str(cwd) if cwd else None, + timeout=timeout, + ) + + +def _clone(target: Path, commit: Optional[str], clone_protocol: str, verbose: bool) -> bool: + """Clone PTO-ISA to `target`, optionally at `commit`. Returns True on success.""" + if not _is_git_available(): + if verbose: + logger.warning("git command not available, cannot clone pto-isa") + return False + + try: + target.parent.mkdir(parents=True, exist_ok=True) + except OSError as e: + if verbose: + logger.warning(f"Failed to create clone parent dir: {e}") + return False + + repo_url = _repo_url(clone_protocol) + logger.info(f"Cloning pto-isa to {target} (first run, may take up to a minute)...") + + try: + result = _run_git(["clone", repo_url, str(target)], timeout=300) + if result.returncode != 0: + if verbose: + logger.warning(f"Failed to clone pto-isa:\n{result.stderr}") + return False + + if commit: + result = _run_git(["checkout", commit], cwd=target, timeout=30) + if result.returncode != 0: + if verbose: + logger.warning(f"Failed to checkout pto-isa commit {commit}:\n{result.stderr}") + return False + + if verbose: + suffix = f" at commit {commit}" if commit else "" + logger.info(f"pto-isa cloned successfully{suffix}: {target}") + return True + except subprocess.TimeoutExpired: + if verbose: + logger.warning("Clone operation timed out") + return False + except Exception as e: # noqa: BLE001 + if verbose: + logger.warning(f"Failed to clone pto-isa: {e}") + return False -def ensure_pto_isa_root() -> str: - """Return the PTO-ISA root directory. - Resolution order: - 1. PTO_ISA_ROOT environment variable - 2. Default clone location: examples/scripts/_deps/pto-isa +def checkout_pto_isa_commit(clone_path: Path, commit: str, verbose: bool = False) -> None: + """Switch an existing clone to `commit` if it isn't already there (idempotent).""" + try: + result = _run_git(["rev-parse", "--short", "HEAD"], cwd=clone_path, timeout=5) + current = result.stdout.strip() if result.returncode == 0 else "" + if current and not commit.startswith(current) and not current.startswith(commit): + if verbose: + logger.info(f"pto-isa at {current}, checking out {commit}...") + _run_git(["fetch", "origin"], cwd=clone_path, timeout=120, check=True) + _run_git(["checkout", commit], cwd=clone_path, timeout=30, check=True) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.warning(f"Failed to checkout pto-isa commit {commit}: {e.stderr if hasattr(e, 'stderr') else e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Unexpected error checking out pto-isa commit {commit}: {e}") - Returns: - Absolute path to PTO-ISA root. + +def _update_to_latest(clone_path: Path, verbose: bool) -> None: + """Fetch and reset existing clone to the remote default branch.""" + try: + if verbose: + logger.info("Updating pto-isa to latest...") + _run_git(["fetch", "origin"], cwd=clone_path, timeout=120, check=True) + _run_git(["reset", "--hard", "origin/HEAD"], cwd=clone_path, timeout=30, check=True) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + logger.warning(f"Failed to update pto-isa to latest: {e.stderr if hasattr(e, 'stderr') else e}") + except Exception as e: # noqa: BLE001 + logger.warning(f"Unexpected error updating pto-isa: {e}") + + +def ensure_pto_isa_root( + commit: Optional[str] = None, + clone_protocol: str = "ssh", + update_if_exists: bool = False, + verbose: bool = False, +) -> str: + """Resolve or auto-clone PTO-ISA. Return absolute path. + + Args: + commit: if provided, check out this revision after clone/in existing clone. + clone_protocol: "ssh" (default) or "https". + update_if_exists: when `commit` is None and a clone already exists, + fetch origin and reset to origin/HEAD. Used by ci.py to guarantee + reproducibility against latest main; scene_test leaves it False so + repeated pytest runs don't issue network requests. + verbose: log progress via `logger.info` / `logger.warning`. Raises: - OSError: If PTO-ISA root cannot be found. + OSError: when PTO_ISA_ROOT is unset and auto-clone fails. """ env_root = os.environ.get("PTO_ISA_ROOT") if env_root and Path(env_root).is_dir(): + if verbose: + logger.info(f"Using existing PTO_ISA_ROOT: {env_root}") return env_root - # Default location (relative to project root) - project_root = Path(__file__).parent.parent - default_path = project_root / "examples" / "scripts" / "_deps" / "pto-isa" - if default_path.is_dir(): - return str(default_path) + clone_path = get_pto_isa_clone_path() + lock_path = clone_path.parent / ".pto-isa.lock" + lock_path.parent.mkdir(parents=True, exist_ok=True) + with open(lock_path, "w") as lock_fd: + fcntl.flock(lock_fd, fcntl.LOCK_EX) + resolved = _ensure_locked( + clone_path, + commit=commit, + clone_protocol=clone_protocol, + update_if_exists=update_if_exists, + verbose=verbose, + ) - raise OSError( - "PTO_ISA_ROOT not found. Either:\n" - " export PTO_ISA_ROOT=/path/to/pto-isa\n" - "or ensure examples/scripts/_deps/pto-isa exists (run: python examples/scripts/run_example.py once)" - ) + if resolved is None: + raise OSError( + f"PTO-ISA not available.\n" + f" Either export PTO_ISA_ROOT=/path/to/pto-isa,\n" + f" or manually clone to {clone_path}:\n" + f" git clone {_repo_url(clone_protocol)} {clone_path}" + ) + return resolved + + +def _ensure_locked( + clone_path: Path, + commit: Optional[str], + clone_protocol: str, + update_if_exists: bool, + verbose: bool, +) -> Optional[str]: + """Inner logic executed while holding the file lock.""" + if not _is_cloned(clone_path): + if not _clone(clone_path, commit=commit, clone_protocol=clone_protocol, verbose=verbose): + # A parallel process may have won the race + if not _is_cloned(clone_path): + return None + if verbose: + logger.info("pto-isa already cloned by another process") + if commit: + checkout_pto_isa_commit(clone_path, commit, verbose=verbose) + elif update_if_exists: + _update_to_latest(clone_path, verbose=verbose) + elif commit: + checkout_pto_isa_commit(clone_path, commit, verbose=verbose) + elif update_if_exists: + _update_to_latest(clone_path, verbose=verbose) + + if not _is_cloned(clone_path): + if verbose: + logger.warning(f"pto-isa path exists but missing include directory: {clone_path / 'include'}") + return None + + return str(clone_path.resolve()) diff --git a/simpler_setup/scene_test.py b/simpler_setup/scene_test.py index 1ea89676..6c49e8db 100644 --- a/simpler_setup/scene_test.py +++ b/simpler_setup/scene_test.py @@ -27,6 +27,8 @@ from pathlib import Path from typing import Any, NamedTuple +from .log_config import DEFAULT_LOG_LEVEL, LOG_LEVEL_CHOICES, configure_logging + _compile_cache: dict[tuple[str, str, str], object] = {} @@ -633,12 +635,13 @@ def run_module(module_name): parser.add_argument("--enable-profiling", action="store_true", help="Enable profiling (first round only)") parser.add_argument("--build", action="store_true", help="Compile runtime from source") parser.add_argument( - "--log-level", choices=["error", "warn", "info", "debug"], help="Set PTO_LOG_LEVEL environment variable" + "--log-level", + choices=LOG_LEVEL_CHOICES, + default=DEFAULT_LOG_LEVEL, + help=f"Root logger level (default: {DEFAULT_LOG_LEVEL})", ) args = parser.parse_args() - - if args.log_level: - os.environ["PTO_LOG_LEVEL"] = args.log_level + configure_logging(args.log_level) module = sys.modules[module_name] test_classes = [