diff --git a/bin/kt b/bin/kt index 68c5b57..f7e492f 100755 --- a/bin/kt +++ b/bin/kt @@ -5,6 +5,7 @@ import logging import click from kt.commands.checkout.command import checkout +from kt.commands.content_release.command import content_release from kt.commands.git_push.command import git_push from kt.commands.list_kernels.command import list_kernels from kt.commands.setup.command import setup @@ -30,6 +31,7 @@ def main(): cli.add_command(checkout) cli.add_command(git_push) cli.add_command(vm) + cli.add_command(content_release) cli() diff --git a/kernel_install_dep.sh b/kernel_install_dep.sh index 372cd81..2f8c337 100755 --- a/kernel_install_dep.sh +++ b/kernel_install_dep.sh @@ -166,6 +166,7 @@ install_kselftest_deps_10() { iptables \ iputils \ ipvsadm \ + jq \ kernel-devel \ kernel-selftests-internal \ kernel-tools \ @@ -179,6 +180,7 @@ install_kselftest_deps_10() { llvm \ ncurses-devel \ net-tools \ + netsniff-ng \ nftables \ nmap-ncat \ numactl-devel \ diff --git a/kt/KT.md b/kt/KT.md index 100456a..498e13d 100644 --- a/kt/KT.md +++ b/kt/KT.md @@ -295,3 +295,58 @@ and then ``` /kernel-src-tree-tools/kernel-kselftest.sh ``` + +### kt content-release + +Manages the complete content release workflow for kernel packages. This command +automates the process of preparing, building, and testing kernel releases. + +The command has three steps that can be run individually or all together: + +#### --prepare +Prepares the content release by: +- Validating git user.name and user.email are configured +- Running mkdistgitdiff.py to generate the staging branch and release files +- Checking out the staging branch {automation_tmp}_ +- Creating and displaying the new release tag + +#### --build +Builds kernel RPMs by: +- Verifying mock is installed and user is in mock group +- Checking DEPOT_USER and DEPOT_TOKEN environment variables are set +- Creating a temporary mock config with depot credentials +- Downloading sources using getsrc.sh +- Building SRPM with mock +- Building binary RPMs from the SRPM +- Listing all created RPMs + +Requirements: +- mock must be installed +- User must be in the mock group +- DEPOT_USER and DEPOT_TOKEN environment variables must be set + +#### --test +Tests the built kernel by: +- Spinning up a VM (creates if needed, boots if stopped) +- Installing the built kernel RPMs from build_files +- Rebooting the VM +- Running kselftests using /usr/libexec/kselftests/run_kselftest.sh +- Reporting number of tests passed + +Output logs: +- install.log: RPM installation output +- selftest-.log: Kselftest results + +#### Example: + +Run all steps: +``` +$ DEPOT_USER=user@example.com DEPOT_TOKEN=token kt content-release lts-9.2 +``` + +Run individual steps: +``` +$ kt content-release lts-9.2 --prepare +$ DEPOT_USER=user@example.com DEPOT_TOKEN=token kt content-release lts-9.2 --build +$ kt content-release lts-9.2 --test +``` diff --git a/kt/commands/content_release/command.py b/kt/commands/content_release/command.py new file mode 100644 index 0000000..217dabc --- /dev/null +++ b/kt/commands/content_release/command.py @@ -0,0 +1,75 @@ +import click + +from kt.commands.content_release.impl import run_prepare, run_build, run_test +from kt.ktlib.shell_completion import ShellCompletion + +epilog = """ +Manages the complete content release workflow for kernel packages. + +This command automates the kernel content release process through three steps: + +--prepare: Validates git config, runs mkdistgitdiff.py to create staging branch, + and creates a new tagged release in the src_worktree. + +--build: Builds both source and binary RPMs using mock. Downloads sources via + getsrc.sh and builds kernel packages in build_files directory. + Requires DEPOT_USER and DEPOT_TOKEN environment variables. + +--test: Spins up a VM, installs the built kernel RPMs, reboots, and runs + kselftests using /usr/libexec/kselftests/run_kselftest.sh. + +When run without options, executes all three steps sequentially. + +Examples: + +\b +$ DEPOT_USER=user@example.com DEPOT_TOKEN=token kt content-release lts-9.2 +\b +$ kt content-release lts-9.2 --prepare +\b +$ DEPOT_USER=user@example.com DEPOT_TOKEN=token kt content-release lts-9.2 --build +\b +$ kt content-release lts-9.2 --test +""" + + +@click.command(epilog=epilog) +@click.argument("kernel_workspace", required=True, shell_complete=ShellCompletion.show_kernel_workspaces) +@click.option( + "--prepare", + is_flag=True, + help="Run only the prepare step", +) +@click.option( + "--build", + is_flag=True, + help="Run only the build step", +) +@click.option( + "--test", + is_flag=True, + help="Run only the test step", +) +def content_release(kernel_workspace, prepare, build, test): + """Manage content release workflow (prepare, build, test).""" + + # Check if any specific step was requested + any_step_specified = prepare or build or test + + # Determine which steps to run + run_prepare_step = prepare or not any_step_specified + run_build_step = build or not any_step_specified + run_test_step = test or not any_step_specified + + if not any_step_specified: + click.echo(f"Running all content-release steps for {kernel_workspace}: prepare, build, test") + + # Run the selected steps + if run_prepare_step: + run_prepare(kernel_workspace=kernel_workspace) + + if run_build_step: + run_build(kernel_workspace=kernel_workspace) + + if run_test_step: + run_test(kernel_workspace=kernel_workspace) diff --git a/kt/commands/content_release/impl.py b/kt/commands/content_release/impl.py new file mode 100644 index 0000000..86b435a --- /dev/null +++ b/kt/commands/content_release/impl.py @@ -0,0 +1,561 @@ +import logging +import os +import re +import subprocess +import time + +from git import GitCommandError, Repo + +from kt.commands.vm.impl import setup_and_spinup_vm +from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.repo import check_git_config_value +from kt.ktlib.ssh import SshCommand +from kt.ktlib.util import Constants, run_command, run_command_with_output +from release_config import release_map + +# Source download configuration for kernels that don't work with getsrc.sh +SOURCE_DOWNLOAD_CONFIG = { + "lts-8.6": { + "base_url": "https://rocky-linux-sources-staging.a1.rockylinux.org", + "files": [ + ("cd67969ef0be82516b144066d3897b071f59f2a2", "kernel-abi-stablelists-4.18.0-372.tar.bz2"), + ("89ce72b86bacc9c2cd712784e9053d9c36f37c23", "kernel-kabi-dw-4.18.0-372.tar.bz2"), + ("c48b00ba5e77fcf4bc9e2dd5e58f1791ae71e8c8", "linux-4.18.0-372.32.1.el8_6.tar.xz"), + ], + }, + "fipslegacy-8.6": { + "base_url": "https://rocky-linux-sources-staging.a1.rockylinux.org", + "files": [ + ("feac61524ad00b8b03f2985f8ac330c7939ba425", "kernel-abi-stablelists-4.18.0-425.tar.bz2"), + ("f2fb49be6e6fe2782bc58e2914d8dcc7b2948764", "kernel-kabi-dw-4.18.0-425.tar.bz2"), + ("57cc7ba600df4d74be3a1b8c2324ea69b92699e4", "linux-4.18.0-425.13.1.el8_7.tar.xz"), + ], + }, + "cbr-7.9": { + "base_url": "https://git.centos.org/sources/kernel/c7", + "files": [ + ("ba5599148e52ecd126ebcf873672e26d3288323e", "kernel-abi-whitelists-1160.tar.bz2"), + ("5000b85c42ef87b6835dd8eef063e4623c2e0fa9", "kernel-kabi-dw-1160.tar.bz2"), + ("83cf85ab62fc9dca6d34175c60cc17cb917d7e0d", "linux-3.10.0-1160.119.1.el7.tar.xz"), + ], + }, +} + + +def run_prepare(kernel_workspace: str): + """ + Prepare step for content release. + + Ensures git user.name and user.email are configured in the kernel-dist-git repo + or globally. + + Args: + kernel_workspace: The name of the kernel workspace (e.g., 'lts-9.4') + """ + logging.info(f"Running prepare step for kernel workspace: {kernel_workspace}") + + # Load the kernel workspace to get the dist-git repo path + kernel_workspace_obj, config = KernelWorkspace.load_from_name(kernel_workspace) + + # Get the kernel-dist-git repo + dist_git_path = kernel_workspace_obj.dist_worktree.folder + repo = Repo(dist_git_path) + + logging.info(f"Checking git config in {dist_git_path}") + + # Check user.name and user.email + user_name = check_git_config_value(repo, "user", "name") + user_email = check_git_config_value(repo, "user", "email") + + missing = [] + if not user_name: + missing.append("user.name") + if not user_email: + missing.append("user.email") + + if missing: + raise RuntimeError( + f"Git config {' and '.join(missing)} not set in {dist_git_path} or globally. " + f"Please configure them using:\n" + f" git config --global user.name 'Your Name'\n" + f" git config --global user.email 'your.email@example.com'\n" + f"Or set them locally in the repository:\n" + f" cd {dist_git_path}\n" + f" git config user.name 'Your Name'\n" + f" git config user.email 'your.email@example.com'" + ) + + logging.info(f"Git config validated: user.name='{user_name}', user.email='{user_email}'") + + # Run mkdistgitdiff.py + mkdistgitdiff_script = config.base_path / "kernel-tools" / "mkdistgitdiff.py" + if not mkdistgitdiff_script.exists(): + raise RuntimeError(f"mkdistgitdiff.py not found at {mkdistgitdiff_script}") + + logging.info(f"Running mkdistgitdiff.py from {dist_git_path}") + + # Get just the branch name from remote_branch (strip "origin/" prefix) + src_branch_name = str(kernel_workspace_obj.src_worktree.remote_branch).split("/")[-1] + staging_branch = f"{{automation_tmp}}_{src_branch_name}" + + cmd = [ + str(mkdistgitdiff_script), + "--srcgit", + str(kernel_workspace_obj.src_worktree.folder.absolute()), + "--srcgit-branch", + kernel_workspace_obj.src_worktree.local_branch, + "--distgit", + ".", + "--distgit-branch", + kernel_workspace_obj.dist_worktree.local_branch, + "--distgit-staging-branch", + staging_branch, + "--last-tag", + "--bump", + ] + + output_file = kernel_workspace_obj.folder.absolute() / "mkdistgitdiff.log" + logging.info(f"Output will be written to {output_file}") + + run_command_with_output( + cmd, + output_file=output_file, + error_msg="mkdistgitdiff.py failed", + cwd=dist_git_path, + success_msg="mkdistgitdiff.py completed successfully", + ) + + # Checkout the new staging branch created by mkdistgitdiff + logging.info(f"Checking out staging branch: {staging_branch}") + + try: + repo.git.checkout(staging_branch) + logging.info(f"Successfully checked out {staging_branch}") + except GitCommandError as e: + logging.error(f"Failed to checkout {staging_branch}: {e}") + raise RuntimeError(f"Failed to checkout staging branch {staging_branch}") + + # Verify and display the newly created tag from mkdistgitdiff output + try: + with open(output_file, "r") as f: + log_content = f.read() + # Look for the "Content Release" line which contains the new tag + # Pattern matches version-like strings: kernel-X.Y.Z-A.B+C.D.elN_M_ciq + tag_pattern = r"Content Release\s+(kernel-[\d\.]+-[\d\.]+\+[\d\.]+\.el\d+_\d+_ciq)" + match = re.search(tag_pattern, log_content) + if match: + tag_name = match.group(1) + logging.info(f"New tag created in src_worktree: {tag_name}") + else: + # Fallback to old method if pattern doesn't match + for line in log_content.split("\n"): + if "Content Release" in line: + parts = line.split("Content Release") + if len(parts) > 1: + tag_name = parts[1].strip() + logging.info(f"New tag created in src_worktree: {tag_name}") + break + else: + logging.warning("Could not find new tag in mkdistgitdiff output") + except OSError as e: + logging.warning(f"Could not read tag from mkdistgitdiff output: {e}") + + logging.info("Prepare step completed") + + +def download_sources(kernel_workspace: str, dist_git_path, sources_dir): + """ + Download kernel sources using appropriate method. + + Args: + kernel_workspace: The name of the kernel workspace (e.g., 'lts-9.4') + dist_git_path: Path to the dist-git repository + sources_dir: Path to the SOURCES directory + """ + download_config = SOURCE_DOWNLOAD_CONFIG.get(kernel_workspace) + + if download_config: + # Direct download for special cases + logging.info(f"Downloading {kernel_workspace} source files directly...") + for hash_or_id, filename in download_config["files"]: + url = f"{download_config['base_url']}/{hash_or_id}" + run_command( + ["curl", url, "-o", str(sources_dir / filename)], + error_msg=f"Failed to download {filename}", + ) + logging.info(f"{kernel_workspace} source files downloaded successfully") + else: + # Use getsrc.sh for all other kernels + logging.info("Downloading getsrc.sh script...") + run_command( + ["curl", "-O", "https://raw.githubusercontent.com/rocky-linux/rocky-tools/main/getsrc/getsrc.sh"], + error_msg="Failed to download getsrc.sh", + cwd=dist_git_path, + success_msg="getsrc.sh downloaded successfully", + ) + + getsrc_script = dist_git_path / "getsrc.sh" + run_command( + ["chmod", "+x", str(getsrc_script)], + error_msg="Failed to make getsrc.sh executable", + success_msg="getsrc.sh made executable", + ) + + run_command( + [str(getsrc_script)], + error_msg="getsrc.sh failed", + cwd=dist_git_path, + success_msg="getsrc.sh completed successfully", + ) + + +def run_build(kernel_workspace: str): + """ + Build step for content release. + + Verifies mock is available and user is in mock group, then builds the kernel RPMs. + + Args: + kernel_workspace: The name of the kernel workspace (e.g., 'lts-9.4') + """ + logging.info(f"Running build step for kernel workspace: {kernel_workspace}") + + # Verify mock command is available + try: + result = subprocess.run( + ["which", "mock"], + capture_output=True, + text=True, + check=True, + ) + mock_path = result.stdout.strip() + logging.info(f"mock command found at: {mock_path}") + except subprocess.CalledProcessError: + raise RuntimeError("mock command not found. Please install mock:\n sudo dnf install mock") + + # Verify current user is in mock group + try: + result = subprocess.run( + ["groups"], + capture_output=True, + text=True, + check=True, + ) + groups = result.stdout.strip().split() + if "mock" not in groups: + raise RuntimeError( + "Current user is not in the mock group. Please add yourself to the mock group:\n" + " sudo usermod -a -G mock $USER\n" + "Then log out and log back in for the group change to take effect." + ) + logging.info("User is in mock group") + except subprocess.CalledProcessError as e: + raise RuntimeError(f"Failed to check user groups: {e}") + + # Verify DEPOT_USER and DEPOT_TOKEN environment variables are set + depot_user = os.environ.get("DEPOT_USER") + depot_token = os.environ.get("DEPOT_TOKEN") + + if not depot_user: + raise RuntimeError( + "DEPOT_USER environment variable is not set. Please set it:\n export DEPOT_USER=your_username" + ) + if not depot_token: + raise RuntimeError( + "DEPOT_TOKEN environment variable is not set. Please set it:\n export DEPOT_TOKEN=your_token" + ) + logging.info("DEPOT_USER and DEPOT_TOKEN environment variables are set") + + # Load kernel workspace + kernel_workspace_obj, config = KernelWorkspace.load_from_name(kernel_workspace) + + # Get the mock config name from the central release mapping + if kernel_workspace not in release_map: + raise RuntimeError( + f"Unknown kernel workspace: {kernel_workspace}\n" + f"Supported workspaces: {', '.join(sorted(release_map.keys()))}" + ) + + mock_config_base = release_map[kernel_workspace]["mock_config"] + + # CBR is a special case - no depot config, no credential replacement needed + is_cbr = kernel_workspace.startswith("cbr-") + + if is_cbr: + mock_config_name = f"{mock_config_base}-x86_64.cfg" + else: + mock_config_name = f"{mock_config_base}-depot-x86_64.cfg" + + # Find the mock config file + mock_configs_dir = config.base_path / "mock-configs" + mock_config_source = mock_configs_dir / mock_config_name + + if not mock_config_source.exists(): + raise RuntimeError( + f"Mock config not found: {mock_config_source}\nExpected to find {mock_config_name} in {mock_configs_dir}" + ) + + logging.info(f"Found mock config: {mock_config_source}") + + # For CBR, use the config directly. For others, create temp config with credentials + if is_cbr: + mock_config_to_use = mock_config_source + logging.info("Using CBR mock config directly (no depot credentials needed)") + else: + # Create a temporary mock config with DEPOT credentials replaced + temp_mock_config = kernel_workspace_obj.folder / f"temp_{mock_config_name}" + + logging.info(f"Creating temporary mock config: {temp_mock_config}") + + with open(mock_config_source, "r") as src: + config_content = src.read() + + # Replace DEPOT_USER and DEPOT_TOKEN + config_content = config_content.replace("DEPOT_USER", depot_user) + config_content = config_content.replace("DEPOT_TOKEN", depot_token) + + with open(temp_mock_config, "w") as dest: + dest.write(config_content) + + mock_config_to_use = temp_mock_config + logging.info("Temporary mock config created with credentials") + + # Create build_files directory + build_files_dir = kernel_workspace_obj.folder / "build_files" + build_files_dir.mkdir(exist_ok=True) + if not build_files_dir.exists(): + raise RuntimeError(f"Failed to create build_files directory: {build_files_dir}") + logging.info(f"Build files directory: {build_files_dir}") + + # Get dist_worktree path (where we'll run mock from) + dist_git_path = kernel_workspace_obj.dist_worktree.folder + logging.info(f"Running mock from: {dist_git_path}") + + # Download kernel sources + sources_dir = dist_git_path / "SOURCES" + download_sources(kernel_workspace, dist_git_path, sources_dir) + + # Build SRPM using mock + logging.info("Building SRPM with mock...") + + mock_srpm_log = kernel_workspace_obj.folder / "mock_buildsrpm.log" + logging.info(f"Mock SRPM build output will be written to {mock_srpm_log}") + + mock_cmd = [ + "mock", + "-v", + "-r", + str(mock_config_to_use), + f"--resultdir={build_files_dir}", + "--buildsrpm", + "--spec=SPECS/kernel.spec", + "--sources=SOURCES", + ] + + run_command_with_output( + mock_cmd, + output_file=mock_srpm_log, + error_msg="Mock SRPM build failed", + cwd=dist_git_path, + success_msg="SRPM build completed successfully", + ) + + # Find the SRPM that was just built + srpm_files = list(build_files_dir.glob("*.src.rpm")) + if not srpm_files: + raise RuntimeError(f"No SRPM found in {build_files_dir}") + srpm_file = srpm_files[0] + logging.info(f"Found SRPM: {srpm_file.name}") + + # Build binary RPMs from the SRPM + logging.info("Building binary RPMs with mock...") + + mock_build_log = kernel_workspace_obj.folder / "mock_build.log" + logging.info(f"Mock binary RPM build output will be written to {mock_build_log}") + + mock_build_cmd = [ + "mock", + "-v", + "-r", + str(mock_config_to_use), + f"--resultdir={build_files_dir}", + str(srpm_file), + ] + + run_command_with_output( + mock_build_cmd, + output_file=mock_build_log, + error_msg="Mock binary RPM build failed", + cwd=dist_git_path, + success_msg="Binary RPM build completed successfully", + ) + + # List all RPMs created + rpm_files = sorted(build_files_dir.glob("*.rpm")) + if rpm_files: + logging.info(f"Created {len(rpm_files)} RPM(s):") + for rpm in rpm_files: + logging.info(f" {rpm.name}") + else: + logging.warning("No RPM files found in build directory") + + # Clean up temporary mock config (only needed for non-CBR) + if not is_cbr: + try: + mock_config_to_use.unlink() + logging.info("Removed temporary mock config") + except OSError as e: + logging.warning(f"Failed to remove temporary mock config: {e}") + + logging.info("Build step completed") + + +def run_test(kernel_workspace: str): + """ + Test step for content release. + + Spins up VM, installs built RPMs, reboots, and runs kselftests. + + Args: + kernel_workspace: The name of the kernel workspace (e.g., 'lts-9.4') + """ + if not kernel_workspace: + logging.error("kernel_workspace is required for the test command") + raise ValueError("kernel_workspace argument is required for test command") + + logging.info(f"Running test step for kernel workspace: {kernel_workspace}") + + # Load kernel workspace + kernel_workspace_obj, config = KernelWorkspace.load_from_name(kernel_workspace) + + # Setup and spin up the VM (reuses common code from vm command) + vm_instance, config = setup_and_spinup_vm(kernel_workspace_name=kernel_workspace) + + # Wait for dependencies to be installed if VM was just created + logging.info("Waiting for VM dependencies to be installed...") + time.sleep(Constants.VM_DEPS_INSTALL_WAIT_SECONDS) + + # Install the built RPMs + build_files_dir = kernel_workspace_obj.folder / "build_files" + logging.info(f"Installing RPMs from {build_files_dir}...") + + # Special case for fipslegacy-8.6: enable depot to prevent issues with secure boot shim + if kernel_workspace == "fipslegacy-8.6": + logging.info("Enabling depot for fipslegacy-8.6...") + depot_user = os.environ.get("DEPOT_USER") + depot_token = os.environ.get("DEPOT_TOKEN") + + if not depot_user or not depot_token: + raise RuntimeError( + "DEPOT_USER and DEPOT_TOKEN environment variables must be set for fipslegacy-8.6 testing" + ) + + try: + # Install depot client + SshCommand.run( + domain=vm_instance.domain, + command=[ + 'sudo dnf install -y "https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm"' + ], + ) + logging.info("Depot client installed") + + # Register depot with credentials + SshCommand.run(domain=vm_instance.domain, command=[f"sudo depot register -u {depot_user} -t {depot_token}"]) + logging.info("Depot registered") + + # Enable fips-legacy-8 + SshCommand.run(domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"]) + logging.info("fips-legacy-8 enabled via depot") + except RuntimeError as e: + logging.error(f"Failed to enable depot for fipslegacy-8.6: {e}") + raise RuntimeError(f"Failed to enable depot for fipslegacy-8.6: {e}") + + # Build the list of RPMs to install (exclude src, rt, and debug RPMs) + all_rpms = list(build_files_dir.glob("*.rpm")) + install_rpms = [ + rpm + for rpm in all_rpms + if not rpm.name.endswith(".src.rpm") + and not rpm.name.startswith("kernel-rt") + and not rpm.name.startswith("kernel-debug-") + ] + + if not install_rpms: + raise RuntimeError(f"No installable RPMs found in {build_files_dir}") + + rpm_paths = " ".join(str(rpm.absolute()) for rpm in install_rpms) + install_cmd = f"sudo dnf install --allowerasing {rpm_paths} -y" + + install_log = kernel_workspace_obj.folder.absolute() / "install.log" + logging.info(f"Installing {len(install_rpms)} RPM(s)") + logging.info(f"RPM install output will be written to {install_log}") + + try: + SshCommand.run_with_output(output_file=install_log, domain=vm_instance.domain, command=[install_cmd]) + logging.info("RPMs installed successfully") + except RuntimeError as e: + logging.error(f"RPM installation failed: {e}") + logging.error(f"Check {install_log} for details") + raise RuntimeError(f"RPM installation failed. See {install_log} for details") + + # Determine expected kernel version from the built RPMs + kernel_rpms = list(build_files_dir.glob("kernel-[0-9]*.x86_64.rpm")) + expected_version = None + if kernel_rpms: + # Extract version from RPM filename: kernel-5.14.0-284.30.1+23.1.el9_2_ciq.x86_64.rpm + rpm_name = kernel_rpms[0].name + # Use regex to extract version (more robust than string replacement) + # Pattern: kernel-VERSION.ARCH.rpm where VERSION includes everything up to .x86_64 + version_match = re.match(r"kernel-(.*?)\.rpm$", rpm_name) + if version_match: + expected_version = version_match.group(1) + logging.info(f"Expected kernel version: {expected_version}") + else: + # Fallback to old string replacement method + expected_version = rpm_name.replace("kernel-", "").replace(".rpm", "") + logging.info(f"Expected kernel version (fallback): {expected_version}") + else: + logging.warning("Could not determine expected kernel version from RPMs") + + # Reboot the VM + logging.info("Rebooting VM...") + vm_instance.reboot() + + # Get the running kernel version from the VM + try: + kernel_version = SshCommand.run(domain=vm_instance.domain, command=["uname -r"]).strip() + logging.info(f"Running kernel version: {kernel_version}") + except RuntimeError as e: + logging.error(f"Failed to get kernel version from VM: {e}") + raise RuntimeError(f"Failed to get kernel version from VM: {e}") + + # Verify the kernel version matches what we installed + if expected_version and kernel_version != expected_version: + raise RuntimeError(f"Kernel version mismatch! Expected: {expected_version}, Running: {kernel_version}") + logging.info("Verified VM is running the newly installed kernel") + + # Run kselftests using the installed kselftests + logging.info("Running kernel selftests...") + kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log" + logging.info(f"Kselftest output will be written to {kselftest_log}") + + kselftest_cmd = "sudo /usr/libexec/kselftests/run_kselftest.sh" + + try: + SshCommand.run_with_output(output_file=kselftest_log, domain=vm_instance.domain, command=[kselftest_cmd]) + logging.info("Kselftests completed successfully") + except RuntimeError as e: + logging.error(f"Kselftests failed: {e}") + logging.error(f"Check {kselftest_log} for details") + raise RuntimeError(f"Kselftests failed. See {kselftest_log} for details") + + # Count passed tests + try: + with open(kselftest_log, "r") as f: + passed_tests = sum(1 for line in f if line.startswith("ok")) + logging.info(f"Kselftests passed: {passed_tests} tests") + except OSError as e: + logging.warning(f"Could not count passed tests: {e}") + + logging.info("Test step completed successfully") diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index e48038f..030ff1d 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -13,7 +13,7 @@ from kt.ktlib.config import Config from kt.ktlib.kernel_workspace import KernelWorkspace from kt.ktlib.ssh import SshCommand -from kt.ktlib.util import Constants +from kt.ktlib.util import Constants, run_command from kt.ktlib.virt import VirtHelper, VmCommand # TODO move this to a separate repo @@ -136,6 +136,9 @@ def _create_image(self, config: Config): # Copy qcow2 image to work dir self.qcow2_source_path.copy(self.qcow2_path) + # Resize the disk to 30GB + self._resize_disk() + self._virt_install(config=config) time.sleep(Constants.VM_STARTUP_WAIT_SECONDS) @@ -148,6 +151,14 @@ def _virt_install(self, config: Config): common_dir=config.base_path, ) + def _resize_disk(self): + """Resize the qcow2 disk image to 30GB.""" + logging.info(f"Resizing disk {self.qcow2_path} to 30G") + run_command( + ["qemu-img", "resize", str(self.qcow2_path), "30G"], + error_msg="Failed to resize disk image", + ) + def setup(self, config: Config): self._download_source_image() @@ -263,19 +274,36 @@ def console(self): VmCommand.console(vm_name=self.name) -def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool, test: bool = False): - if list_all: - VmCommand.list_all() - return +def load_vm_from_workspace(kernel_workspace_name: str): + """ + Load VM, config, and kernel workspace from a workspace name. + + Args: + kernel_workspace_name: The name of the kernel workspace + Returns: + Tuple of (vm, config) + """ config = Config.load() - kernel_workpath = config.kernels_dir / name + kernel_workpath = config.kernels_dir / kernel_workspace_name kernel_workspace = KernelWorkspace.load_from_filepath(folder=kernel_workpath) - vm = Vm.load(config=config, kernel_workspace=kernel_workspace) - if destroy: - vm.destroy() - return + + return vm, config + + +def setup_and_spinup_vm(kernel_workspace_name: str, override: bool = False): + """ + Common function to load config, setup and spin up a VM. + + Args: + kernel_workspace_name: The name of the kernel workspace + override: If True, destroy and recreate the VM + + Returns: + Tuple of (vm_instance, config) + """ + vm, config = load_vm_from_workspace(kernel_workspace_name) if override: vm.destroy() @@ -283,6 +311,28 @@ def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool vm.setup(config=config) vm_instance = vm.spin_up(config=config) + return vm_instance, config + + +def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool, test: bool = False): + if list_all: + VmCommand.list_all() + return + + if not name: + raise RuntimeError("kernel_workspace is required") + + # If neither test nor console is requested, we just spin up the VM and exit. + # This is useful for starting a VM that you'll interact with manually (e.g., via SSH). + # If this behavior is not desired, consider requiring at least one action flag. + + if destroy: + vm, config = load_vm_from_workspace(name) + vm.destroy() + return + + vm_instance, config = setup_and_spinup_vm(kernel_workspace_name=name, override=override) + if test: # Wait for the dependencies to be installed logging.info("Waiting for the deps to be installed") diff --git a/kt/ktlib/kernel_workspace.py b/kt/ktlib/kernel_workspace.py index 3257712..9b7ed43 100644 --- a/kt/ktlib/kernel_workspace.py +++ b/kt/ktlib/kernel_workspace.py @@ -125,6 +125,22 @@ def load_from_filepath(cls, folder: Path): src_worktree=src_worktree, ) + @classmethod + def load_from_name(cls, kernel_workspace_name: str): + """ + Load a kernel workspace by name. + + Args: + kernel_workspace_name: The name of the kernel workspace (e.g., 'lts-9.4') + + Returns: + Tuple of (KernelWorkspace, Config) + """ + config = Config.load() + kernel_workpath = config.kernels_dir / kernel_workspace_name + workspace = cls.load_from_filepath(folder=kernel_workpath) + return workspace, config + @classmethod def load(cls, name: str, config: Config, kernel_info: KernelInfo, extra: str): if extra: diff --git a/kt/ktlib/repo.py b/kt/ktlib/repo.py index 74e1c95..32fb4db 100644 --- a/kt/ktlib/repo.py +++ b/kt/ktlib/repo.py @@ -6,6 +6,31 @@ from pathlib3x import Path +def check_git_config_value(repo, section, option): + """ + Get a git config value from repo or global config. + + Args: + repo: Git repository object + section: Config section (e.g., 'user') + option: Config option (e.g., 'name') + + Returns: + The config value, or None if not found + """ + try: + # Try repo-specific config first + return repo.config_reader().get_value(section, option) + except Exception: + pass + + try: + # Fall back to global config + return repo.config_reader("global").get_value(section, option) + except Exception: + return None + + @dataclass class RepoInfo: """ diff --git a/kt/ktlib/util.py b/kt/ktlib/util.py index 3a15254..63d52dd 100644 --- a/kt/ktlib/util.py +++ b/kt/ktlib/util.py @@ -1,3 +1,53 @@ +import logging +import subprocess + + +def run_command(cmd: list, error_msg: str, cwd=None, success_msg: str = None): + """ + Run a subprocess command with error handling. + + Args: + cmd: Command and arguments as a list + error_msg: Error message to raise on failure + cwd: Working directory for the command + success_msg: Optional success message to log on success + """ + try: + subprocess.run(cmd, check=True, cwd=cwd) + if success_msg: + logging.info(success_msg) + except subprocess.CalledProcessError as e: + raise RuntimeError(f"{error_msg}: {e}") + + +def run_command_with_output(cmd: list, output_file, error_msg: str, cwd=None, success_msg: str = None): + """ + Run a subprocess command with output redirected to a file. + + Args: + cmd: Command and arguments as a list + output_file: Path to output file (stdout and stderr will be written here) + error_msg: Error message to raise on failure + cwd: Working directory for the command + success_msg: Optional success message to log on success + """ + try: + with open(output_file, "w") as f: + subprocess.run( + cmd, + cwd=cwd, + stdout=f, + stderr=subprocess.STDOUT, + check=True, + ) + if success_msg: + logging.info(success_msg) + except subprocess.CalledProcessError as e: + logging.error(f"{error_msg} with exit code {e.returncode}") + logging.error(f"Check {output_file} for details") + raise RuntimeError(f"{error_msg}. See {output_file} for details") + + class Constants: PRIVATE_REPOS_CONFIG_FILE = ".private_repos.yaml" SRC_TREE = "kernel-src-tree"