diff --git a/auxiliary/cpu_freq_logger.sh b/auxiliary/cpu_freq_logger.sh index 7b438ea..edadb58 100755 --- a/auxiliary/cpu_freq_logger.sh +++ b/auxiliary/cpu_freq_logger.sh @@ -12,6 +12,20 @@ fi rm -f $OUTFILE +HAS_FREQ_FILE=0 +for cpu in /sys/devices/system/cpu/cpu*/cpufreq; do + if [[ -f "$cpu/cpuinfo_cur_freq" || -f "$cpu/scaling_cur_freq" ]]; then + HAS_FREQ_FILE=1 + break + fi +done + +if [[ $HAS_FREQ_FILE -eq 0 ]]; then + echo "UNSUPPORTED" > "$OUTFILE" + echo "CPU frequency logging unsupported on this platform" + exit 0 +fi + nohup bash -c " while true; do TS=\$(date +\"%Y-%m-%d_%H:%M:%S\") diff --git a/configs/bench_fio_compare.toml b/configs/bench_fio_compare.toml new file mode 100644 index 0000000..7016f26 --- /dev/null +++ b/configs/bench_fio_compare.toml @@ -0,0 +1,3 @@ +[fio_compare] +fio_size = "16GiB" +output_root = "cijoe-output/artifacts/fio-compare" diff --git a/configs/devices_16.toml b/configs/devices_16.toml index c84c86f..45b4963 100644 --- a/configs/devices_16.toml +++ b/configs/devices_16.toml @@ -2,7 +2,7 @@ fail_fast=true [xnvme.driver] -prefix = "PCI_BLACKLIST=0000:01:00.0" +prefix = "PCI_BLACKLIST=\"0000:01:00.0\"" [[devices]] pci_addr = "0000:4a:00.0" diff --git a/scripts/bench_helper.py b/scripts/bench_helper.py index eef1d0b..068381d 100644 --- a/scripts/bench_helper.py +++ b/scripts/bench_helper.py @@ -32,6 +32,7 @@ from bdevperf import bdevperf_cmd, create_config as bdevperf_config from dcgm_helper import DcgmHelper +from fio_xnvme import fio_xnvme_cmd, fio_xnvme_prefill_cmd from spdk_nvme_perf import spdk_nvme_perf_cmd from xnvmeperf import xnvmeperf_cmd, xnvmeperf_cuda_cmd @@ -45,6 +46,7 @@ def __init__( cfm: CpuFrequencyHelper, tool: str = "bdevperf", backend: str = "spdk", + fio_size: str = "16GiB", ): self.initialised = False @@ -55,7 +57,7 @@ def __init__( self.stress = False self.backend = backend self.tool = tool - + self.fio_size = fio_size self.dcgm = DcgmHelper(cijoe) if backend == "upcie-cuda" else None self.use_thrsib = False @@ -79,8 +81,11 @@ def __init__( self.bin = Path(spdk_path) / "build" / "bin" / "spdk_nvme_perf" elif tool in ["xnvmeperf", "xnvmeperf-cuda"]: self.bin = "xnvmeperf" + elif tool == "fio_xnvme": + self.bin = "fio" else: log.error(f"Failed: Unknown tool({tool})") + return self.initialised = True @@ -98,10 +103,9 @@ def use_thread_siblings(self, use_thrsib: bool) -> int: return err self.use_thrsib = use_thrsib - return 0 - def run_benchmark(self, depth: int, size: int, ndevs: int, ncpus: int, time: int, cpu_freq: float, suffix: str = "", nqueues: int = 1): + def run_benchmark(self, rw: str, depth: int, size: int, ndevs: int, ncpus: int, time: int, cpu_freq: float, suffix: str = "", nqueues: int = 1, prefill_only: int = 0): if not self.initialised: log.error("Failed: benchmarker not initialised correctly") return 1, None @@ -114,10 +118,18 @@ def run_benchmark(self, depth: int, size: int, ndevs: int, ncpus: int, time: int f"be_{self.backend}-tool_{self.tool}" f"{suffix}.out" ) + elif self.tool == "fio_xnvme": + filename = ( + f"d{ndevs}-c{ncpus}-o{size}-f_{self.fio_size}-" + f"rw_{rw}-q{depth}-be_{self.backend}-tool_{self.tool}-" + f"thrsib{1 if self.use_thrsib else 0}-" + f"freq_{cpu_freq}-" + f"stress{1 if self.stress else 0}" + f"{suffix}.out" + ) else: filename = ( - f"d{ndevs}-c{ncpus}-o{size}-q{depth}-" - f"be_{self.backend}-tool_{self.tool}-" + f"d{ndevs}-c{ncpus}-o{size}-q{depth}-be_{self.backend}-tool_{self.tool}-" f"thrsib{1 if self.use_thrsib else 0}-" f"freq_{cpu_freq}-" f"stress{1 if self.stress else 0}" @@ -131,17 +143,35 @@ def run_benchmark(self, depth: int, size: int, ndevs: int, ncpus: int, time: int return 0, result bench_args = { - "iopattern": "randread", + "iopattern": rw, "qdepth": depth, "iosize": size, "runtime": time, "devices": [d["pci_addr"] for d in self.devices[0:ndevs]], } + if self.tool == "fio_xnvme": + bench_args["backend"] = self.backend + bench_args["rw"] = rw + bench_args["fio_size"] = self.fio_size + if not is_cuda: bench_args["cpumask"] = self.cpu_masks[ncpus] - command = f"/usr/bin/time " + if self.tool == "fio_xnvme" and ndevs != 1: + log.error("Failed: fio_xnvme currently supports exactly 1 device per benchmark point") + return 1, None + + if is_cuda: + selected_cpus = [] + command = f"/usr/bin/time " + else: + selected_cpus = [v[0] for v in self.cpu_pairs if int(bench_args["cpumask"], 16) & (1 << v[0])] + command = f"/usr/bin/time " + + if self.tool == "fio_xnvme": + cpu_list = ",".join(str(cpu) for cpu in selected_cpus) + command = f"taskset -c {cpu_list} {command}" if self.tool == "bdevperf": config_local_path = self.configs_path / f"d{ndevs}.json" @@ -150,10 +180,8 @@ def run_benchmark(self, depth: int, size: int, ndevs: int, ncpus: int, time: int bench_args["config_path"] = self.remote_config command += bdevperf_cmd(self.bin, bench_args) - elif self.tool == "spdk_nvme_perf": command += spdk_nvme_perf_cmd(self.bin, bench_args) - elif self.tool == "xnvmeperf": bench_args["backend"] = self.backend command += xnvmeperf_cmd(self.bin, bench_args) @@ -163,15 +191,15 @@ def run_benchmark(self, depth: int, size: int, ndevs: int, ncpus: int, time: int bench_args["nqueues"] = nqueues command += xnvmeperf_cuda_cmd(self.bin, bench_args) + elif self.tool == "fio_xnvme": + if prefill_only: + command += fio_xnvme_prefill_cmd(self.bin, bench_args) + else: + command += fio_xnvme_cmd(self.bin, bench_args) else: log.error(f"Unknown tool: {self.tool}") return -1, None - if is_cuda: - selected_cpus = [] - else: - selected_cpus = [v[0] for v in self.cpu_pairs if int(bench_args["cpumask"], 16) & (1 << v[0])] - if self.stress and (stressed_cpus := [str(x) for x in range(len(self.cpu_pairs)) if x not in selected_cpus]): command = "\n".join([ f"taskset -c {','.join(stressed_cpus)} stress-ng --cpu {len(stressed_cpus)} --timeout {time + 5}s &", @@ -231,22 +259,29 @@ def abort_monitors(): log.error("Failed: DcgmHelper.stop_and_parse()") return err, None - cpu_freqs = [[cpu_freqs[idx], self.cpu_pairs[idx]] for idx in selected_cpus] + if self.cfm.cpu_control_supported and cpu_freqs: + cpu_freqs = [[cpu_freqs[idx], self.cpu_pairs[idx]] for idx in selected_cpus] + else: + cpu_freqs = [] result = { + "rw": rw, "qdepth": depth, "iosize": size, + "fio_size": self.fio_size, "ndevs": ndevs, "ncpus": ncpus, "nqueues": nqueues, + "device_bdf": bench_args["devices"][0] if bench_args["devices"] else None, "cpu_usage": cpu_usage, "cpu_freqs": cpu_freqs, "fixed_freq": self.cfm.fixed_freq, "cpu_governor": self.cfm.governor, - "thr_sib": 1 if self.use_thrsib else 0, + "cpu_control_supported": 1 if self.cfm.cpu_control_supported else 0, + "thr_sib": self.use_thrsib, "smt": 1 if "SMT1" in suffix else 0, "turbo": 1 if "turbo1" in suffix else 0, - "stress": 1 if self.stress else 0, + "stress": self.stress, "tool": self.tool, "backend": self.backend, "iops": bench_result["total"]["iops"], @@ -271,7 +306,6 @@ def _create_cpumasks(self, use_thrsib: bool): `(err, cpu_masks)` where a non-zero value for `err` describes that an error occured either while running `lscpu` or while parsing the output. """ - err, state = self.cijoe.run("lscpu -e") if err: log.error(f"Failed: lscpu -e") @@ -284,7 +318,6 @@ def _create_cpumasks(self, use_thrsib: bool): return 1, cpu_pairs = [[int(v) for v in match.groupdict().values()] for match in matches] - if use_thrsib: pairs = sorted(cpu_pairs, key=lambda p: p[1]) else: @@ -305,7 +338,6 @@ def _create_cpumasks(self, use_thrsib: bool): self.cpu_masks = cpu_masks self.cpu_pairs = cpu_pairs - return 0 def _parse_bench_results(self, table: str) -> Tuple[int, dict[str, list]]: @@ -316,7 +348,6 @@ def _parse_bench_results(self, table: str) -> Tuple[int, dict[str, list]]: Returns `(err, result)`, where a non-zero value for `err` describes that the output did not match the expected format. """ - result = { "devices": [] } table_regex = None @@ -326,6 +357,8 @@ def _parse_bench_results(self, table: str) -> Tuple[int, dict[str, list]]: table_regex = r"\s*(?P.+?)\s*?:\s+(?P[0-9.]+)\s+(?P[0-9.]+)\s+(?P[0-9.]+)\s+(?P[0-9.]+)\s+(?P[0-9.]+)" elif self.tool in ["xnvmeperf", "xnvmeperf-cuda"]: table_regex = r"\s*(?P\w+):?\s+(?P[0-9,]+)?\s+(?P[0-9.]+)\s+(?P[0-9.]+)\s+(?P[0-9.]+)" + elif self.tool == "fio_xnvme": + return self._parse_fio_results(table) else: log.error(f"Unkown tool: {self.tool}") return -1, None @@ -341,6 +374,47 @@ def _parse_bench_results(self, table: str) -> Tuple[int, dict[str, list]]: return 0, result + def _parse_fio_results(self, output: str) -> Tuple[int, dict[str, list]]: + start = output.find("{") + end = output.rfind("}") + if start < 0 or end < 0 or end <= start: + log.error("Failed: could not find fio JSON output") + return 1, None + + try: + payload = json.loads(output[start:end + 1]) + except json.JSONDecodeError: + log.error("Failed: invalid fio JSON output") + return 1, None + + jobs = payload.get("jobs", []) + if not jobs: + log.error("Failed: fio returned no jobs") + return 1, None + + total_iops = 0.0 + total_mibs = 0.0 + devices = [] + for job in jobs: + metrics = job.get("read", {}) + if not float(metrics.get("iops", 0.0)): + write_metrics = job.get("write", {}) + if float(write_metrics.get("iops", 0.0)): + metrics = write_metrics + iops = float(metrics.get("iops", 0.0)) + mibs = float(metrics.get("bw_bytes", 0.0)) / (1024 * 1024) + total_iops += iops + total_mibs += mibs + devices.append({"iops": iops, "mibs": mibs}) + + return 0, { + "devices": devices, + "total": { + "iops": total_iops, + "mibs": total_mibs, + }, + } + def _parse_time_output(self, output: str) -> Tuple[int, int]: """ Find the CPU usage from /usr/bin/time in from the output of /usr/bin/time. @@ -348,7 +422,6 @@ def _parse_time_output(self, output: str) -> Tuple[int, int]: Returns `(err, cpu_usage)`, where a non-zero value for `err` describes that the output did not match the expected format. """ - time_regex = r"(?P[0-9.]+)user (?P[0-9.]+)system (?P[0-9.:]+)elapsed (?P[0-9.]+)%CPU .*k" m = search(time_regex, output) diff --git a/scripts/bench_runall.py b/scripts/bench_runall.py index ac45f27..b23c433 100644 --- a/scripts/bench_runall.py +++ b/scripts/bench_runall.py @@ -35,11 +35,14 @@ def add_args(parser: ArgumentParser): parser.add_argument("--smt", type=int, default=[0,1], nargs="+", help="0 for SMT off, 1 for SMT on, [0,1] for testing both") parser.add_argument("--hyperthreads", type=int, default=[0,1], nargs="+", help="0 for hyper threads off, 1 for hyper threads on, [0,1] for testing both. Note that you cannot test with hyper threads if SMT is turned off") parser.add_argument("--stress", type=int, default=[0,1], nargs="+", help="0 for not stressing unused CPUs, 1 for stressing unused CPUs, [0,1] for testing both") - parser.add_argument("--time", type=int, default=10, help="Time for for bdevperf to run for each test") + parser.add_argument("--time", type=int, default=5, help="Time for each benchmark run") + parser.add_argument("--fio_size", type=str, default="16GiB", help="Working-set size for fio_xnvme") parser.add_argument("--results_dir", type=Path, default=None, help="Path to existing directory in which the results should be saved. Note: Already existing results will not be benchmarked again") parser.add_argument("--repetitions", type=int, default=5, help="The amount of times each benchmark will be repeated. The result will be average of the repetitions") parser.add_argument("--nqueues", type=int, default=[1], nargs="+", help="Number of queues per device (used by xnvmeperf-cuda)") - parser.add_argument("--tool", choices=["bdevperf", "xnvmeperf", "spdk_nvme_perf", "xnvmeperf-cuda"], default="xnvmeperf") + parser.add_argument("--rws", type=str, default=["randread"], nargs="+", help="List of I/O patterns to test") + parser.add_argument("--prefill_only", type=int, default=0, help="Run fio_xnvme in prefill-only mode") + parser.add_argument("--tool", choices=["bdevperf", "xnvmeperf", "spdk_nvme_perf", "xnvmeperf-cuda", "fio_xnvme"], default="xnvmeperf") parser.add_argument("--backend", type=str, default="upcie") @@ -75,7 +78,7 @@ def main(args, cijoe: Cijoe): log.error("Failed: transfer_cpu_frequency_logger()") return err - benchmarker = BenchHelper(cijoe, bdev_configs, bdev_results, cfm, args.tool, args.backend) + benchmarker = BenchHelper(cijoe, bdev_configs, bdev_results, cfm, args.tool, args.backend, args.fio_size) if not benchmarker.initialised: log.error("Failed: could not initialise BenchHelper") return 1 @@ -95,7 +98,7 @@ def main(args, cijoe: Cijoe): now = time() for i in range(args.repetitions): - err, result = benchmarker.run_benchmark(qd, iosz, devs, 0, args.time, "N/A", f"-{i}", nq) + err, result = benchmarker.run_benchmark("randread", qd, iosz, devs, 0, args.time, "N/A", f"-{i}", nq, args.prefill_only) if err: log.error("Failed: run_benchmark()") return err @@ -124,17 +127,17 @@ def main(args, cijoe: Cijoe): tests = [] if 0 in args.hyperthreads: - tests += product([0], args.turbo, args.smt, args.stress, args.cpu_freqs, test_devs, test_cpus, args.sizes, args.depths) + tests += product([0], args.turbo, args.smt, args.stress, args.cpu_freqs, test_devs, test_cpus, args.rws, args.sizes, args.depths) if 1 in args.hyperthreads: # shift range to match cpu hyperthreads test_cpus = [x for cpu in test_cpus for x in [cpu*2-1, cpu*2]] - tests += product([1], args.turbo, args.smt, args.stress, args.cpu_freqs, test_devs, test_cpus, args.sizes, args.depths) + tests += product([1], args.turbo, args.smt, args.stress, args.cpu_freqs, test_devs, test_cpus, args.rws, args.sizes, args.depths) - tests = [(ht,tu,sm,st,f,d,c,o,q) for (ht,tu,sm,st,f,d,c,o,q) in tests if not (not sm and ht)] + tests = [(ht,tu,sm,st,f,d,c,rw,o,q) for (ht,tu,sm,st,f,d,c,rw,o,q) in tests if not (not sm and ht)] finished, total, now = 0, len(tests), time() - for ht, tu, sm, st, freq, devs, cpus, iosz, qd in tests: + for ht, tu, sm, st, freq, devs, cpus, rw, iosz, qd in tests: err = cfm.toggle_smt(sm) if err: log.error(f"Failed: cfm.toggle_smt({sm})") @@ -160,7 +163,7 @@ def main(args, cijoe: Cijoe): now = time() for i in range(args.repetitions): - err, result = benchmarker.run_benchmark(qd, iosz, devs, cpus, args.time, freq, f"{suffix}-{i}") + err, result = benchmarker.run_benchmark(rw, qd, iosz, devs, cpus, args.time, freq, f"{suffix}-{i}", prefill_only=args.prefill_only) if err: log.error("Failed: run_benchmark()") return err diff --git a/scripts/bench_visualize.py b/scripts/bench_visualize.py index ee3f333..66a6302 100644 --- a/scripts/bench_visualize.py +++ b/scripts/bench_visualize.py @@ -18,6 +18,7 @@ def add_args(parser: ArgumentParser): parser.add_argument("--path", type=Path, default=None, help="Path to results.json") + parser.add_argument("--html_path", type=Path, default=None, help="Path to output HTML") parser.add_argument("--template", type=str, default="benchmark-io") @@ -26,7 +27,7 @@ def main(args, cijoe): artifacts = Path(args.output) / "artifacts" json_path = args.path if args.path else artifacts / "benchmark-results.json" - html_path = artifacts / "benchmark-results.html" + html_path = args.html_path if args.html_path else artifacts / "benchmark-results.html" if not json_path.exists(): log.error(f"Failed: could not find benchmark results on path({json_path})") @@ -36,6 +37,11 @@ def main(args, cijoe): results = json.load(file) datasets = convert_to_data(results) + cpu_control_warning = any( + not row.get("cpu_control_supported", True) + for dataset in datasets + for row in dataset["data"] + ) cuda_bandwidth = "" bandwidth_path = artifacts / "cuda-sample-p2p-bandwidth" @@ -55,8 +61,13 @@ def main(args, cijoe): template_env = jinja2.Environment(loader=template_loader) template = template_env.get_template(f"{args.template}.jinja2") + html_path.parent.mkdir(parents=True, exist_ok=True) with html_path.open("w") as body: - body.write(template.render({ "datasets": datasets, "cuda_bandwidth": cuda_bandwidth })) + body.write(template.render({ + "datasets": datasets, + "cuda_bandwidth": cuda_bandwidth, + "cpu_control_warning": cpu_control_warning, + })) return 0 diff --git a/scripts/cpu_freq_helper.py b/scripts/cpu_freq_helper.py index 424ac7c..8841c7e 100644 --- a/scripts/cpu_freq_helper.py +++ b/scripts/cpu_freq_helper.py @@ -15,6 +15,7 @@ def __init__(self, cijoe: Cijoe): self._steps = None self._smt = None self._turbo = None + self.cpu_control_supported = True def get_cpu_frequency_steps(self) -> Tuple[int, List[float]]: """ @@ -27,6 +28,7 @@ def get_cpu_frequency_steps(self) -> Tuple[int, List[float]]: err, state = self.cijoe.run('cpupower frequency-info | grep "available frequency steps"') if err or not state.output(): log.error("Failed: cpupower") + self.cpu_control_supported = False return 1, None line_regex = r"\s*available frequency steps:\s+(([\d.]+ GHz,? ?)+)" @@ -65,8 +67,11 @@ def set_cpu_freq(self, value: Union[float, str], cpus: List[int]) -> int: err, _ = self.cijoe.run(cmd) if err: - log.error("Failed: cpupower") - return 1 + log.warning("cpupower unavailable or unsupported; keeping current CPU frequency policy") + self.cpu_control_supported = False + self.fixed_freq = freq + self.governor = gvnr + return 0 self.fixed_freq = freq self.governor = gvnr @@ -83,8 +88,11 @@ def _clear_fixed_cpu_freq(self, governor: str = "ondemand") -> int: err, _ = self.cijoe.run(f"cpupower frequency-set -g {governor} --max 4.0GHz --min 0.8GHz") if err: - log.error("Failed: cpupower") - return 1 + log.warning("cpupower unavailable or unsupported; skipping fixed CPU frequency reset") + self.cpu_control_supported = False + self.fixed_freq = 0 + self.governor = governor + return 0 self.fixed_freq = 0 self.governor = governor @@ -105,16 +113,27 @@ def toggle_turbo(self, on: bool) -> int: return 0 no_turbo_path = "/sys/devices/system/cpu/intel_pstate/no_turbo" - cmd = f"echo {0 if on else 1} > {no_turbo_path}" + boost_path = "/sys/devices/system/cpu/cpufreq/boost" + cmd = None err, state = self.cijoe.run(f"ls {no_turbo_path}") if err or no_turbo_path != state.output().strip(): - boost_path = "/sys/devices/system/cpu/cpufreq/boost" + err, state = self.cijoe.run(f"ls {boost_path}") + if err or boost_path != state.output().strip(): + log.warning("Turbo control not supported on this platform; skipping turbo toggle") + self.cpu_control_supported = False + self._turbo = on + return 0 cmd = f"echo {1 if on else 0} > {boost_path}" + else: + cmd = f"echo {0 if on else 1} > {no_turbo_path}" err, _ = self.cijoe.run(cmd) if err: - return err + log.warning("Turbo toggle failed; continuing without enforcing turbo state") + self.cpu_control_supported = False + self._turbo = on + return 0 self._turbo = on @@ -168,8 +187,17 @@ def stop_logging_and_parse(self) -> Tuple[int, List]: log.error(f"Failed: cat {self._output}") return 1, None - lines = state.output().split("\n") - lo, hi = int(len(lines)*0.1), int(len(lines)*0.9) + lines = [line for line in state.output().split("\n") if line.strip()] + if not lines: + log.warning("CPU frequency logger produced no output; skipping cpu frequency collection") + self.cpu_control_supported = False + return 0, [] + if lines[0] == "UNSUPPORTED": + log.warning("CPU frequency logger unsupported on this platform; skipping cpu frequency collection") + self.cpu_control_supported = False + return 0, [] + + lo, hi = int(len(lines) * 0.1), int(len(lines) * 0.9) data = [[int(f) for f in line.split()[1:]] for line in lines[lo:hi]] avgs = [] diff --git a/scripts/fio_compare_collect.py b/scripts/fio_compare_collect.py new file mode 100644 index 0000000..e4c47e4 --- /dev/null +++ b/scripts/fio_compare_collect.py @@ -0,0 +1,140 @@ +""" +Collect FIO/xNVMe compare raw results into a single JSON document +=============================================================== + +Retargetable: False +------------------- +""" + +from argparse import ArgumentParser +from collections import defaultdict +from json import dump as json_dump, load as json_load +from pathlib import Path +from re import match +import logging as log + +def add_args(parser: ArgumentParser): + parser.add_argument( + "--results_dirs", + "--results-dirs", + nargs="+", + type=Path, + required=True, + help="Directories containing raw .out files for each workload", + ) + parser.add_argument( + "--output_path", + "--output-path", + type=Path, + required=True, + help="Path for the final merged benchmark-results.json", + ) + + +def main(args, cijoe): + output_path = args.output_path + output_path.parent.mkdir(parents=True, exist_ok=True) + + all_results = defaultdict(list) + include_all = ["cpu_freqs", "iops", "mibs", "cpu_usage"] + + for results_dir in args.results_dirs: + if not results_dir.exists(): + log.error(f"Failed: missing raw results dir({results_dir})") + return 1 + + err = collect_results(results_dir, all_results, include_all) + if err: + return err + + final_results = { + label: sorted(rows, key=sort_key) + for label, rows in all_results.items() + } + + if output_path.exists(): + output_path.unlink() + + with output_path.open("w") as handle: + json_dump(final_results, handle, indent=2) + + return 0 + + +def collect_results(results_dir: Path, all_results: dict, include_all: list[str]) -> int: + for path in sorted(results_dir.glob("*-0.out")): + label = parse_label(path.stem) + if label is None: + log.error(f"Failed parsing filename({path.stem})") + return 1 + + repeated_results = [] + for run in sorted(results_dir.glob(f"{path.stem[:-1]}*")): + with run.open("r") as handle: + repeated_results.append(json_load(handle)) + + err, result = merge_dicts(repeated_results, include_all) + if err: + log.error("Failed: merge_dicts()") + return err + + result["iops"] = avg_stddev(result["iops"]) + result["mibs"] = avg_stddev(result["mibs"]) + result["cpu_usage"] = avg_stddev(result["cpu_usage"]) + all_results[label].append(result) + + return 0 + + +def parse_label(stem: str) -> str | None: + regex = r".*-thrsib(?P[01])-freq_.*-stress(?P[01])-SMT(?P[01])-turbo(?P[01])-\d" + parsed = match(regex, stem) + if not parsed: + return None + + ht, st, sm, tu = map(int, parsed.groups()) + return ( + f"{'U' if ht else 'Not u'}sing thread siblings; " + f"SMT {'on' if sm else 'off'}; " + f"stress {'on' if st else 'off'}; " + f"turbo {'on' if tu else 'off'}" + ) + + +def avg_stddev(ns: list[int | float]): + avg = sum(ns) / len(ns) + stddev = (sum((x - avg) ** 2 for x in ns) / len(ns)) ** 0.5 + return avg, stddev + + +def merge_dicts(dicts: list[dict], include_all: list[str]) -> tuple[int, dict]: + first, merged = dicts[0], {} + keys = set(first.keys()) + + if not all(set(d.keys()) == keys for d in dicts): + failed = next(d for d in dicts if set(d.keys()) != keys) + log.error(f"Error: Expected keys of all dicts to be equal: {set(failed.keys())} != {keys}") + return 1, None + + for key in keys: + if key in include_all: + merged[key] = [d[key] for d in dicts] + else: + if not all(d[key] == first[key] for d in dicts): + log.error(f"Error: Expected all values for non-excluded key({key}) to be equal") + return 1, None + merged[key] = first[key] + + return 0, merged + + +def sort_key(row: dict) -> tuple: + return ( + row.get("device_bdf", ""), + row.get("rw", ""), + row.get("iosize", 0), + row.get("qdepth", 0), + row.get("backend", ""), + row.get("ncpus", 0), + row.get("ndevs", 0), + ) diff --git a/scripts/fio_xnvme.py b/scripts/fio_xnvme.py new file mode 100644 index 0000000..f561f27 --- /dev/null +++ b/scripts/fio_xnvme.py @@ -0,0 +1,72 @@ +import logging as log +from shlex import quote + + +REQUIRED_KEYS = ["devices", "iosize", "qdepth", "rw", "backend", "fio_size"] +PREFILL_REQUIRED_KEYS = ["devices", "backend", "fio_size"] +FIO_PREFILL_BS = 131072 +FIO_PREFILL_QD = 32 + + +def _escape_pci_addr(pci_addr: str) -> str: + return pci_addr.replace(":", r"\:") + + +def fio_xnvme_cmd(bin: str, args: dict) -> str: + if any(key not in args for key in REQUIRED_KEYS): + log.error(f"Failed: Missing arguments for {bin}") + return "" + + if len(args["devices"]) != 1: + log.error("Failed: fio_xnvme expects exactly one device") + return "" + + pci_addr = _escape_pci_addr(args["devices"][0]) + + parameters = [ + f"{bin}", + "--name=job0", + "--thread=1", + "--direct=1", + "--group_reporting=1", + "--ioengine=xnvme", + f"--xnvme_be={args['backend']}", + "--xnvme_dev_nsid=1", + f"--rw={args['rw']}", + f"--bs={args['iosize']}", + f"--iodepth={args['qdepth']}", + f"--io_size={args['fio_size']}", + f"--filename={quote(pci_addr)}", + "--output-format=json", + ] + return " ".join(parameters) + + +def fio_xnvme_prefill_cmd(bin: str, args: dict) -> str: + if any(key not in args for key in PREFILL_REQUIRED_KEYS): + log.error(f"Failed: Missing prefill arguments for {bin}") + return "" + + if len(args["devices"]) != 1: + log.error("Failed: fio_xnvme prefill expects exactly one device") + return "" + + pci_addr = _escape_pci_addr(args["devices"][0]) + + parameters = [ + f"{bin}", + "--name=prefill", + "--thread=1", + "--direct=1", + "--group_reporting=1", + "--ioengine=xnvme", + f"--xnvme_be={args['backend']}", + "--xnvme_dev_nsid=1", + "--rw=write", + f"--bs={FIO_PREFILL_BS}", + f"--iodepth={FIO_PREFILL_QD}", + f"--io_size={args['fio_size']}", + f"--filename={quote(pci_addr)}", + "--output-format=json", + ] + return " ".join(parameters) diff --git a/tasks/bench_fio_compare.yaml b/tasks/bench_fio_compare.yaml new file mode 100644 index 0000000..6c9bc9a --- /dev/null +++ b/tasks/bench_fio_compare.yaml @@ -0,0 +1,190 @@ +doc: | + Run FIO/xNVMe backend comparison benchmarks + =========================================== + + This workflow compares the xNVMe `spdk` and `upcie` backends using fio under + identical benchmark parameters. + + Configuration files + ------------------- + Run this workflow with: + + `configs/transport.toml` + `configs/bench_fio_compare.toml` + `configs/devices_*.toml` + + The compare config controls the fio working-set size and the compare artifact root. + The device config must provide exactly one benchmark target and use + `xnvme.driver.prefix` to blacklist any NVMe boot device. + + Output layout + ------------- + A single run generates: + + `/cijoe-output/artifacts/fio-compare//raw//...raw .out files` + `/cijoe-output/artifacts/fio-compare//benchmark-results.json` + `/cijoe-output/artifacts/fio-compare//benchmark-results.html` + + The final JSON and HTML are generated once, after all workloads complete. + +steps: +- name: validate_device_config + run: bash -lc 'test {{ config.devices | length }} -eq 1 || { echo "bench_fio_compare requires exactly one benchmark device"; exit 1; }' + +- name: allocate_hugepages + run: | + sysctl -w vm.nr_hugepages=1024 + mkdir -p /dev/hugepages + mountpoint -q /dev/hugepages || mount -t hugetlbfs nodev /dev/hugepages + +- name: prepare_spdk + run: | + {{ config.xnvme.driver.prefix }} xnvme-driver reset || true + modprobe uio_pci_generic + DRIVER_OVERRIDE=uio_pci_generic {{ config.xnvme.driver.prefix }} xnvme-driver + +- name: trim_before_read_spdk + run: bash -lc 'info=$(xnvme info --be spdk {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be spdk --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: prefill_readlike_spdk + uses: bench_runall + with: &prefill-params-spdk + depths: [1] + sizes: [4096] + rws: ["write"] + fio_size: "{{ config.fio_compare.fio_size }}" + numcpus_specific: [1] + numdevs_specific: [1] + cpu_freqs: ["performance"] + turbo: [1] + smt: [1] + hyperthreads: [0] + stress: [0] + repetitions: 1 + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/prefill-spdk" + tool: fio_xnvme + backend: spdk + prefill_only: 1 + +- name: run_read_spdk + uses: bench_runall + with: &bench-params-read-spdk + depths: [1, 4, 16, 32] + sizes: [4096, 16384, 65536, 131072] + rws: ["read"] + fio_size: "{{ config.fio_compare.fio_size }}" + numcpus_specific: [1] + numdevs_specific: [1] + cpu_freqs: ["performance"] + turbo: [1] + smt: [1] + hyperthreads: [0] + stress: [0] + repetitions: 3 + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/read" + tool: fio_xnvme + backend: spdk + +- name: run_randread_spdk + uses: bench_runall + with: + <<: *bench-params-read-spdk + rws: ["randread"] + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randread" + +- name: trim_before_write_spdk + run: bash -lc 'info=$(xnvme info --be spdk {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be spdk --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: run_write_spdk + uses: bench_runall + with: &bench-params-write-spdk + depths: [1, 4, 16, 32] + sizes: [4096, 16384, 65536, 131072] + rws: ["write"] + fio_size: "{{ config.fio_compare.fio_size }}" + numcpus_specific: [1] + numdevs_specific: [1] + cpu_freqs: ["performance"] + turbo: [1] + smt: [1] + hyperthreads: [0] + stress: [0] + repetitions: 3 + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/write" + tool: fio_xnvme + backend: spdk + +- name: trim_before_randwrite_spdk + run: bash -lc 'info=$(xnvme info --be spdk {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be spdk --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: run_randwrite_spdk + uses: bench_runall + with: + <<: *bench-params-write-spdk + rws: ["randwrite"] + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randwrite" + +- name: prepare_upcie + run: | + {{ config.xnvme.driver.prefix }} xnvme-driver reset || true + modprobe uio_pci_generic + DRIVER_OVERRIDE=uio_pci_generic {{ config.xnvme.driver.prefix }} xnvme-driver + +- name: trim_before_read_upcie + run: bash -lc 'info=$(xnvme info --be upcie {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be upcie --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: prefill_readlike_upcie + uses: bench_runall + with: + <<: *prefill-params-spdk + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/prefill-upcie" + backend: upcie + +- name: run_read_upcie + uses: bench_runall + with: &bench-params-read-upcie + <<: *bench-params-read-spdk + backend: upcie + +- name: run_randread_upcie + uses: bench_runall + with: + <<: *bench-params-read-upcie + rws: ["randread"] + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randread" + +- name: trim_before_write_upcie + run: bash -lc 'info=$(xnvme info --be upcie {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be upcie --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: run_write_upcie + uses: bench_runall + with: &bench-params-write-upcie + <<: *bench-params-write-spdk + backend: upcie + +- name: trim_before_randwrite_upcie + run: bash -lc 'info=$(xnvme info --be upcie {{ config.devices[0].pci_addr }}); nsid=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsid:/{print \$2; exit}"); nsect=$(printf "%s\n" "$info" | awk "/^[[:space:]]+nsect:/{print \$2; exit}"); test -n "$nsid" -a -n "$nsect" || { echo "failed to discover nsid/nsect"; exit 1; }; xnvme dsm {{ config.devices[0].pci_addr }} --be upcie --dev-nsid "$nsid" --nsid "$nsid" --ad --slba 0x0 --llb "$((nsect - 1))"' + +- name: run_randwrite_upcie + uses: bench_runall + with: + <<: *bench-params-write-upcie + rws: ["randwrite"] + results_dir: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randwrite" + +- name: collect + uses: fio_compare_collect + with: + results_dirs: + - "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/read" + - "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randread" + - "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/write" + - "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/raw/randwrite" + output_path: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/benchmark-results.json" + +- name: visualize + uses: bench_visualize + with: + path: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/benchmark-results.json" + html_path: "{{ local.env.PWD }}/{{ config.fio_compare.output_root }}/{{ config.devices[0].pci_addr }}/benchmark-results.html" + template: benchmark-fio-compare diff --git a/tasks/setup_udmabuf_import.yaml b/tasks/setup_udmabuf_import.yaml index 4f71ff9..e6161ec 100644 --- a/tasks/setup_udmabuf_import.yaml +++ b/tasks/setup_udmabuf_import.yaml @@ -53,6 +53,16 @@ steps: cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --set-str SYSTEM_TRUSTED_KEYS "" cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --set-str SYSTEM_REVOCATION_KEYS "" cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --disable MODULE_SIG + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --disable MODULE_SIG + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable X86_AMD_PSTATE + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable ACPI_CPPC_LIB + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable X86_ACPI_CPUFREQ + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ_GOV_PERFORMANCE + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ_GOV_POWERSAVE + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ_GOV_USERSPACE + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ_GOV_ONDEMAND + cd /usr/src/{{ config.linux.source.pkg }}; scripts/config --enable CPU_FREQ_GOV_SCHEDUTIL - name: prepare run: | diff --git a/templates/benchmark-fio-compare.jinja2 b/templates/benchmark-fio-compare.jinja2 new file mode 100644 index 0000000..dc39236 --- /dev/null +++ b/templates/benchmark-fio-compare.jinja2 @@ -0,0 +1,686 @@ + + + + + +FIO xNVMe Compare + + + +
+
+

FIO xNVMe Backend Compare

+

+ Compare `spdk` and `upcie` directly across queue depth and block size. + This report focuses on backend differences instead of the generic benchmark + framework axes. +

+
+ CPU control warning + + Turbo, governor, or fixed-frequency control was not fully enforced on this platform. + Treat the reported CPU policy fields as requested settings unless the row says + CPU control was supported. + +
+
+ +
+ + +
+
+
+
+

Queue depth comparison

+

+
+
+ SPDK + uPCIe +
+
+
+ +
+
+ +
+
+
+

Detailed Table

+

Delta is relative to the SPDK value for the same block size and queue depth.

+
+
+
+ + + + + + + + + + + + + +
Block SizeQDSPDK IOPSSPDK MiB/suPCIe IOPSuPCIe MiB/sDelta
+
+
+
+
+
+ + + + diff --git a/tests/test_fio_compare_artifacts.py b/tests/test_fio_compare_artifacts.py new file mode 100644 index 0000000..d582e15 --- /dev/null +++ b/tests/test_fio_compare_artifacts.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import json +import os +import shutil +import sys +from argparse import Namespace +from pathlib import Path +import tempfile +import unittest + + +REPO = Path(__file__).resolve().parents[1] +CIJOE_VENV_SITEPACKAGES = ( + Path(shutil.which("cijoe")).resolve().parent.parent + / "lib" + / f"python{sys.version_info.major}.{sys.version_info.minor}" + / "site-packages" +) +sys.path.insert(0, str(CIJOE_VENV_SITEPACKAGES)) +sys.path.insert(0, str(REPO / "scripts")) + +import bench_visualize # noqa: E402 +import fio_compare_collect # noqa: E402 + + +LABEL = "Not using thread siblings; SMT on; stress off; turbo on" + + +def _make_row(*, rw: str, backend: str, iosize: int, qdepth: int, rep: int) -> dict: + base = 1000 + iosize // 4096 * 100 + qdepth * 10 + (0 if backend == "spdk" else 500) + return { + "thr_sib": 0, + "ndevs": 1, + "cpu_usage": 50 + rep + (0 if backend == "spdk" else 2), + "device_bdf": "0000:02:00.0", + "cpu_governor": "performance", + "iosize": iosize, + "fixed_freq": 0, + "backend": backend, + "ncpus": 1, + "smt": 1, + "qdepth": qdepth, + "stress": 0, + "tool": "fio_xnvme", + "mibs": float(base / 10 + rep), + "turbo": 1, + "cpu_control_supported": 0, + "cpu_freqs": [[[5000000 + rep, 0]]], + "rw": rw, + "iops": float(base + rep), + "fio_size": "2GiB", + } + + +def _write_raw_workload_dir(root: Path, workload: str) -> Path: + results_dir = root / workload + results_dir.mkdir(parents=True, exist_ok=True) + + for iosize in (4096, 16384): + for qdepth in (1, 4): + for backend in ("spdk", "upcie"): + stem = ( + f"d1-c1-o{iosize}-f_2GiB-rw_{workload}-q{qdepth}" + f"-be_{backend}-tool_fio_xnvme-thrsib0-freq_performance-stress0-SMT1-turbo1" + ) + for rep in range(3): + payload = _make_row( + rw=workload, + backend=backend, + iosize=iosize, + qdepth=qdepth, + rep=rep, + ) + (results_dir / f"{stem}-{rep}.out").write_text(json.dumps(payload)) + + return results_dir + + +def _collect_merged_json(tmp_path: Path) -> Path: + raw_root = tmp_path / "artifacts" / "fio-compare" / "0000:02:00.0" / "raw" + read_dir = _write_raw_workload_dir(raw_root, "read") + randread_dir = _write_raw_workload_dir(raw_root, "randread") + merged = tmp_path / "artifacts" / "fio-compare" / "0000:02:00.0" / "benchmark-results.json" + + rc = fio_compare_collect.main( + Namespace(results_dirs=[read_dir, randread_dir], output_path=merged), + None, + ) + assert rc == 0 + return merged + + +# Artifact smoke tests for the compare benchmark. +# +# These tests build fake raw .out files, run the collector to merge them into a +# single benchmark-results.json, and then render the compare HTML from that +# merged JSON. This keeps the collector and visualizer testable without running +# the full benchmark workflow. +class TestFioCompareArtifacts(unittest.TestCase): + def test_fio_compare_collect_merges_fake_raw_results(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + merged = _collect_merged_json(tmp_path) + + with merged.open() as handle: + data = json.load(handle) + + self.assertEqual(list(data), [LABEL]) + rows = data[LABEL] + self.assertEqual(len(rows), 16) + self.assertEqual({row["rw"] for row in rows}, {"read", "randread"}) + self.assertEqual({row["backend"] for row in rows}, {"spdk", "upcie"}) + self.assertEqual({row["device_bdf"] for row in rows}, {"0000:02:00.0"}) + self.assertEqual({row["cpu_control_supported"] for row in rows}, {0}) + + def test_bench_visualize_renders_compare_html_from_merged_json(self): + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + merged = _collect_merged_json(tmp_path) + html_path = tmp_path / "artifacts" / "fio-compare" / "0000:02:00.0" / "benchmark-results.html" + + cwd = Path.cwd() + try: + os.chdir(REPO) + rc = bench_visualize.main( + Namespace( + output=tmp_path, + path=merged, + html_path=html_path, + template="benchmark-fio-compare", + ), + None, + ) + finally: + os.chdir(cwd) + + self.assertEqual(rc, 0) + self.assertTrue(html_path.exists()) + + html = html_path.read_text() + self.assertIn("FIO xNVMe Backend Compare", html) + self.assertIn("SPDK", html) + self.assertIn("uPCIe", html) + self.assertIn("CPU control warning", html) diff --git a/tests/test_fio_compare_validation.py b/tests/test_fio_compare_validation.py new file mode 100644 index 0000000..997b7a5 --- /dev/null +++ b/tests/test_fio_compare_validation.py @@ -0,0 +1,100 @@ +from pathlib import Path +import subprocess +import tempfile +import unittest + + +# Validation-only tests for the fio compare workflow. +# +# These cases do not run the full benchmark. They only verify that the +# compare workflow rejects multi-device configs and accepts a single-device +# config before any heavy benchmark setup starts. +class TestBenchFioCompareValidation(unittest.TestCase): + def test_bench_fio_compare_rejects_multi_device_configs(self): + repo = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + output = tmp_path / "cijoe-output" + workflow = tmp_path / "validate-only.yaml" + + workflow.write_text( + """ +doc: | + Validate device config only. +steps: +- name: validate_device_config + run: bash -lc 'test {{ config.devices | length }} -eq 1 || { echo "bench_fio_compare requires exactly one benchmark device"; exit 1; }' +""".lstrip() + ) + + result = subprocess.run( + [ + "cijoe", + "--monitor", + "--output", + str(output), + "-c", + str(repo / "configs" / "bench_fio_compare.toml"), + "-c", + str(repo / "configs" / "devices_16.toml"), + str(workflow), + ], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + + self.assertNotEqual(result.returncode, 0) + combined_output = result.stdout + result.stderr + self.assertIn("bench_fio_compare requires exactly one benchmark device", combined_output) + + def test_bench_fio_compare_accepts_single_device_configs(self): + repo = Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as tmpdir: + tmp_path = Path(tmpdir) + output = tmp_path / "cijoe-output" + config = tmp_path / "single-device.toml" + workflow = tmp_path / "validate-only.yaml" + + config.write_text( + """ +[fio_compare] +fio_size = "2GiB" +output_root = "cijoe-output/artifacts/fio-compare" + +[xnvme.driver] +prefix = "PCI_BLACKLIST=0000:0c:00.0" + +[[devices]] +pci_addr = "0000:02:00.0" +""".lstrip() + ) + + workflow.write_text( + """ +doc: | + Validate device config only. +steps: +- name: validate_device_config + run: bash -lc 'test {{ config.devices | length }} -eq 1 || { echo "bench_fio_compare requires exactly one benchmark device"; exit 1; }' +""".lstrip() + ) + + result = subprocess.run( + [ + "cijoe", + "--monitor", + "--output", + str(output), + "-c", + str(config), + str(workflow), + ], + cwd=repo, + capture_output=True, + text=True, + check=False, + ) + + self.assertEqual(result.returncode, 0)