From c01ebe91c413e805f344beb812d8cd96bd9b3709 Mon Sep 17 00:00:00 2001 From: Yinan Xu Date: Tue, 12 May 2026 23:29:29 +0800 Subject: [PATCH] fix(spec2017): prepare x264 inputs on host Pass CROSS_COMPILE through the runcpu environment, build the x264 input generator on the host, and preprocess x264 bitstreams before runsetup. Populate x264 run directories with the generated inputs and synthesize x264 command lists when speccmds.cmd is absent. Map 625.x264_s refspeed to the 525.x264_r refrate control/data set. --- workloads/linux/spec2017/spec2017-package.py | 325 +++++++++++++++++-- 1 file changed, 300 insertions(+), 25 deletions(-) diff --git a/workloads/linux/spec2017/spec2017-package.py b/workloads/linux/spec2017/spec2017-package.py index 8d27c8a..10478f9 100755 --- a/workloads/linux/spec2017/spec2017-package.py +++ b/workloads/linux/spec2017/spec2017-package.py @@ -422,10 +422,11 @@ def export_build_log_artifact(build_log, out_dir): return exported_log -def runcpu_env(spec_dir, compiler_root, gnu_toolchain_root): +def runcpu_env(spec_dir, cross_compile, compiler_root, gnu_toolchain_root): env = os.environ.copy() env.update(spec_env(spec_dir)) env["SPEC"] = str(spec_dir) + env["CROSS_COMPILE"] = str(cross_compile) env["SPEC2017_COMPILER_ROOT"] = str(compiler_root) env["SPEC2017_GNU_TOOLCHAIN_ROOT"] = str(gnu_toolchain_root) return env @@ -463,7 +464,7 @@ def build_elf(spec_dir, bench_dir, spec_cfg, spec_source, out_dir, log_dir, cros shared_log_dir.mkdir(parents=True, exist_ok=True) build_local_config(spec_cfg, generated_cfg, output_root, jobs) status(f"Generated SPEC cfg: {generated_cfg}") - env = runcpu_env(spec_dir, compiler_root, gnu_toolchain_root) + env = runcpu_env(spec_dir, cross_compile, compiler_root, gnu_toolchain_root) cmd = [ str(spec_dir / "bin" / "runcpu"), "--action", @@ -508,9 +509,237 @@ def select_run_dir(output_root, bench_dir, tune, workload): return max(candidates, key=lambda path: path.stat().st_mtime) -def prepare_run_dir(spec_dir, bench_dir, workload, generated_output_root, spec_cfg, tune, log_dir, compiler_root, gnu_toolchain_root): +def bench_dir_for_run_dir(run_dir): + return run_dir.parent.parent.name + + +def is_x264_bench(bench_dir): + return bench_dir in {"525.x264_r", "625.x264_s"} + + +def x264_source_root(spec_dir): + return spec_dir / "benchspec" / "CPU" / "525.x264_r" + + +def x264_stage_root(spec_dir): + return spec_dir.parent / "_host-tools" / "x264-inputgen" + + +def x264_data_workload(workload): + return "refrate" if workload == "refspeed" else workload + + +def parse_qw_sources(object_pm, key): + text = object_pm.read_text(encoding="utf-8", errors="replace") + match = re.search(rf"'{re.escape(key)}'\s*=>\s*\[qw\((.*?)\)\]", text, re.S) + if match is None: + raise RuntimeError(f"could not locate source list for {key} in {object_pm}") + return tuple(token for token in match.group(1).split() if token) + + +def find_host_c_compiler(): + for candidate in ("gcc", "cc"): + found = shutil.which(candidate) + if found: + return Path(found).resolve() + raise RuntimeError("host gcc/cc is required to prepare x264 input data on the host") + + +def build_x264_host_inputgen(spec_dir, log_dir): + source_root = x264_source_root(spec_dir) + object_pm = source_root / "Spec" / "object.pm" + src_root = source_root / "src" + build_root = x264_stage_root(spec_dir) + obj_root = build_root / "obj" + exe_path = build_root / "ldecod-host" + meta_path = build_root / "build-meta.json" + build_log = log_dir / "x264-host-build.log" + compiler = find_host_c_compiler() + sources = parse_qw_sources(object_pm, "ldecod_r") + include_dirs = ( + src_root / "ldecod_src" / "inc", + src_root / "x264_src", + src_root / "x264_src" / "extras", + src_root / "x264_src" / "common", + ) + cflags = [ + "-std=c99", + "-O2", + "-fcommon", + "-fno-strict-aliasing", + "-Wno-implicit-int", + "-Wno-int-conversion", + "-Wno-incompatible-pointer-types", + "-Wno-implicit-function-declaration", + "-DSPEC", + "-DSPEC_LINUX", + "-DSPEC_LINUX_X64", + "-DSPEC_AUTO_SUPPRESS_OPENMP", + "-DSPEC_AUTO_BYTEORDER=0x12345678", + *(f"-I{path}" for path in include_dirs), + ] + metadata = { + "compiler": str(compiler), + "cflags": cflags, + "sources": {src: file_sha256(src_root / src) for src in sources}, + } + if exe_path.is_file() and meta_path.is_file(): + try: + if load_json(meta_path) == metadata: + status(f"Reusing host-side x264 input generator: {exe_path}") + return exe_path + except Exception: + pass + if build_root.exists(): + shutil.rmtree(build_root) + obj_root.mkdir(parents=True, exist_ok=True) + status(f"Building host-side x264 input generator: {exe_path}") + objects = [] + for src in sources: + src_path = src_root / src + obj_path = obj_root / Path(src).with_suffix(".o") + obj_path.parent.mkdir(parents=True, exist_ok=True) + run( + [str(compiler), *cflags, "-c", str(src_path), "-o", str(obj_path)], + log_path=build_log, + ) + objects.append(obj_path) + run( + [str(compiler), *(str(obj) for obj in objects), "-lm", "-o", str(exe_path)], + log_path=build_log, + summary=f"Linking host-side x264 input generator (log: {build_log})", + ) + write_json(meta_path, metadata) + return exe_path + + +def x264_input_controls(spec_dir, workload=None): + data_root = x264_source_root(spec_dir) / "data" + if not data_root.is_dir(): + raise RuntimeError(f"x264 data directory not found under {data_root}") + if workload is not None: + control = data_root / x264_data_workload(workload) / "input" / "control" + if not control.is_file(): + raise RuntimeError(f"x264 control file not found for workload {workload}: {control}") + return [control] + controls = sorted(data_root.glob("*/input/control")) + if not controls: + raise RuntimeError(f"no x264 control files found under {data_root}") + return controls + + +def parse_x264_control(control_path): + fields = control_path.read_text(encoding="utf-8", errors="replace").split() + if len(fields) < 9: + raise RuntimeError(f"unexpected x264 control format in {control_path}") + return fields[0], fields[1] + + +def x264_workload_for_run_dir(run_dir): + match = re.search(r"run_[^_]+_(test|train|refrate|refspeed)_", run_dir.name) + if match is None: + raise RuntimeError(f"cannot determine x264 workload from run directory name: {run_dir.name}") + return match.group(1) + + +def x264_staged_output_path(spec_dir, control_path): + _, output_name = parse_x264_control(control_path) + workload = control_path.parent.parent.name + return x264_stage_root(spec_dir) / "generated" / workload / output_name + + +def prepare_x264_input_data(spec_dir, log_dir, workload): + decoder = build_x264_host_inputgen(spec_dir, log_dir) + prep_log = log_dir / "x264-host-inputgen.log" + stage_root = x264_stage_root(spec_dir) / "work" + prepared = False + for control_path in x264_input_controls(spec_dir, workload): + input_name, output_name = parse_x264_control(control_path) + input_dir = control_path.parent + input_path = input_dir / input_name + staged_output_path = x264_staged_output_path(spec_dir, control_path) + staged_output_path.parent.mkdir(parents=True, exist_ok=True) + for extra_name in ("dataDec.txt", "log.dec", output_name): + extra_path = input_dir / extra_name + if extra_path.exists(): + extra_path.unlink() + if not input_path.is_file(): + raise RuntimeError(f"x264 input bitstream not found: {input_path}") + if staged_output_path.is_file() and staged_output_path.stat().st_size > 0 and staged_output_path.stat().st_mtime_ns >= input_path.stat().st_mtime_ns: + continue + stage_dir = stage_root / input_dir.parent.name + if stage_dir.exists(): + shutil.rmtree(stage_dir) + stage_dir.mkdir(parents=True, exist_ok=True) + run( + [str(decoder), "-i", str(input_path), "-o", str(staged_output_path)], + cwd=stage_dir, + env=os.environ.copy(), + log_path=prep_log, + summary=f"Preparing host-side x264 input data in {input_dir} (log: {prep_log})", + ) + prepared = True + if not prepared: + status(f"Reusing prepared host-side x264 input data under {x264_stage_root(spec_dir) / 'generated'}") + + +def populate_x264_run_dir(spec_dir, run_dir): + workload = x264_workload_for_run_dir(run_dir) + input_dir = x264_source_root(spec_dir) / "data" / x264_data_workload(workload) / "input" + control_path = input_dir / "control" + staged_output = x264_staged_output_path(spec_dir, control_path) + if not staged_output.is_file(): + raise RuntimeError(f"prepared x264 input data not found: {staged_output}") + run_dir.mkdir(parents=True, exist_ok=True) + for path in sorted(input_dir.iterdir()): + if not path.is_file(): + continue + shutil.copy2(path, run_dir / path.name) + shutil.copy2(staged_output, run_dir / staged_output.name) + exe_dir = run_dir.parent.parent / "exe" + if not exe_dir.is_dir(): + raise RuntimeError(f"x264 executable directory not found: {exe_dir}") + copied = False + for exe_path in sorted(exe_dir.iterdir()): + if not exe_path.is_file() or not exe_path.name.startswith("x264_"): + continue + shutil.copy2(exe_path, run_dir / exe_path.name) + copied = True + if not copied: + raise RuntimeError(f"x264 executable not found in {exe_dir}") + status(f"Populated x264 run directory {run_dir.name} with host-prepared inputs for {workload}") + +def render_specinvoke_commands(spec_dir, run_dir, cmd_file, log_path): + if not cmd_file.is_file(): + raise RuntimeError(f"{cmd_file.name} not found in run directory: {run_dir}") + env = os.environ.copy() + env.update(spec_env(spec_dir)) + output = run( + [str(spec_dir / "bin" / "specinvoke"), "-nn", str(cmd_file)], + cwd=run_dir, + env=env, + log_path=log_path, + summary=f"Rendering {cmd_file.name} to shell (log: {log_path})", + capture=True, + ) + commands = [] + for line in output.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or stripped.startswith("specinvoke exit:"): + continue + if stripped.startswith("export ") or stripped.startswith("unset ") or stripped.startswith("cd "): + continue + if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", stripped): + continue + commands.append(stripped) + return commands + + +def prepare_run_dir(spec_dir, bench_dir, workload, generated_output_root, spec_cfg, tune, log_dir, cross_compile, compiler_root, gnu_toolchain_root): setup_log = log_dir / "runsetup.log" - env = runcpu_env(spec_dir, compiler_root, gnu_toolchain_root) + env = runcpu_env(spec_dir, cross_compile, compiler_root, gnu_toolchain_root) + if is_x264_bench(bench_dir): + prepare_x264_input_data(spec_dir, log_dir, workload) cmd = [ str(spec_dir / "bin" / "runcpu"), "--action", @@ -529,7 +758,65 @@ def prepare_run_dir(spec_dir, bench_dir, workload, generated_output_root, spec_c bench_dir, ] run(cmd, cwd=spec_dir, env=env, log_path=setup_log, summary=f"Preparing run directory for {bench_dir}/{workload} (log: {setup_log})") - return select_run_dir(generated_output_root, bench_dir, tune, workload) + run_dir = select_run_dir(generated_output_root, bench_dir, tune, workload) + if is_x264_bench(bench_dir): + populate_x264_run_dir(spec_dir, run_dir) + if not (run_dir / "speccmds.cmd").is_file(): + status(f"x264 run directory prepared without speccmds.cmd; using synthesized x264 command list for {run_dir.name}") + return run_dir + + +def x264_control_path(spec_dir, run_dir): + candidates = [run_dir / "control"] + try: + workload = x264_workload_for_run_dir(run_dir) + candidates.append(x264_source_root(spec_dir) / "data" / x264_data_workload(workload) / "input" / "control") + except RuntimeError: + pass + candidates.append(x264_source_root(spec_dir) / "data" / "refrate" / "input" / "control") + for candidate in candidates: + if candidate.is_file(): + return candidate + raise RuntimeError(f"control file not found for x264 run directory: {run_dir}") + + +def x264_specinvoke_commands(spec_dir, run_dir): + control_path = x264_control_path(spec_dir, run_dir) + fields = control_path.read_text(encoding="utf-8", errors="replace").split() + if len(fields) < 9: + raise RuntimeError(f"unexpected x264 control format in {control_path}") + _, yuv_output, _, new_x264, size, seek_val, frames, frames2, yuvdump, *_ = fields + candidates = sorted(path for path in run_dir.iterdir() if path.is_file() and path.name.startswith("x264_")) + if not candidates: + raise RuntimeError(f"x264 executable not found in run directory: {run_dir}") + x264_exe = candidates[0].name + exe_ref = f"../{run_dir.name}/{x264_exe}" + + def format_command(args, stdout_name, stderr_name): + parts = [exe_ref, *[shlex.quote(arg) for arg in args], ">", stdout_name, "2>>", stderr_name] + return " ".join(parts) + + def padded_range_name(end_value, start_value="0", suffix=""): + width = len(str(int(end_value))) + return f"run_{int(start_value):0{width}d}-{end_value}_{x264_exe}_x264{suffix}" + + dumpyuv_args = ["--dumpyuv", yuvdump] if yuvdump != "" else [] + io_args = ["-o", new_x264, yuv_output, size] + commands = [] + if int(frames2) != 0: + filebase = padded_range_name(frames) + pass1_args = ["--pass", "1", "--stats", "x264_stats.log", "--bitrate", "1000", "--frames", frames, *io_args] + commands.append(format_command(pass1_args, f"{filebase}_pass1.out", f"{filebase}_pass1.err")) + pass2_args = ["--pass", "2", "--stats", "x264_stats.log", "--bitrate", "1000", *dumpyuv_args, "--frames", frames, *io_args] + commands.append(format_command(pass2_args, f"{filebase}_pass2.out", f"{filebase}_pass2.err")) + filebase = padded_range_name(frames2, start_value=seek_val) + extra_args = ["--seek", seek_val, *dumpyuv_args, "--frames", frames2, *io_args] + commands.append(format_command(extra_args, f"{filebase}.out", f"{filebase}.err")) + else: + filebase = padded_range_name(frames) + args = [*dumpyuv_args, "--frames", frames, *io_args] + commands.append(format_command(args, f"{filebase}.out", f"{filebase}.err")) + return commands def copy_tree_contents(src, dst): @@ -550,29 +837,16 @@ def copy_tree_contents(src, dst): def shell_from_specinvoke(spec_dir, run_dir, log_dir): cmd_file = run_dir / "speccmds.cmd" - if not cmd_file.is_file(): - raise RuntimeError(f"speccmds.cmd not found in run directory: {run_dir}") log_path = log_dir / "specinvoke-dryrun.log" - env = os.environ.copy() - env.update(spec_env(spec_dir)) - output = run( - [str(spec_dir / "bin" / "specinvoke"), "-nn", str(cmd_file)], - cwd=run_dir, - env=env, - log_path=log_path, - summary=f"Rendering speccmds.cmd to shell (log: {log_path})", - capture=True, - ) + if cmd_file.is_file(): + output = render_specinvoke_commands(spec_dir, run_dir, cmd_file, log_path) + elif is_x264_bench(bench_dir_for_run_dir(run_dir)): + output = x264_specinvoke_commands(spec_dir, run_dir) + else: + raise RuntimeError(f"speccmds.cmd not found in run directory: {run_dir}") commands = [] run_dir_ref = f"../{run_dir.name}/" - for line in output.splitlines(): - stripped = line.strip() - if not stripped or stripped.startswith("#") or stripped.startswith("specinvoke exit:"): - continue - if stripped.startswith("export ") or stripped.startswith("unset ") or stripped.startswith("cd "): - continue - if re.match(r"^[A-Za-z_][A-Za-z0-9_]*=", stripped): - continue + for line in output: line = line.replace(run_dir_ref, "./") line = line.replace(str(run_dir) + "/", "./") line = line.replace(str(run_dir), "/spec") @@ -752,6 +1026,7 @@ def package_case(args): shared_build_dir_for(out_dir, case["bench_dir"], args.tune) / "runcpu-config" / spec_cfg.name, args.tune, log_dir, + cross_compile, compiler_root, gnu_toolchain_root, )