From c7cdff83ed3affa8b2f664e4d840cc0910fb738a Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Mon, 6 Jul 2026 17:03:21 +0100 Subject: [PATCH 1/7] Add hdf5 sweep reader and writer --- httomo/data/param_sweep_store.py | 154 ++++++++++++++++++++++++++++++- 1 file changed, 153 insertions(+), 1 deletion(-) diff --git a/httomo/data/param_sweep_store.py b/httomo/data/param_sweep_store.py index 8f0e2f458..741536c81 100644 --- a/httomo/data/param_sweep_store.py +++ b/httomo/data/param_sweep_store.py @@ -1,11 +1,16 @@ from __future__ import annotations +from pathlib import Path from typing import Literal, Optional, Tuple import numpy as np +import h5py from httomo.runner.auxiliary_data import AuxiliaryData from httomo.sweep_runner.param_sweep_block import ParamSweepBlock -from httomo.sweep_runner.param_sweep_store_interfaces import ParamSweepSource +from httomo.sweep_runner.param_sweep_store_interfaces import ( + ParamSweepSink, + ParamSweepSource, +) class ParamSweepReader(ParamSweepSource): @@ -57,9 +62,67 @@ def read_sweep_results(self) -> ParamSweepBlock: ) +class ParamSweepReaderHdf5(ParamSweepSource): + """ + Read parameter sweep results from hdf5 file produced by writer, extracting middle slices + along the `detector_y` dim + """ + + def __init__(self, source: ParamSweepWriterHdf5) -> None: + self._no_of_sweeps = source.no_of_sweeps + self._extract_dim: Literal[1] = 1 + self._single_shape = source.single_shape + self._total_shape = source.total_shape + assert ( + source._dataset is not None + ), "Reader should be created from writer with dataset not `None`" + self._dataset = source._dataset + self._aux_data = source.aux_data + self._h5file = source._h5file + self.filepath: Path = source.filepath + + @property + def no_of_sweeps(self) -> int: + return self._no_of_sweeps + + @property + def extract_dim(self) -> Literal[0, 1, 2]: + return self._extract_dim + + @property + def single_shape(self) -> Tuple[int, int, int]: + return self._single_shape + + @property + def total_shape(self) -> Tuple[int, int, int]: + return self._total_shape + + @property + def aux_data(self) -> AuxiliaryData: + return self._aux_data + + def read_sweep_results(self) -> ParamSweepBlock: + slices = [slice(None, None, 1)] * 3 + slices_per_sweep = self.single_shape[self.extract_dim] + first_middle_slice_index = slices_per_sweep // 2 + detector_y_slice = slice(first_middle_slice_index, None, slices_per_sweep) + slices[self.extract_dim] = detector_y_slice + return ParamSweepBlock( + data=self._dataset[slices[0], slices[1], slices[2]], + aux_data=self.aux_data, + slicing_dim=self.extract_dim, + ) + + def finalize(self) -> None: + self._h5file.close() + + class ParamSweepWriter: """Write parameter sweep results, concatenating them along the `detector_y` dim""" + # TODO: Add params for temppath and backing enum, like DataSetStoreWriter, and switch + # between RAM and hdf5 file within a single class, rather than using separate classes which + # have the majority of the same code def __init__(self, no_of_sweeps: int) -> None: self._concat_dim: Literal[1] = 1 self._no_of_sweeps = no_of_sweeps @@ -140,3 +203,92 @@ def write_sweep_result(self, block: ParamSweepBlock): ) self._data[slices[0], slices[1], slices[2]] = block.data self.increment_no_of_sweeps_written() + + +class ParamSweepWriterHdf5(ParamSweepSink): + """Write parameter sweep results, concatenating them along the `detector_y` dim""" + + def __init__(self, no_of_sweeps: int, filepath: Path) -> None: + self._concat_dim: Literal[1] = 1 + self._no_of_sweeps = no_of_sweeps + self._no_of_sweeps_written: int = 0 + self._single_shape: Optional[Tuple[int, int, int]] = None + self._total_shape: Optional[Tuple[int, int, int]] = None + self._dataset: Optional[h5py.Dataset] = None + self._slices_per_sweep: Optional[int] = None + self.filepath = filepath + self._h5file = h5py.File(filepath, "w") + + @property + def no_of_sweeps(self) -> int: + return self._no_of_sweeps + + @property + def concat_dim(self) -> Literal[0, 1, 2]: + return self._concat_dim + + @property + def single_shape(self) -> Tuple[int, int, int]: + if self._single_shape is None: + err_str = "Shape of single sweep result isn't known until the first write has occurred" + raise ValueError(err_str) + return self._single_shape + + @property + def total_shape(self) -> Tuple[int, int, int]: + if self._total_shape is None: + err_str = ( + "Shape of full array holding sweep results isn't known until the first " + "write has occurred" + ) + raise ValueError(err_str) + return self._total_shape + + def make_reader(self) -> ParamSweepReaderHdf5: + if self._dataset is None: + raise ValueError("Cannot make reader when no data has been written yet") + return ParamSweepReaderHdf5(self) + + @property + def no_of_sweeps_written(self) -> int: + return self._no_of_sweeps_written + + def increment_no_of_sweeps_written(self): + self._no_of_sweeps_written += 1 + + @property + def slices_per_sweep(self) -> int: + if self._slices_per_sweep is None: + raise ValueError( + "Slices per sweep isn't known until the first write has occurred" + ) + return self._slices_per_sweep + + @property + def aux_data(self) -> AuxiliaryData: + return self._aux_data + + def write_sweep_result(self, block: ParamSweepBlock): + block.to_cpu() + if self._dataset is None: + self._single_shape = block.shape + self._total_shape = ( + self.single_shape[0], + self.single_shape[1] * self.no_of_sweeps, + self.single_shape[2], + ) + self._slices_per_sweep = self.single_shape[self.concat_dim] + self._dataset = self._h5file.create_dataset( + "/data", shape=self.total_shape, dtype=block.data.dtype + ) + self._aux_data = block.aux_data + + slices = [slice(None, None, 1)] * 3 + sweep_res_start = self.no_of_sweeps_written * self.slices_per_sweep + slices[self._concat_dim] = slice( + sweep_res_start, + sweep_res_start + self.slices_per_sweep, + 1, + ) + self._dataset[slices[0], slices[1], slices[2]] = block.data + self.increment_no_of_sweeps_written() From 8743c59a35842bba1c48b496d9493928d7db9be6 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Mon, 6 Jul 2026 17:04:56 +0100 Subject: [PATCH 2/7] Add proof-of-concept sweep runner backed by hdf5 file --- httomo/sweep_runner/param_sweep_runner.py | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/httomo/sweep_runner/param_sweep_runner.py b/httomo/sweep_runner/param_sweep_runner.py index df24b6bf4..0350681d7 100644 --- a/httomo/sweep_runner/param_sweep_runner.py +++ b/httomo/sweep_runner/param_sweep_runner.py @@ -7,7 +7,12 @@ import numpy as np import httomo -from httomo.data.param_sweep_store import ParamSweepReader, ParamSweepWriter +from httomo import globals +from httomo.data.param_sweep_store import ( + ParamSweepReaderHdf5, + ParamSweepWriter, + ParamSweepWriterHdf5, +) from httomo.method_wrappers.images import ImagesWrapper from httomo.method_wrappers.save_intermediate import SaveIntermediateFilesWrapper from httomo.runner.block_split import BlockSplitter @@ -42,6 +47,7 @@ def __init__( self._start_sweep_idx : self._stop_sweep_idx ] self._set_image_saver_params() + self._comm = comm @property def block(self) -> ParamSweepBlock: @@ -212,8 +218,16 @@ def execute_after_sweep(self): def execute_sweep(self): """Execute all param variations of the same method in the sweep""" - writer = ParamSweepWriter(len(self._sweep_values)) method = self._stages.sweep.method + if method.method_name == "paganin_filter": + sweep_output_filepath = ( + globals.run_out_dir / f"sweep-output-rank-{self._comm.rank}.h5" + ) + writer = ParamSweepWriterHdf5( + len(self._sweep_values), sweep_output_filepath + ) + else: + writer = ParamSweepWriter(len(self._sweep_values)) log_once(f"Running {method.method_name} ({method.package_name})") log_once(" Beginning parameter sweep") @@ -254,8 +268,10 @@ def execute_sweep(self): log_once(" Finished parameter sweep") - reader = ParamSweepReader(writer) + reader = writer.make_reader() self._block = reader.read_sweep_results() + if isinstance(reader, ParamSweepReaderHdf5): + reader.finalize() def execute(self): """Load input data and execute all stages (before sweep, sweep, after sweep)""" From ea3d45497b0cd278ffd29912588dd9d30ad47baf Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 8 Jul 2026 11:51:41 +0100 Subject: [PATCH 3/7] Switch between hdf5 and RAM within sweep writer and reader There's enough common code across the hdf5 and RAM versions of the sweep writer and reader that having a single class for each of the reader and writer makes more sense than duplicating code across an hdf5 and RAM version of each of the writer and reader. --- httomo/data/param_sweep_store.py | 204 +++++++--------------- httomo/sweep_runner/param_sweep_runner.py | 26 +-- tests/data/test_param_sweep_store.py | 48 +++-- 3 files changed, 104 insertions(+), 174 deletions(-) diff --git a/httomo/data/param_sweep_store.py b/httomo/data/param_sweep_store.py index 741536c81..9345139df 100644 --- a/httomo/data/param_sweep_store.py +++ b/httomo/data/param_sweep_store.py @@ -1,16 +1,16 @@ from __future__ import annotations +from os import PathLike from pathlib import Path from typing import Literal, Optional, Tuple +from mpi4py import MPI import numpy as np import h5py +from httomo.runner.dataset_store_backing import DataSetStoreBacking from httomo.runner.auxiliary_data import AuxiliaryData from httomo.sweep_runner.param_sweep_block import ParamSweepBlock -from httomo.sweep_runner.param_sweep_store_interfaces import ( - ParamSweepSink, - ParamSweepSource, -) +from httomo.sweep_runner.param_sweep_store_interfaces import ParamSweepSource class ParamSweepReader(ParamSweepSource): @@ -28,6 +28,15 @@ def __init__(self, source: ParamSweepWriter) -> None: ), "Reader should be created from writer with data not `None`" self._data = source._data self._aux_data = source.aux_data + self._h5filename: Optional[Path] = None + self.h5file: Optional[h5py.File] = None + + if source.is_file_based: + self._h5filename = source.filename + self.h5file = h5py.File(source.filename, "r") + self._data = self.h5file["/data"] + else: + self._data = source._data @property def no_of_sweeps(self) -> int: @@ -61,76 +70,49 @@ def read_sweep_results(self) -> ParamSweepBlock: slicing_dim=self.extract_dim, ) - -class ParamSweepReaderHdf5(ParamSweepSource): - """ - Read parameter sweep results from hdf5 file produced by writer, extracting middle slices - along the `detector_y` dim - """ - - def __init__(self, source: ParamSweepWriterHdf5) -> None: - self._no_of_sweeps = source.no_of_sweeps - self._extract_dim: Literal[1] = 1 - self._single_shape = source.single_shape - self._total_shape = source.total_shape - assert ( - source._dataset is not None - ), "Reader should be created from writer with dataset not `None`" - self._dataset = source._dataset - self._aux_data = source.aux_data - self._h5file = source._h5file - self.filepath: Path = source.filepath + def finalize(self): + self._data = None + if self.h5file is not None: + self.h5file.close() + self.h5file = None @property - def no_of_sweeps(self) -> int: - return self._no_of_sweeps + def is_file_based(self) -> bool: + return self._h5filename is not None @property - def extract_dim(self) -> Literal[0, 1, 2]: - return self._extract_dim - - @property - def single_shape(self) -> Tuple[int, int, int]: - return self._single_shape - - @property - def total_shape(self) -> Tuple[int, int, int]: - return self._total_shape - - @property - def aux_data(self) -> AuxiliaryData: - return self._aux_data - - def read_sweep_results(self) -> ParamSweepBlock: - slices = [slice(None, None, 1)] * 3 - slices_per_sweep = self.single_shape[self.extract_dim] - first_middle_slice_index = slices_per_sweep // 2 - detector_y_slice = slice(first_middle_slice_index, None, slices_per_sweep) - slices[self.extract_dim] = detector_y_slice - return ParamSweepBlock( - data=self._dataset[slices[0], slices[1], slices[2]], - aux_data=self.aux_data, - slicing_dim=self.extract_dim, - ) - - def finalize(self) -> None: - self._h5file.close() + def filename(self) -> Optional[Path]: + return self._h5filename class ParamSweepWriter: """Write parameter sweep results, concatenating them along the `detector_y` dim""" - # TODO: Add params for temppath and backing enum, like DataSetStoreWriter, and switch - # between RAM and hdf5 file within a single class, rather than using separate classes which - # have the majority of the same code - def __init__(self, no_of_sweeps: int) -> None: + def __init__( + self, + no_of_sweeps: int, + comm: MPI.Comm, + temppath: PathLike, + backing: DataSetStoreBacking, + ) -> None: self._concat_dim: Literal[1] = 1 self._no_of_sweeps = no_of_sweeps self._no_of_sweeps_written: int = 0 self._single_shape: Optional[Tuple[int, int, int]] = None self._total_shape: Optional[Tuple[int, int, int]] = None - self._data: Optional[np.ndarray] = None + self._data: Optional[np.ndarray | h5py.Dataset] = None self._slices_per_sweep: Optional[int] = None + self._comm = comm + self._backing = backing + self._temppath = temppath + self._h5filename: Optional[Path] = None + self.h5file: Optional[h5py.File] = None + + if backing == DataSetStoreBacking.File: + self._h5filename = ( + Path(self._temppath) / f"sweep-output-rank-{self._comm.rank}.h5" + ) + self.h5file = h5py.File(self._h5filename, "w") @property def no_of_sweeps(self) -> int: @@ -191,8 +173,14 @@ def write_sweep_result(self, block: ParamSweepBlock): self.single_shape[2], ) self._slices_per_sweep = self.single_shape[self.concat_dim] - self._data = np.empty(shape=self.total_shape, dtype=block.data.dtype) self._aux_data = block.aux_data + if self._backing == DataSetStoreBacking.RAM: + self._data = np.empty(shape=self.total_shape, dtype=block.data.dtype) + elif self._backing == DataSetStoreBacking.File: + assert self.h5file is not None + self._data = self.h5file.create_dataset( + "/data", shape=self.total_shape, dtype=block.data.dtype + ) slices = [slice(None, None, 1)] * 3 sweep_res_start = self.no_of_sweeps_written * self.slices_per_sweep @@ -201,94 +189,22 @@ def write_sweep_result(self, block: ParamSweepBlock): sweep_res_start + self.slices_per_sweep, 1, ) + assert ( + self._data is not None + ), "Numpy array or hdf5 file should be created prior to writing sweep result" self._data[slices[0], slices[1], slices[2]] = block.data self.increment_no_of_sweeps_written() - -class ParamSweepWriterHdf5(ParamSweepSink): - """Write parameter sweep results, concatenating them along the `detector_y` dim""" - - def __init__(self, no_of_sweeps: int, filepath: Path) -> None: - self._concat_dim: Literal[1] = 1 - self._no_of_sweeps = no_of_sweeps - self._no_of_sweeps_written: int = 0 - self._single_shape: Optional[Tuple[int, int, int]] = None - self._total_shape: Optional[Tuple[int, int, int]] = None - self._dataset: Optional[h5py.Dataset] = None - self._slices_per_sweep: Optional[int] = None - self.filepath = filepath - self._h5file = h5py.File(filepath, "w") - - @property - def no_of_sweeps(self) -> int: - return self._no_of_sweeps + def finalize(self): + self._data = None + if self.h5file is not None: + self.h5file.close() + self.h5file = None @property - def concat_dim(self) -> Literal[0, 1, 2]: - return self._concat_dim + def is_file_based(self) -> bool: + return self._backing is DataSetStoreBacking.File @property - def single_shape(self) -> Tuple[int, int, int]: - if self._single_shape is None: - err_str = "Shape of single sweep result isn't known until the first write has occurred" - raise ValueError(err_str) - return self._single_shape - - @property - def total_shape(self) -> Tuple[int, int, int]: - if self._total_shape is None: - err_str = ( - "Shape of full array holding sweep results isn't known until the first " - "write has occurred" - ) - raise ValueError(err_str) - return self._total_shape - - def make_reader(self) -> ParamSweepReaderHdf5: - if self._dataset is None: - raise ValueError("Cannot make reader when no data has been written yet") - return ParamSweepReaderHdf5(self) - - @property - def no_of_sweeps_written(self) -> int: - return self._no_of_sweeps_written - - def increment_no_of_sweeps_written(self): - self._no_of_sweeps_written += 1 - - @property - def slices_per_sweep(self) -> int: - if self._slices_per_sweep is None: - raise ValueError( - "Slices per sweep isn't known until the first write has occurred" - ) - return self._slices_per_sweep - - @property - def aux_data(self) -> AuxiliaryData: - return self._aux_data - - def write_sweep_result(self, block: ParamSweepBlock): - block.to_cpu() - if self._dataset is None: - self._single_shape = block.shape - self._total_shape = ( - self.single_shape[0], - self.single_shape[1] * self.no_of_sweeps, - self.single_shape[2], - ) - self._slices_per_sweep = self.single_shape[self.concat_dim] - self._dataset = self._h5file.create_dataset( - "/data", shape=self.total_shape, dtype=block.data.dtype - ) - self._aux_data = block.aux_data - - slices = [slice(None, None, 1)] * 3 - sweep_res_start = self.no_of_sweeps_written * self.slices_per_sweep - slices[self._concat_dim] = slice( - sweep_res_start, - sweep_res_start + self.slices_per_sweep, - 1, - ) - self._dataset[slices[0], slices[1], slices[2]] = block.data - self.increment_no_of_sweeps_written() + def filename(self) -> Optional[Path]: + return self._h5filename diff --git a/httomo/sweep_runner/param_sweep_runner.py b/httomo/sweep_runner/param_sweep_runner.py index 0350681d7..a2cec59cc 100644 --- a/httomo/sweep_runner/param_sweep_runner.py +++ b/httomo/sweep_runner/param_sweep_runner.py @@ -8,14 +8,11 @@ import httomo from httomo import globals -from httomo.data.param_sweep_store import ( - ParamSweepReaderHdf5, - ParamSweepWriter, - ParamSweepWriterHdf5, -) +from httomo.data.param_sweep_store import ParamSweepWriter from httomo.method_wrappers.images import ImagesWrapper from httomo.method_wrappers.save_intermediate import SaveIntermediateFilesWrapper from httomo.runner.block_split import BlockSplitter +from httomo.runner.dataset_store_backing import DataSetStoreBacking from httomo.runner.method_wrapper import MethodWrapper from httomo.runner.pipeline import Pipeline from httomo.sweep_runner.param_sweep_block import ParamSweepBlock @@ -220,14 +217,19 @@ def execute_sweep(self): """Execute all param variations of the same method in the sweep""" method = self._stages.sweep.method if method.method_name == "paganin_filter": - sweep_output_filepath = ( - globals.run_out_dir / f"sweep-output-rank-{self._comm.rank}.h5" - ) - writer = ParamSweepWriterHdf5( - len(self._sweep_values), sweep_output_filepath + writer = ParamSweepWriter( + len(self._sweep_values), + self._comm, + globals.run_out_dir, + DataSetStoreBacking.File, ) else: - writer = ParamSweepWriter(len(self._sweep_values)) + writer = ParamSweepWriter( + len(self._sweep_values), + self._comm, + globals.run_out_dir, + DataSetStoreBacking.RAM, + ) log_once(f"Running {method.method_name} ({method.package_name})") log_once(" Beginning parameter sweep") @@ -270,7 +272,7 @@ def execute_sweep(self): reader = writer.make_reader() self._block = reader.read_sweep_results() - if isinstance(reader, ParamSweepReaderHdf5): + if reader.is_file_based: reader.finalize() def execute(self): diff --git a/tests/data/test_param_sweep_store.py b/tests/data/test_param_sweep_store.py index 5505fde5b..b0e3428c4 100644 --- a/tests/data/test_param_sweep_store.py +++ b/tests/data/test_param_sweep_store.py @@ -1,29 +1,35 @@ +from os import PathLike + import pytest import numpy as np +from mpi4py.MPI import COMM_WORLD from pytest_mock import MockerFixture from httomo.data.param_sweep_store import ParamSweepWriter from httomo.runner.auxiliary_data import AuxiliaryData +from httomo.runner.dataset_store_backing import DataSetStoreBacking from httomo.sweep_runner.param_sweep_block import ParamSweepBlock -def make_param_sweep_writer() -> ParamSweepWriter: +def make_param_sweep_writer(tmp_path: PathLike) -> ParamSweepWriter: NO_OF_SWEEPS = 5 - return ParamSweepWriter(NO_OF_SWEEPS) + return ParamSweepWriter(NO_OF_SWEEPS, COMM_WORLD, tmp_path, DataSetStoreBacking.RAM) -def test_param_sweep_writer_get_no_of_sweeps(): - writer = make_param_sweep_writer() +def test_param_sweep_writer_get_no_of_sweeps(tmp_path: PathLike): + writer = make_param_sweep_writer(tmp_path) assert writer.no_of_sweeps == 5 -def test_param_sweep_writer_get_concat_dim(): - writer = make_param_sweep_writer() +def test_param_sweep_writer_get_concat_dim(tmp_path: PathLike): + writer = make_param_sweep_writer(tmp_path) assert writer.concat_dim == 1 -def test_param_sweep_writer_get_single_shape_before_write_raises_error(): - writer = make_param_sweep_writer() +def test_param_sweep_writer_get_single_shape_before_write_raises_error( + tmp_path: PathLike, +): + writer = make_param_sweep_writer(tmp_path) with pytest.raises(ValueError) as e: writer.single_shape assert ( @@ -31,15 +37,19 @@ def test_param_sweep_writer_get_single_shape_before_write_raises_error(): ) in str(e) -def test_param_sweep_writer_get_slices_per_sweep_before_write_raises_error(): - writer = make_param_sweep_writer() +def test_param_sweep_writer_get_slices_per_sweep_before_write_raises_error( + tmp_path: PathLike, +): + writer = make_param_sweep_writer(tmp_path) with pytest.raises(ValueError) as e: writer.slices_per_sweep assert ("Slices per sweep isn't known until the first write has occurred") in str(e) -def test_param_sweep_writer_get_total_shape_before_write_raises_error(): - writer = make_param_sweep_writer() +def test_param_sweep_writer_get_total_shape_before_write_raises_error( + tmp_path: PathLike, +): + writer = make_param_sweep_writer(tmp_path) with pytest.raises(ValueError) as e: writer.total_shape assert ( @@ -48,8 +58,8 @@ def test_param_sweep_writer_get_total_shape_before_write_raises_error(): ) in str(e) -def test_param_sweep_write_make_reader_errors_if_data_none(): - writer = make_param_sweep_writer() +def test_param_sweep_write_make_reader_errors_if_data_none(tmp_path: PathLike): + writer = make_param_sweep_writer(tmp_path) with pytest.raises(ValueError) as e: writer.make_reader() @@ -57,10 +67,10 @@ def test_param_sweep_write_make_reader_errors_if_data_none(): def test_param_sweep_writer_write_sweep_result_transfers_gpu_arr_to_cpu( - mocker: MockerFixture, + mocker: MockerFixture, tmp_path: PathLike ): SWEEP_RES_SHAPE = (180, 3, 160) - writer = make_param_sweep_writer() + writer = make_param_sweep_writer(tmp_path) block = ParamSweepBlock( data=np.ones(SWEEP_RES_SHAPE, dtype=np.float32), aux_data=AuxiliaryData(angles=np.ones(SWEEP_RES_SHAPE[0], dtype=np.float32)), @@ -77,7 +87,7 @@ def test_param_sweep_writer_write_sweep_result_transfers_gpu_arr_to_cpu( to_cpu_spy.assert_called_once() -def test_param_sweep_writer_reader_write_res_and_read(): +def test_param_sweep_writer_reader_write_res_and_read(tmp_path: PathLike): CONCAT_DIM = 1 SWEEP_RES_SHAPE = (180, 3, 160) NO_OF_SWEEPS = 2 @@ -86,7 +96,9 @@ def test_param_sweep_writer_reader_write_res_and_read(): SWEEP_RES_SHAPE[1] * NO_OF_SWEEPS, SWEEP_RES_SHAPE[2], ) - writer = ParamSweepWriter(NO_OF_SWEEPS) + writer = ParamSweepWriter( + NO_OF_SWEEPS, COMM_WORLD, tmp_path, DataSetStoreBacking.RAM + ) # Define an array that will contain data representing the fake result of two parameter # sweep executions. From 50cdfab559ee56dc41b7b76fb0e43895cb523b2b Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 8 Jul 2026 12:23:22 +0100 Subject: [PATCH 4/7] Remove intermediate hdf5 file backing sweep runs --- httomo/data/param_sweep_store.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/httomo/data/param_sweep_store.py b/httomo/data/param_sweep_store.py index 9345139df..c2b12fd58 100644 --- a/httomo/data/param_sweep_store.py +++ b/httomo/data/param_sweep_store.py @@ -75,6 +75,9 @@ def finalize(self): if self.h5file is not None: self.h5file.close() self.h5file = None + if self._h5filename is not None: + self._h5filename.unlink() + self._h5filename = None @property def is_file_based(self) -> bool: From 7424854e50074e11e748702ea50e50088e016bab Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 8 Jul 2026 14:16:06 +0100 Subject: [PATCH 5/7] Add tests for hdf5 backing of param sweep store --- tests/data/test_param_sweep_store.py | 55 ++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/data/test_param_sweep_store.py b/tests/data/test_param_sweep_store.py index b0e3428c4..cede9bad7 100644 --- a/tests/data/test_param_sweep_store.py +++ b/tests/data/test_param_sweep_store.py @@ -143,3 +143,58 @@ def test_param_sweep_writer_reader_write_res_and_read(tmp_path: PathLike): np.testing.assert_array_equal( block_with_middle_slices.data[:, 1, :], SWEEP_RESULT_TWO_MID_SLICE ) + + +def test_writer_closes_file_on_finalize(mocker: MockerFixture, tmp_path: PathLike): + CONCAT_DIM = 1 + BLOCK_SHAPE = (180, 3, 160) + block_data = np.arange(np.prod(BLOCK_SHAPE), dtype=np.float32).reshape(BLOCK_SHAPE) + aux_data = AuxiliaryData(np.ones(BLOCK_SHAPE[0], dtype=np.float32)) + writer = ParamSweepWriter( + no_of_sweeps=1, + comm=COMM_WORLD, + temppath=tmp_path, + backing=DataSetStoreBacking.File, + ) + writer.write_sweep_result( + ParamSweepBlock( + data=block_data, + aux_data=aux_data, + slicing_dim=CONCAT_DIM, + ) + ) + mock_fileclose = mocker.patch.object(writer.h5file, "close") + writer.finalize() + mock_fileclose.assert_called_once() + + +def test_making_reader_closes_file_and_deletes(tmp_path: PathLike): + CONCAT_DIM = 1 + BLOCK_SHAPE = (180, 3, 160) + block_data = np.arange(np.prod(BLOCK_SHAPE), dtype=np.float32).reshape(BLOCK_SHAPE) + aux_data = AuxiliaryData(np.ones(BLOCK_SHAPE[0], dtype=np.float32)) + writer = ParamSweepWriter( + no_of_sweeps=1, + comm=COMM_WORLD, + temppath=tmp_path, + backing=DataSetStoreBacking.File, + ) + writer.write_sweep_result( + ParamSweepBlock( + data=block_data, + aux_data=aux_data, + slicing_dim=CONCAT_DIM, + ) + ) + reader = writer.make_reader() + + assert writer.h5file is None + assert writer.filename is not None + assert writer.filename.exists() + assert reader.filename == writer.filename + assert reader.h5file is not None + assert reader.h5file.get("data", None) is not None + + reader.finalize() + + assert not writer.filename.exists() From f454c0ce238b6a8c9baac78846fd25d385b25b20 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 8 Jul 2026 14:16:31 +0100 Subject: [PATCH 6/7] Finalise source writer when reader created from it --- httomo/data/param_sweep_store.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/httomo/data/param_sweep_store.py b/httomo/data/param_sweep_store.py index c2b12fd58..a5b645d79 100644 --- a/httomo/data/param_sweep_store.py +++ b/httomo/data/param_sweep_store.py @@ -38,6 +38,8 @@ def __init__(self, source: ParamSweepWriter) -> None: else: self._data = source._data + source.finalize() + @property def no_of_sweeps(self) -> int: return self._no_of_sweeps From 750736454e19b82c045c6685dcf654d9f2436a38 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Wed, 8 Jul 2026 14:36:15 +0100 Subject: [PATCH 7/7] Add threshold for sweep run switch from RAM to file --- httomo/sweep_runner/param_sweep_runner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/httomo/sweep_runner/param_sweep_runner.py b/httomo/sweep_runner/param_sweep_runner.py index a2cec59cc..fc20ffa7d 100644 --- a/httomo/sweep_runner/param_sweep_runner.py +++ b/httomo/sweep_runner/param_sweep_runner.py @@ -23,6 +23,10 @@ from httomo.preview import PreviewConfig, PreviewDimConfig from httomo.runner.dataset_store_interfaces import DataSetSource +PAGANIN_SWEEP_VALUE_THRESHOLD = 50 +"""The number of sweep values for the paganin filter above which the sweep results should be +written to an hdf5 file as opposed to kept in RAM.""" + class ParamSweepRunner: def __init__( @@ -216,7 +220,10 @@ def execute_after_sweep(self): def execute_sweep(self): """Execute all param variations of the same method in the sweep""" method = self._stages.sweep.method - if method.method_name == "paganin_filter": + if ( + method.method_name == "paganin_filter" + and len(self._stages.sweep.values) > PAGANIN_SWEEP_VALUE_THRESHOLD + ): writer = ParamSweepWriter( len(self._sweep_values), self._comm,