diff --git a/docs/source/multi-dd.rst b/docs/source/multi-dd.rst index ae1175f..ea0e9f1 100644 --- a/docs/source/multi-dd.rst +++ b/docs/source/multi-dd.rst @@ -276,27 +276,110 @@ You need to explicitly convert the data, which you can do as follows: entry.put(imas.convert_ids(equilibrium, entry.dd_version)) +.. _`UDA backend and DD versions`: -.. _`DD background`: +UDA backend caching and Data Dictionary versions +------------------------------------------------ -Background information ----------------------- +If you try to load data from a different Data Dictionary version with the UDA backend, +you may see the following error: + +.. code-block:: text + + The Data Dictionary version of the data (3.38.1) is different from the Data + Dictionary version of the DBEntry (3.42.0). This is not supported when using the + UDA backend. + +There are three possible workarounds. The first two require passing an additional option +in the IMAS UDA URI: please see the `imas-core documentation +`__ +for more details on these URI options. + +1. Use UDA fetch to bypass the cache problem. You can do this by appending ``&fetch=1`` + to the URI when you create the :py:class:`~imas.db_entry.DBEntry`. + + Note that this will download the entire IDS files from the remote server, this may + not be desired if you only want to read a single time slice. +2. Disable the UDA cache. You can do this by appending ``&cache_mode=none`` to the URI + when you create the :py:class:`~imas.db_entry.DBEntry`. + + Note that this may make the ``get()`` (a lot) slower, since a separate request needs + to be sent to the remote UDA server for every data variable. However, this may still + be the best performing option if you are only interested in a subset of all the data + in an IDS (and use :ref:`lazy loading`). +3. Explicitly provide the data dictionary version when you create the + :py:class:`~imas.db_entry.DBEntry`, setting it to match the Data Dictionary version + of the data you want to load. To obtain the version of the data on the remote server + from the field `ids_properties.put_version.data_dictionary` via a _lazy_ ``get()`` + with ``autoconvert=False`` option and using the ``&cache_mode=none`` query in the URI. + + Note that you may need to call ``imas.convert_ids`` to convert the IDS to your + desired Data Dictionary version. + +All three possible workarounds are shown in the examples below: + +.. md-tab-set:: + + .. md-tab-item:: Original code + + .. code-block:: python + + import imas + + URI = ( + "imas://uda.iter.org:56565/uda?backend=hdf5" + "&path=/work/imas/shared/imasdb/ITER/3/121013/50" + ) + with imas.DBEntry(URI, "r") as entry: + cp = entry.get("core_profiles") -Since IMAS-Python needs to have access to multiple DD versions it was chosen to -bundle these with the code at build-time, in setup.py. If a git clone of the -Data Dictionary succeeds, the setup tools automatically download saxon and -generate ``IDSDef.xml`` for each of the tagged versions in the DD git -repository. These are then gathered into ``IDSDef.zip``, which is -distributed inside the IMAS-Python package. + .. md-tab-item:: 1. Use UDA fetch -To update the set of data dictionaries new versions can be added to the zipfile. -A reinstall of the package will ensure that all available versions are included -in IMAS-Python. Additionally an explicit path to an XML file can be specified, which -is useful for development. + .. code-block:: python -Automated tests have been provided that check the loading of all of the DD -versions tagged in the data-dictionary git repository. + import imas + URI = ( + "imas://uda.iter.org:56565/uda?backend=hdf5" + "&path=/work/imas/shared/imasdb/ITER/3/121013/50&fetch=1" + ) + with imas.DBEntry(URI, "r") as entry: + cp = entry.get("core_profiles") + + .. md-tab-item:: 2. Disable the UDA cache + + .. code-block:: python + + import imas + + URI = ( + "imas://uda.iter.org:56565/uda?backend=hdf5" + "&path=/work/imas/shared/imasdb/ITER/3/121013/50&cache_mode=none" + ) + with imas.DBEntry(URI, "r") as entry: + cp = entry.get("core_profiles") + + .. md-tab-item:: 3. Explicitly provide the DD version + + .. code-block:: python + + import imas + + URI = ( + "imas://uda.iter.org:56565/uda?backend=hdf5" + "&path=/work/imas/shared/imasdb/ITER/3/121013/50" + ) + with imas.DBEntry(URI, "r", dd_version="3.38.1") as entry: + cp = entry.get("core_profiles") + + # Optional: convert the IDS to your desired DD version + cp = imas.convert_ids(cp, "3.42.0") + + +.. _`DD background`: + +Background information +---------------------- Data Dictionary definitions ''''''''''''''''''''''''''' diff --git a/imas/backends/imas_core/db_entry_al.py b/imas/backends/imas_core/db_entry_al.py index 8559b0c..0a8bf4a 100644 --- a/imas/backends/imas_core/db_entry_al.py +++ b/imas/backends/imas_core/db_entry_al.py @@ -257,6 +257,22 @@ def read_dd_version(self, ids_name: str, occurrence: int) -> str: raise DataEntryException( f"IDS {ids_name!r}, occurrence {occurrence} is empty." ) + + # UDA caching doesn't play well when the DD version of the on-disk IDS doesn't + # match the DD version of this DBEntry. See GH#97 + if self.backend == "uda" and dd_version != self._ids_factory.dd_version: + cache_mode = self._querydict.get("cache_mode") + fetch = self._querydict.get("fetch") + if cache_mode != "none" and fetch not in ("1", "true"): + raise RuntimeError( + f"The Data Dictionary version of the data ({dd_version}) is " + "different from the Data Dictionary version of the DBEntry " + f"({self._ids_factory.dd_version}). This is not supported when " + f"using the UDA backend. See {imas.PUBLISHED_DOCUMENTATION_ROOT}" + "multi-dd.html#uda-backend-caching-and-data-dictionary-versions " + "for more details and workarounds." + ) + return dd_version def put(self, ids: IDSToplevel, occurrence: int, is_slice: bool) -> None: diff --git a/imas/test/test_dbentry_uda.py b/imas/test/test_dbentry_uda.py new file mode 100644 index 0000000..102f5c0 --- /dev/null +++ b/imas/test/test_dbentry_uda.py @@ -0,0 +1,80 @@ +from pathlib import Path +import os +from unittest.mock import patch + +import pytest +from packaging.version import Version + +from imas import DBEntry +from imas.ids_defs import READ_OP + + +@pytest.fixture +def mock_read_data(): + return { + "ids_properties/homogeneous_time": 1, + "ids_properties/version_put/data_dictionary": "4.0.0", + } + + +@pytest.fixture +def mock_ll_interface(mock_read_data): + """Mock the IMAS lowlevel interface so we can still test the our UDA-specific logic. + + Since we don't have a public UDA server available to test against, this is the + next-best thing. + """ + with patch("imas.backends.imas_core.db_entry_al.ll_interface") as mock_ll_interface: + mock_ll_interface.begin_dataentry_action.return_value = (0, 0) + mock_ll_interface.begin_global_action.return_value = (0, 0) + mock_ll_interface.begin_arraystruct_action.return_value = (0, 0, 0) + mock_ll_interface.close_pulse.return_value = 0 + mock_ll_interface._al_version = Version("5.6.0") + + def read_data(ctx, fieldPath, pyTimebasePath, ualDataType, dim): + return 0, mock_read_data.get(fieldPath) + + mock_ll_interface.read_data.side_effect = read_data + + # Also patch in al_context.py: + with patch( + "imas.backends.imas_core.al_context.ll_interface", mock_ll_interface + ): + yield mock_ll_interface + + +def test_uda_idsdef_path(mock_ll_interface): + # Check that IDSDEF_PATH env variable is set for the UDA backend + with DBEntry("imas:uda?mock", "r", dd_version="4.0.0"): + assert "IDSDEF_PATH" in os.environ + path1 = Path(os.environ["IDSDEF_PATH"]) + assert path1.exists() + with DBEntry("imas:uda?mock", "r", dd_version="3.42.0"): + assert "IDSDEF_PATH" in os.environ + path2 = Path(os.environ["IDSDEF_PATH"]) + assert path2.exists() + assert path1 != path2 + + +def test_uda_datapath(mock_ll_interface): + # Check that datapath is set when requesting the dd version + with DBEntry("imas:uda?mock", "r", dd_version="4.0.0") as entry: + mock_ll_interface.begin_global_action.assert_not_called() + entry.get("mhd", lazy=True) + # pulseCtx=0, dataobjectname="mhd", rwmode=READ_OP, datapath="ids_properties" + mock_ll_interface.begin_global_action.assert_called_with( + 0, "mhd", READ_OP, "ids_properties" + ) + + +def test_uda_version_mismatch_exception(mock_ll_interface): + # Check that we get an exception when versions mismatch + with pytest.raises(RuntimeError, match="Data Dictionary version"): + DBEntry("imas:uda?path=mock", "r", dd_version="4.1.0").get("mhd") + # No exceptions when using cache_mode=none + DBEntry("imas:uda?path=mock&cache_mode=none", "r", dd_version="4.1.0").get("mhd") + # Or when using fetch + DBEntry("imas:uda?path=mock&fetch=true", "r", dd_version="4.1.0").get("mhd") + DBEntry("imas:uda?path=mock&fetch=1", "r", dd_version="4.1.0").get("mhd") + # Or when using the exact same DD version + DBEntry("imas:uda?path=mock", "r", dd_version="4.0.0").get("mhd")