diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py index 21705391c0..5942fe1823 100644 --- a/dace/sdfg/sdfg.py +++ b/dace/sdfg/sdfg.py @@ -8,6 +8,7 @@ import os import json from hashlib import md5, sha256 +import pathlib import random import shutil import sys @@ -148,6 +149,59 @@ def _replace_dict_values(d, old, new): d[k] = new +def _sdfg_build_folder_getter(sdfg: "SDFG") -> str: + """Returns the path to the build cache folder for ``sdfg``. + + If the build folder was explicitly set it is returned. If not set, then the function + will derive the root through ``build_folder_root()``, i.e. from the configuration keys + ``default_build_folder`` and ``cache_distaware``, and name the folder inside it according + to the configuration key ``cache``. + It is also important that retrieving the folder through this function does not set + the build folder in the SDFG. + + :note: This function is used as getter for the ``SDFG.build_folder`` property, do not use directly. + :note: It is unspecific if the path is absolute or not. + """ + if getattr(sdfg, "_build_folder", None) is not None: + return sdfg._build_folder + cache_config = Config.get('cache') + base_folder = build_folder_root() + if cache_config == 'single': + # Always use the same directory, overwriting any other program, + # preventing parallelism and caching of multiple programs, but + # saving space and potentially build time + return os.path.join(base_folder, 'single_cache') + elif cache_config == 'hash': + # Any change to the SDFG will result in a new cache folder + md5_hash = md5(str(sdfg.to_json()).encode('utf-8')).hexdigest() + return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}') + elif cache_config == 'unique': + # Base name on location in memory, so no caching is possible between + # processes or subsequent invocations + md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest() + return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}') + elif cache_config == 'name': + # Overwrites previous invocations, and can clash with other programs + # if executed in parallel in the same working directory + return os.path.join(base_folder, sdfg.name) + else: + raise ValueError(f'Unknown cache configuration: {cache_config}') + + +def _sdfg_build_folder_setter(sdfg: "SDFG", new_build_folder: Union[str, None, pathlib.Path]) -> None: + if new_build_folder is None: + sdfg._build_folder = None + elif isinstance(new_build_folder, (str, pathlib.Path)): + sdfg._build_folder = str(new_build_folder) + if len(sdfg._build_folder) == 0: + raise ValueError( + f'Passed the empty string as new build folder to SDFG "{sdfg.name}", to clear it use `None`.') + else: + raise TypeError( + f'Can not assign "{new_build_folder}" ({type(new_build_folder).__name__}) as new build folder to SDFG "{sdfg.name}".' + ) + + def memlets_in_ast(node: ast.AST, arrays: Dict[str, dt.Data], *, include_scalars: bool = False) -> List[mm.Memlet]: """ Generates a list of memlets from each of the subscripts that appear in the Python AST. @@ -523,8 +577,16 @@ class SDFG(ControlFlowRegion): default=False, desc="Whether the SDFG contains explicit control flow constructs") - # Explicitly-set build folder, or None to derive it from the configuration - _build_folder = None + build_folder = Property( + dtype=str, + default=None, + allow_none=True, + desc='Returns the path to the build cache folder for SDFG. For a in dept ' + 'description see ``_sdfg_build_folder_getter()``.', + serialize_if=lambda sdfg: sdfg._build_folder is not None, + getter=_sdfg_build_folder_getter, + setter=_sdfg_build_folder_setter, + ) def __init__(self, name: str, @@ -1247,38 +1309,6 @@ def as_schedule_tree(self, in_place: bool = False) -> 'ScheduleTreeRoot': from dace.sdfg.analysis.schedule_tree import sdfg_to_tree as s2t return s2t.as_schedule_tree(self, in_place=in_place) - @property - def build_folder(self) -> str: - """ Returns a relative path to the build cache folder for this SDFG. """ - if self._build_folder is not None: - return self._build_folder - cache_config = Config.get('cache') - base_folder = build_folder_root() - if cache_config == 'single': - # Always use the same directory, overwriting any other program, - # preventing parallelism and caching of multiple programs, but - # saving space and potentially build time - return os.path.join(base_folder, 'single_cache') - elif cache_config == 'hash': - # Any change to the SDFG will result in a new cache folder - md5_hash = md5(str(self.to_json()).encode('utf-8')).hexdigest() - return os.path.join(base_folder, f'{self.name}_{md5_hash}') - elif cache_config == 'unique': - # Base name on location in memory, so no caching is possible between - # processes or subsequent invocations - md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest() - return os.path.join(base_folder, f'{self.name}_{md5_hash}') - elif cache_config == 'name': - # Overwrites previous invocations, and can clash with other programs - # if executed in parallel in the same working directory - return os.path.join(base_folder, self.name) - else: - raise ValueError(f'Unknown cache configuration: {cache_config}') - - @build_folder.setter - def build_folder(self, newfolder: str): - self._build_folder = newfolder - def remove_data(self, name, validate=True): """ Removes a data descriptor from the SDFG. diff --git a/tests/sdfg/build_folder_serialization_test.py b/tests/sdfg/build_folder_serialization_test.py new file mode 100644 index 0000000000..cdb28af7c6 --- /dev/null +++ b/tests/sdfg/build_folder_serialization_test.py @@ -0,0 +1,96 @@ +# Copyright 2019-2026 ETH Zurich and the DaCe authors. All rights reserved. +"""Serialization of an explicitly-set build folder. + +An explicitly-set ``build_folder`` is part of the user's contract and survives +a JSON round-trip. A configuration-derived folder (``_build_folder is None``) +is environment state: it is deliberately NOT serialized - the key is omitted +entirely, which keeps the serialized form (and thus hashes and build caches) +of such SDFGs unchanged - and it is restored as derived (``None``), which also +covers files written before the key existed. +""" + +import pathlib +import warnings + +import dace + + +def _make_sdfg() -> dace.SDFG: + sdfg = dace.SDFG('bf_ser_probe') + sdfg.add_array('A', [4], dace.float64) + return sdfg + + +def test_explicit_build_folder_roundtrip(): + sdfg = _make_sdfg() + sdfg.build_folder = '/some/explicit/folder' + + j = sdfg.to_json() + assert j['attributes']['build_folder'] == '/some/explicit/folder' + + with warnings.catch_warnings(): + # A leftover 'build_folder' key would trigger the "Unused properties" + # warning of set_properties_from_json; it must be consumed cleanly. + warnings.simplefilter('error') + restored = dace.SDFG.from_json(j) + assert restored._build_folder == '/some/explicit/folder' + assert restored.build_folder == '/some/explicit/folder' + + +def test_relative_explicit_build_folder_kept_verbatim(): + sdfg = _make_sdfg() + sdfg.build_folder = 'relative/dir' + restored = dace.SDFG.from_json(sdfg.to_json()) + assert restored._build_folder == 'relative/dir' + + +def test_pathlib_build_folder_serializes_as_string(): + sdfg = _make_sdfg() + sdfg.build_folder = pathlib.Path('/pathlib/folder') + + # As an implementation detail it is transformed into a string. + assert isinstance(sdfg._build_folder, str) + + j = sdfg.to_json() + assert j['attributes']['build_folder'] == '/pathlib/folder' + assert isinstance(j['attributes']['build_folder'], str) + assert dace.SDFG.from_json(j)._build_folder == '/pathlib/folder' + + +def test_derived_build_folder_not_serialized(): + # No explicit folder: the key is absent - the serialized form of such + # SDFGs is byte-identical to before the feature (no hash/cache impact). + sdfg = _make_sdfg() + j = sdfg.to_json() + assert 'build_folder' not in j['attributes'] + + restored = dace.SDFG.from_json(j) + assert restored._build_folder is None + + +def test_legacy_json_without_key_restores_derived(): + # Files written before the key existed: nothing to consume, folder derives. + sdfg = _make_sdfg() + sdfg.build_folder = '/explicit/but/stripped' + j = sdfg.to_json() + del j['attributes']['build_folder'] + + restored = dace.SDFG.from_json(j) + assert restored._build_folder is None + + +def test_explicit_build_folder_enters_hash(): + # Pinning a build location is content: it changes the SDFG hash. + plain = _make_sdfg().hash_sdfg() + pinned = _make_sdfg() + pinned.build_folder = '/some/explicit/folder' + assert pinned.hash_sdfg() != plain + + +if __name__ == '__main__': + test_explicit_build_folder_roundtrip() + test_relative_explicit_build_folder_kept_verbatim() + test_pathlib_build_folder_serializes_as_string() + test_derived_build_folder_not_serialized() + test_legacy_json_without_key_restores_derived() + test_explicit_build_folder_enters_hash() diff --git a/tests/custom_build_folder_test.py b/tests/sdfg/custom_build_folder_test.py similarity index 70% rename from tests/custom_build_folder_test.py rename to tests/sdfg/custom_build_folder_test.py index e7100f6428..6d06e4412a 100644 --- a/tests/custom_build_folder_test.py +++ b/tests/sdfg/custom_build_folder_test.py @@ -13,29 +13,72 @@ def customprog(A: dace.float64[20]): return A + 1 -def test_custom_build_folder(): +@pytest.fixture +def unlaunched(monkeypatch): + """Drop the rank and cache settings the surrounding environment exports, which override config.""" + for var in sdfg_module.LAUNCHER_RANK_VARS: + monkeypatch.delenv(var, raising=False) + for var in ('DACE_cache', 'DACE_cache_distaware', 'DACE_default_build_folder'): + monkeypatch.delenv(var, raising=False) + return monkeypatch + + +def test_default_build_folder(unlaunched): + """Tests if the `default_build_folder` configuration key is respected. + """ with tempfile.TemporaryDirectory() as tmpdir: - with dace.config.set_temporary('default_build_folder', value=tmpdir): + with dace.config.set_temporary('default_build_folder', value=tmpdir), dace.config.set_temporary('cache', + value='single'): # Ensure build folder matches sdfg = customprog.to_sdfg() - assert tmpdir in sdfg.build_folder + assert str(sdfg.build_folder).startswith(tmpdir) + assert str(sdfg.build_folder).endswith("/single_cache") + + # Ensure that `build_folder` is not serialized if it was not specified. + json_dump = sdfg.to_json() + assert 'build_folder' not in json_dump['attributes'] + + sdfg_restore = dace.SDFG.from_json(json_dump) + assert sdfg_restore._build_folder is None + csdfg = sdfg.compile() + assert sdfg._build_folder is None # Ensure files were generated in the right folder - assert os.path.isfile(os.path.join(sdfg.build_folder, 'program.sdfgz')) + sdfg_dump_path = os.path.join(sdfg.build_folder, 'program.sdfgz') + assert os.path.isfile(sdfg_dump_path) + + # Because the build folder was explicitly set during compilation, it should be dumped. + assert csdfg._sdfg._build_folder is not None + assert csdfg._sdfg._build_folder == sdfg.build_folder + + # Also test if it is was stored in the dump. + prog_sdfg = dace.SDFG.from_file(sdfg_dump_path) + assert prog_sdfg._build_folder == sdfg.build_folder # Ensure file is closed so it can be deleted del csdfg -@pytest.fixture -def unlaunched(monkeypatch): - """Drop the rank and cache settings the surrounding environment exports, which override config.""" - for var in sdfg_module.LAUNCHER_RANK_VARS: - monkeypatch.delenv(var, raising=False) - for var in ('DACE_cache', 'DACE_cache_distaware', 'DACE_default_build_folder'): - monkeypatch.delenv(var, raising=False) - return monkeypatch +def test_explicitly_set_build_folder(unlaunched): + with tempfile.TemporaryDirectory() as tmpdir_def, tempfile.TemporaryDirectory() as tmpdir_used: + with dace.config.set_temporary('default_build_folder', + value=tmpdir_def), dace.config.set_temporary('cache', value='single'): + + # Ensure build folder matches + sdfg = customprog.to_sdfg() + sdfg.build_folder = tmpdir_used + + # Because the build folder was set explicitly it is used exactly, i.e. the `cache` mode is ignored. + assert str(sdfg.build_folder) == tmpdir_used + assert str(sdfg._build_folder) == tmpdir_used + + # Ensure that `build_folder` is serialized because it was set explicitly. + json_dump = sdfg.to_json() + assert json_dump['attributes']['build_folder'] == tmpdir_used + + sdfg_restore = dace.SDFG.from_json(json_dump) + assert sdfg_restore._build_folder == tmpdir_used @pytest.mark.parametrize('rank_var', sdfg_module.LAUNCHER_RANK_VARS) @@ -153,4 +196,4 @@ def test_distributed_compile_puts_every_rank_in_rank_0_folder(unlaunched, tmp_pa if __name__ == '__main__': - test_custom_build_folder() + print("Must be called using `pytest`.")