From 398576b58c5ad176c02c7056c71b4f055945c110 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Tue, 28 Jul 2026 15:40:28 +0200 Subject: [PATCH 1/8] Build fodler is also serializd. --- dace/sdfg/sdfg.py | 17 +++- tests/sdfg/build_folder_serialization_test.py | 92 +++++++++++++++++++ 2 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/sdfg/build_folder_serialization_test.py diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py index a6197931a8..5f3c001a85 100644 --- a/dace/sdfg/sdfg.py +++ b/dace/sdfg/sdfg.py @@ -683,6 +683,16 @@ def _strip_transformation_history(json_obj: Any): _strip_transformation_history(tmp) tmp['attributes']['name'] = self.name + # An explicitly-set build folder is part of the user's contract and + # survives serialization (it may be machine-specific - that is the + # user's responsibility, like the folder itself). A configuration- + # derived folder (`_build_folder` is None) is environment state and is + # deliberately NOT serialized: the key is omitted entirely, keeping + # the serialized form - and thus hashes and build caches - of such + # SDFGs unchanged, and it also avoids a cycle with the 'hash' cache + # mode, which derives the folder from this very JSON. + if self._build_folder is not None: + tmp['attributes']['build_folder'] = str(self._build_folder) if hash: tmp['attributes']['hash'] = self.hash_sdfg(tmp) @@ -712,10 +722,15 @@ def from_json(cls, json_obj, context=None): ret = SDFG(name=attrs['name'], constants=constants_prop, parent=context['sdfg']) + # An explicitly-set build folder survives serialization; an absent key + # means the folder is configuration-derived (also the format written + # before this key existed) and stays None. + ret._build_folder = attrs.get('build_folder', None) + dace.serialize.set_properties_from_json(ret, json_obj, context=context, - ignore_properties={'constants_prop', 'name', 'hash'}) + ignore_properties={'constants_prop', 'name', 'hash', 'build_folder'}) nodelist = [] for n in nodes: diff --git a/tests/sdfg/build_folder_serialization_test.py b/tests/sdfg/build_folder_serialization_test.py new file mode 100644 index 0000000000..4d2b96aa39 --- /dev/null +++ b/tests/sdfg/build_folder_serialization_test.py @@ -0,0 +1,92 @@ +# Copyright 2019-2026 ETH Zurich and the DaCe authors. All rights reserved. +"""Serialization of an explicitly-set build folder. + +An explicitly-set ``build_folder`` is part of the user's contract and survives +a JSON round-trip. A configuration-derived folder (``_build_folder is None``) +is environment state: it is deliberately NOT serialized - the key is omitted +entirely, which keeps the serialized form (and thus hashes and build caches) +of such SDFGs unchanged - and it is restored as derived (``None``), which also +covers files written before the key existed. +""" + +import pathlib +import warnings + +import dace + + +def _make_sdfg() -> dace.SDFG: + sdfg = dace.SDFG('bf_ser_probe') + sdfg.add_array('A', [4], dace.float64) + return sdfg + + +def test_explicit_build_folder_roundtrip(): + sdfg = _make_sdfg() + sdfg.build_folder = '/some/explicit/folder' + + j = sdfg.to_json() + assert j['attributes']['build_folder'] == '/some/explicit/folder' + + with warnings.catch_warnings(): + # A leftover 'build_folder' key would trigger the "Unused properties" + # warning of set_properties_from_json; it must be consumed cleanly. + warnings.simplefilter('error') + restored = dace.SDFG.from_json(j) + assert restored._build_folder == '/some/explicit/folder' + assert restored.build_folder == '/some/explicit/folder' + + +def test_relative_explicit_build_folder_kept_verbatim(): + sdfg = _make_sdfg() + sdfg.build_folder = 'relative/dir' + restored = dace.SDFG.from_json(sdfg.to_json()) + assert restored._build_folder == 'relative/dir' + + +def test_pathlib_build_folder_serializes_as_string(): + sdfg = _make_sdfg() + sdfg.build_folder = pathlib.Path('/pathlib/folder') + j = sdfg.to_json() + assert j['attributes']['build_folder'] == '/pathlib/folder' + assert isinstance(j['attributes']['build_folder'], str) + assert dace.SDFG.from_json(j)._build_folder == '/pathlib/folder' + + +def test_derived_build_folder_not_serialized(): + # No explicit folder: the key is absent - the serialized form of such + # SDFGs is byte-identical to before the feature (no hash/cache impact). + sdfg = _make_sdfg() + j = sdfg.to_json() + assert 'build_folder' not in j['attributes'] + + restored = dace.SDFG.from_json(j) + assert restored._build_folder is None + + +def test_legacy_json_without_key_restores_derived(): + # Files written before the key existed: nothing to consume, folder derives. + sdfg = _make_sdfg() + sdfg.build_folder = '/explicit/but/stripped' + j = sdfg.to_json() + del j['attributes']['build_folder'] + + restored = dace.SDFG.from_json(j) + assert restored._build_folder is None + + +def test_explicit_build_folder_enters_hash(): + # Pinning a build location is content: it changes the SDFG hash. + plain = _make_sdfg().hash_sdfg() + pinned = _make_sdfg() + pinned.build_folder = '/some/explicit/folder' + assert pinned.hash_sdfg() != plain + + +if __name__ == '__main__': + test_explicit_build_folder_roundtrip() + test_relative_explicit_build_folder_kept_verbatim() + test_pathlib_build_folder_serializes_as_string() + test_derived_build_folder_not_serialized() + test_legacy_json_without_key_restores_derived() + test_explicit_build_folder_enters_hash() From 014c0147b9afb1b987c79ff768b678b5d83ad4e4 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Tue, 28 Jul 2026 15:42:13 +0200 Subject: [PATCH 2/8] Explicit initialization of _build_folder. --- dace/sdfg/sdfg.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py index 5f3c001a85..a0c0a2a6f1 100644 --- a/dace/sdfg/sdfg.py +++ b/dace/sdfg/sdfg.py @@ -537,6 +537,7 @@ def __init__(self, # Helper fields to avoid code generation and compilation self._regenerate_code = True self._recompile = True + self._build_folder = None # Counter to resolve name conflicts self._orig_name = name From 0c864b8a014f7c367e857484d2c477de1a0a6bb9 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Wed, 29 Jul 2026 11:32:30 +0200 Subject: [PATCH 3/8] Undo some things. --- dace/sdfg/sdfg.py | 99 +++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 51 deletions(-) diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py index a0c0a2a6f1..b154c34002 100644 --- a/dace/sdfg/sdfg.py +++ b/dace/sdfg/sdfg.py @@ -119,6 +119,44 @@ def _replace_dict_values(d, old, new): d[k] = new +def _sdfg_build_folder_getter(sdfg) -> str: + """Returns the path to the build cache folder for ``sdfg``. + + If the build folder was explicitly set it is returned. If not set, then the function + will consult the configuration key ``default_build_folder`` and also ``cache``, + which both influence the returned path. + It is also important that retrieving the folder through this function does not set + the build folder in the SDFG. + + :note: This function is used as getter for the ``SDFG.build_folder`` property, do not use directly. + :note: It is unspecific if the path is absolute or not. + """ + if getattr(sdfg, "_build_folder", None) is not None: + return sdfg._build_folder + cache_config = Config.get('cache') + base_folder = Config.get('default_build_folder') + if cache_config == 'single': + # Always use the same directory, overwriting any other program, + # preventing parallelism and caching of multiple programs, but + # saving space and potentially build time + return os.path.join(base_folder, 'single_cache') + elif cache_config == 'hash': + # Any change to the SDFG will result in a new cache folder + md5_hash = md5(str(sdfg.to_json()).encode('utf-8')).hexdigest() + return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}') + elif cache_config == 'unique': + # Base name on location in memory, so no caching is possible between + # processes or subsequent invocations + md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest() + return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}') + elif cache_config == 'name': + # Overwrites previous invocations, and can clash with other programs + # if executed in parallel in the same working directory + return os.path.join(base_folder, sdfg.name) + else: + raise ValueError(f'Unknown cache configuration: {cache_config}') + + def memlets_in_ast(node: ast.AST, arrays: Dict[str, dt.Data], *, include_scalars: bool = False) -> List[mm.Memlet]: """ Generates a list of memlets from each of the subscripts that appear in the Python AST. @@ -487,8 +525,15 @@ class SDFG(ControlFlowRegion): default=False, desc="Whether the SDFG contains explicit control flow constructs") - # Explicitly-set build folder, or None to derive it from the configuration - _build_folder = None + build_folder = Property( + dtype=str, + default=None, + allow_none=True, + getter=_sdfg_build_folder_getter, + desc='Returns the path to the build cache folder for SDFG. For a in dept ' + 'description see ``_sdfg_build_folder_getter()``.', + serialize_if=lambda sdfg: sdfg._build_folder is not None, + ) def __init__(self, name: str, @@ -537,7 +582,6 @@ def __init__(self, # Helper fields to avoid code generation and compilation self._regenerate_code = True self._recompile = True - self._build_folder = None # Counter to resolve name conflicts self._orig_name = name @@ -684,16 +728,6 @@ def _strip_transformation_history(json_obj: Any): _strip_transformation_history(tmp) tmp['attributes']['name'] = self.name - # An explicitly-set build folder is part of the user's contract and - # survives serialization (it may be machine-specific - that is the - # user's responsibility, like the folder itself). A configuration- - # derived folder (`_build_folder` is None) is environment state and is - # deliberately NOT serialized: the key is omitted entirely, keeping - # the serialized form - and thus hashes and build caches - of such - # SDFGs unchanged, and it also avoids a cycle with the 'hash' cache - # mode, which derives the folder from this very JSON. - if self._build_folder is not None: - tmp['attributes']['build_folder'] = str(self._build_folder) if hash: tmp['attributes']['hash'] = self.hash_sdfg(tmp) @@ -723,15 +757,10 @@ def from_json(cls, json_obj, context=None): ret = SDFG(name=attrs['name'], constants=constants_prop, parent=context['sdfg']) - # An explicitly-set build folder survives serialization; an absent key - # means the folder is configuration-derived (also the format written - # before this key existed) and stays None. - ret._build_folder = attrs.get('build_folder', None) - dace.serialize.set_properties_from_json(ret, json_obj, context=context, - ignore_properties={'constants_prop', 'name', 'hash', 'build_folder'}) + ignore_properties={'constants_prop', 'name', 'hash'}) nodelist = [] for n in nodes: @@ -1224,38 +1253,6 @@ def as_schedule_tree(self, in_place: bool = False) -> 'ScheduleTreeRoot': from dace.sdfg.analysis.schedule_tree import sdfg_to_tree as s2t return s2t.as_schedule_tree(self, in_place=in_place) - @property - def build_folder(self) -> str: - """ Returns a relative path to the build cache folder for this SDFG. """ - if self._build_folder is not None: - return self._build_folder - cache_config = Config.get('cache') - base_folder = Config.get('default_build_folder') - if cache_config == 'single': - # Always use the same directory, overwriting any other program, - # preventing parallelism and caching of multiple programs, but - # saving space and potentially build time - return os.path.join(base_folder, 'single_cache') - elif cache_config == 'hash': - # Any change to the SDFG will result in a new cache folder - md5_hash = md5(str(self.to_json()).encode('utf-8')).hexdigest() - return os.path.join(base_folder, f'{self.name}_{md5_hash}') - elif cache_config == 'unique': - # Base name on location in memory, so no caching is possible between - # processes or subsequent invocations - md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest() - return os.path.join(base_folder, f'{self.name}_{md5_hash}') - elif cache_config == 'name': - # Overwrites previous invocations, and can clash with other programs - # if executed in parallel in the same working directory - return os.path.join(base_folder, self.name) - else: - raise ValueError(f'Unknown cache configuration: {cache_config}') - - @build_folder.setter - def build_folder(self, newfolder: str): - self._build_folder = newfolder - def remove_data(self, name, validate=True): """ Removes a data descriptor from the SDFG. From 0aa6629ffe63f53c15371c8f86442269c0559f22 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Wed, 29 Jul 2026 11:50:09 +0200 Subject: [PATCH 4/8] Updated the tests. --- tests/custom_build_folder_test.py | 50 ++++++++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/tests/custom_build_folder_test.py b/tests/custom_build_folder_test.py index d1d22fb3ac..49f5c1761c 100644 --- a/tests/custom_build_folder_test.py +++ b/tests/custom_build_folder_test.py @@ -11,18 +11,60 @@ def customprog(A: dace.float64[20]): def test_custom_build_folder(): with tempfile.TemporaryDirectory() as tmpdir: - with dace.config.set_temporary('default_build_folder', value=tmpdir): + with dace.config.set_temporary('default_build_folder', value=tmpdir), dace.config.set_temporary('cache', + value='single'): # Ensure build folder matches sdfg = customprog.to_sdfg() - assert tmpdir in sdfg.build_folder + assert str(sdfg.build_folder).startswith(tmpdir) + assert str(sdfg.build_folder).endswith("/single_cache") + + # Ensure that `build_folder` is not serialized if it was not specified. + json_dump = sdfg.to_json() + assert 'build_folder' not in json_dump['attributes'] + + sdfg_restore = dace.SDFG.from_json(json_dump) + assert sdfg_restore._build_folder is None + csdfg = sdfg.compile() + assert sdfg._build_folder is None # Ensure files were generated in the right folder - assert os.path.isfile(os.path.join(sdfg.build_folder, 'program.sdfgz')) + sdfg_dump_path = os.path.join(sdfg.build_folder, 'program.sdfgz') + assert os.path.isfile(sdfg_dump_path) + + # Because the build folder was explicitly set during compilation, it should be dumped. + assert csdfg._sdfg._build_folder is not None + assert csdfg._sdfg._build_folder == sdfg.build_folder + + # Also test if it is was stored in the dump. + prog_sdfg = dace.SDFG.from_file(sdfg_dump_path) + assert prog_sdfg._build_folder == sdfg.build_folder # Ensure file is closed so it can be deleted del csdfg +def test_custom_build_folder_2(): + with tempfile.TemporaryDirectory() as tmpdir_def, tempfile.TemporaryDirectory() as tmpdir_used: + with dace.config.set_temporary('default_build_folder', + value=tmpdir_def), dace.config.set_temporary('cache', value='single'): + + # Ensure build folder matches + sdfg = customprog.to_sdfg() + sdfg.build_folder = tmpdir_used + + # Because the build folder was set explicitly it is used exactly, i.e. the `cache` mode is ignored. + assert str(sdfg.build_folder) == tmpdir_used + assert str(sdfg._build_folder) == tmpdir_used + + # Ensure that `build_folder` is serialized because it was set explicitly. + json_dump = sdfg.to_json() + assert json_dump['attributes']['build_folder'] == tmpdir_used + + sdfg_restore = dace.SDFG.from_json(json_dump) + assert sdfg_restore._build_folder == tmpdir_used + + if __name__ == '__main__': - test_custom_build_folder() + #test_custom_build_folder() + test_custom_build_folder_2() From 61d50126f4d5e730445cacf04b930ec0f8742b7f Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Wed, 29 Jul 2026 11:52:54 +0200 Subject: [PATCH 5/8] Forgot to enable. --- tests/custom_build_folder_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/custom_build_folder_test.py b/tests/custom_build_folder_test.py index 49f5c1761c..587e136525 100644 --- a/tests/custom_build_folder_test.py +++ b/tests/custom_build_folder_test.py @@ -66,5 +66,5 @@ def test_custom_build_folder_2(): if __name__ == '__main__': - #test_custom_build_folder() + test_custom_build_folder() test_custom_build_folder_2() From f65718ef10f64f3d0c8930084b079d6aa3875db5 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Wed, 29 Jul 2026 13:19:41 +0200 Subject: [PATCH 6/8] It seems that a setter needed to provide. --- dace/sdfg/sdfg.py | 20 +++++++++++++++++-- tests/custom_build_folder_test.py | 6 ++++-- tests/sdfg/build_folder_serialization_test.py | 4 ++++ 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/dace/sdfg/sdfg.py b/dace/sdfg/sdfg.py index b154c34002..cc6037348c 100644 --- a/dace/sdfg/sdfg.py +++ b/dace/sdfg/sdfg.py @@ -8,6 +8,7 @@ import os import json from hashlib import md5, sha256 +import pathlib import random import shutil import sys @@ -119,7 +120,7 @@ def _replace_dict_values(d, old, new): d[k] = new -def _sdfg_build_folder_getter(sdfg) -> str: +def _sdfg_build_folder_getter(sdfg: "SDFG") -> str: """Returns the path to the build cache folder for ``sdfg``. If the build folder was explicitly set it is returned. If not set, then the function @@ -157,6 +158,20 @@ def _sdfg_build_folder_getter(sdfg) -> str: raise ValueError(f'Unknown cache configuration: {cache_config}') +def _sdfg_build_folder_setter(sdfg: "SDFG", new_build_folder: Union[str, None, pathlib.Path]) -> None: + if new_build_folder is None: + sdfg._build_folder = None + elif isinstance(new_build_folder, (str, pathlib.Path)): + sdfg._build_folder = str(new_build_folder) + if len(sdfg._build_folder) == 0: + raise ValueError( + f'Passed the empty string as new build folder to SDFG "{sdfg.name}", to clear it use `None`.') + else: + raise TypeError( + f'Can not assign "{new_build_folder}" ({type(new_build_folder).__name__}) as new build folder to SDFG "{sdfg.name}".' + ) + + def memlets_in_ast(node: ast.AST, arrays: Dict[str, dt.Data], *, include_scalars: bool = False) -> List[mm.Memlet]: """ Generates a list of memlets from each of the subscripts that appear in the Python AST. @@ -529,10 +544,11 @@ class SDFG(ControlFlowRegion): dtype=str, default=None, allow_none=True, - getter=_sdfg_build_folder_getter, desc='Returns the path to the build cache folder for SDFG. For a in dept ' 'description see ``_sdfg_build_folder_getter()``.', serialize_if=lambda sdfg: sdfg._build_folder is not None, + getter=_sdfg_build_folder_getter, + setter=_sdfg_build_folder_setter, ) def __init__(self, diff --git a/tests/custom_build_folder_test.py b/tests/custom_build_folder_test.py index 587e136525..b2a30650ab 100644 --- a/tests/custom_build_folder_test.py +++ b/tests/custom_build_folder_test.py @@ -9,7 +9,9 @@ def customprog(A: dace.float64[20]): return A + 1 -def test_custom_build_folder(): +def test_default_build_folder(): + """Tests if the `default_build_folder` configuration key is respected. + """ with tempfile.TemporaryDirectory() as tmpdir: with dace.config.set_temporary('default_build_folder', value=tmpdir), dace.config.set_temporary('cache', value='single'): @@ -44,7 +46,7 @@ def test_custom_build_folder(): del csdfg -def test_custom_build_folder_2(): +def test_explicitly_set_build_folder(): with tempfile.TemporaryDirectory() as tmpdir_def, tempfile.TemporaryDirectory() as tmpdir_used: with dace.config.set_temporary('default_build_folder', value=tmpdir_def), dace.config.set_temporary('cache', value='single'): diff --git a/tests/sdfg/build_folder_serialization_test.py b/tests/sdfg/build_folder_serialization_test.py index 4d2b96aa39..cdb28af7c6 100644 --- a/tests/sdfg/build_folder_serialization_test.py +++ b/tests/sdfg/build_folder_serialization_test.py @@ -47,6 +47,10 @@ def test_relative_explicit_build_folder_kept_verbatim(): def test_pathlib_build_folder_serializes_as_string(): sdfg = _make_sdfg() sdfg.build_folder = pathlib.Path('/pathlib/folder') + + # As an implementation detail it is transformed into a string. + assert isinstance(sdfg._build_folder, str) + j = sdfg.to_json() assert j['attributes']['build_folder'] == '/pathlib/folder' assert isinstance(j['attributes']['build_folder'], str) From 074c7240a9a90f7af2d90f73641040834e8165c0 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Wed, 29 Jul 2026 13:20:11 +0200 Subject: [PATCH 7/8] It makes more sense to keep it there. --- tests/{ => sdfg}/custom_build_folder_test.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{ => sdfg}/custom_build_folder_test.py (100%) diff --git a/tests/custom_build_folder_test.py b/tests/sdfg/custom_build_folder_test.py similarity index 100% rename from tests/custom_build_folder_test.py rename to tests/sdfg/custom_build_folder_test.py From ed0f3454be5e48e69d27d7da597ac80fca308b51 Mon Sep 17 00:00:00 2001 From: "Philip Mueller, CSCS" Date: Fri, 31 Jul 2026 10:29:27 +0200 Subject: [PATCH 8/8] Fixed a test. --- tests/sdfg/custom_build_folder_test.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/sdfg/custom_build_folder_test.py b/tests/sdfg/custom_build_folder_test.py index b2a30650ab..ca91a95671 100644 --- a/tests/sdfg/custom_build_folder_test.py +++ b/tests/sdfg/custom_build_folder_test.py @@ -9,9 +9,10 @@ def customprog(A: dace.float64[20]): return A + 1 -def test_default_build_folder(): +def test_default_build_folder(monkeypatch): """Tests if the `default_build_folder` configuration key is respected. """ + monkeypatch.delenv("DACE_cache", raising=False) with tempfile.TemporaryDirectory() as tmpdir: with dace.config.set_temporary('default_build_folder', value=tmpdir), dace.config.set_temporary('cache', value='single'): @@ -46,7 +47,8 @@ def test_default_build_folder(): del csdfg -def test_explicitly_set_build_folder(): +def test_explicitly_set_build_folder(monkeypatch): + monkeypatch.delenv("DACE_cache", raising=False) with tempfile.TemporaryDirectory() as tmpdir_def, tempfile.TemporaryDirectory() as tmpdir_used: with dace.config.set_temporary('default_build_folder', value=tmpdir_def), dace.config.set_temporary('cache', value='single'): @@ -68,5 +70,4 @@ def test_explicitly_set_build_folder(): if __name__ == '__main__': - test_custom_build_folder() - test_custom_build_folder_2() + print("Must be called using `pytest`.")