build_folder is serialized

dace/sdfg/sdfg.py

@@ -8,6 +8,7 @@
 import os
 import json
 from hashlib import md5, sha256
+import pathlib
 import random
 import shutil
 import sys
@@ -148,6 +149,59 @@ def _replace_dict_values(d, old, new):
             d[k] = new
 
 
+def _sdfg_build_folder_getter(sdfg: "SDFG") -> str:
+    """Returns the path to the build cache folder for ``sdfg``.
+
+    If the build folder was explicitly set it is returned. If not set, then the function
+    will derive the root through ``build_folder_root()``, i.e. from the configuration keys
+    ``default_build_folder`` and ``cache_distaware``, and name the folder inside it according
+    to the configuration key ``cache``.
+    It is also important that retrieving the folder through this function does not set
+    the build folder in the SDFG.
+
+    :note: This function is used as getter for the ``SDFG.build_folder`` property, do not use directly.
+    :note: It is unspecific if the path is absolute or not.
+    """
+    if getattr(sdfg, "_build_folder", None) is not None:
+        return sdfg._build_folder
+    cache_config = Config.get('cache')
+    base_folder = build_folder_root()
+    if cache_config == 'single':
+        # Always use the same directory, overwriting any other program,
+        # preventing parallelism and caching of multiple programs, but
+        # saving space and potentially build time
+        return os.path.join(base_folder, 'single_cache')
+    elif cache_config == 'hash':
+        # Any change to the SDFG will result in a new cache folder
+        md5_hash = md5(str(sdfg.to_json()).encode('utf-8')).hexdigest()
+        return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}')
+    elif cache_config == 'unique':
+        # Base name on location in memory, so no caching is possible between
+        # processes or subsequent invocations
+        md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest()
+        return os.path.join(base_folder, f'{sdfg.name}_{md5_hash}')
+    elif cache_config == 'name':
+        # Overwrites previous invocations, and can clash with other programs
+        # if executed in parallel in the same working directory
+        return os.path.join(base_folder, sdfg.name)
+    else:
+        raise ValueError(f'Unknown cache configuration: {cache_config}')
+
+
+def _sdfg_build_folder_setter(sdfg: "SDFG", new_build_folder: Union[str, None, pathlib.Path]) -> None:
+    if new_build_folder is None:
+        sdfg._build_folder = None
+    elif isinstance(new_build_folder, (str, pathlib.Path)):
+        sdfg._build_folder = str(new_build_folder)
+        if len(sdfg._build_folder) == 0:
+            raise ValueError(
+                f'Passed the empty string as new build folder to SDFG "{sdfg.name}", to clear it use `None`.')
+    else:
+        raise TypeError(
+            f'Can not assign "{new_build_folder}" ({type(new_build_folder).__name__}) as new build folder to SDFG "{sdfg.name}".'
+        )
+
+
 def memlets_in_ast(node: ast.AST, arrays: Dict[str, dt.Data], *, include_scalars: bool = False) -> List[mm.Memlet]:
     """
     Generates a list of memlets from each of the subscripts that appear in the Python AST.
@@ -523,8 +577,16 @@ class SDFG(ControlFlowRegion):
                                            default=False,
                                            desc="Whether the SDFG contains explicit control flow constructs")
 
-    # Explicitly-set build folder, or None to derive it from the configuration
-    _build_folder = None
+    build_folder = Property(
+        dtype=str,
+        default=None,
+        allow_none=True,
+        desc='Returns the path to the build cache folder for SDFG. For a in dept '
+        'description see ``_sdfg_build_folder_getter()``.',
+        serialize_if=lambda sdfg: sdfg._build_folder is not None,
+        getter=_sdfg_build_folder_getter,
+        setter=_sdfg_build_folder_setter,
+    )
 
     def __init__(self,
                  name: str,
@@ -1247,38 +1309,6 @@ def as_schedule_tree(self, in_place: bool = False) -> 'ScheduleTreeRoot':
         from dace.sdfg.analysis.schedule_tree import sdfg_to_tree as s2t
         return s2t.as_schedule_tree(self, in_place=in_place)
 
-    @property
-    def build_folder(self) -> str:
-        """ Returns a relative path to the build cache folder for this SDFG. """
-        if self._build_folder is not None:
-            return self._build_folder
-        cache_config = Config.get('cache')
-        base_folder = build_folder_root()
-        if cache_config == 'single':
-            # Always use the same directory, overwriting any other program,
-            # preventing parallelism and caching of multiple programs, but
-            # saving space and potentially build time
-            return os.path.join(base_folder, 'single_cache')
-        elif cache_config == 'hash':
-            # Any change to the SDFG will result in a new cache folder
-            md5_hash = md5(str(self.to_json()).encode('utf-8')).hexdigest()
-            return os.path.join(base_folder, f'{self.name}_{md5_hash}')
-        elif cache_config == 'unique':
-            # Base name on location in memory, so no caching is possible between
-            # processes or subsequent invocations
-            md5_hash = md5(str(os.getpid()).encode('utf-8')).hexdigest()
-            return os.path.join(base_folder, f'{self.name}_{md5_hash}')
-        elif cache_config == 'name':
-            # Overwrites previous invocations, and can clash with other programs
-            # if executed in parallel in the same working directory
-            return os.path.join(base_folder, self.name)
-        else:
-            raise ValueError(f'Unknown cache configuration: {cache_config}')
-
-    @build_folder.setter
-    def build_folder(self, newfolder: str):
-        self._build_folder = newfolder
-
     def remove_data(self, name, validate=True):
         """ Removes a data descriptor from the SDFG.
 

tests/sdfg/build_folder_serialization_test.py

@@ -0,0 +1,96 @@
+# Copyright 2019-2026 ETH Zurich and the DaCe authors. All rights reserved.
+"""Serialization of an explicitly-set build folder.
+
+An explicitly-set ``build_folder`` is part of the user's contract and survives
+a JSON round-trip. A configuration-derived folder (``_build_folder is None``)
+is environment state: it is deliberately NOT serialized - the key is omitted
+entirely, which keeps the serialized form (and thus hashes and build caches)
+of such SDFGs unchanged - and it is restored as derived (``None``), which also
+covers files written before the key existed.
+"""
+
+import pathlib
+import warnings
+
+import dace
+
+
+def _make_sdfg() -> dace.SDFG:
+    sdfg = dace.SDFG('bf_ser_probe')
+    sdfg.add_array('A', [4], dace.float64)
+    return sdfg
+
+
+def test_explicit_build_folder_roundtrip():
+    sdfg = _make_sdfg()
+    sdfg.build_folder = '/some/explicit/folder'
+
+    j = sdfg.to_json()
+    assert j['attributes']['build_folder'] == '/some/explicit/folder'
+
+    with warnings.catch_warnings():
+        # A leftover 'build_folder' key would trigger the "Unused properties"
+        # warning of set_properties_from_json; it must be consumed cleanly.
+        warnings.simplefilter('error')
+        restored = dace.SDFG.from_json(j)
+    assert restored._build_folder == '/some/explicit/folder'
+    assert restored.build_folder == '/some/explicit/folder'
+
+
+def test_relative_explicit_build_folder_kept_verbatim():
+    sdfg = _make_sdfg()
+    sdfg.build_folder = 'relative/dir'
+    restored = dace.SDFG.from_json(sdfg.to_json())
+    assert restored._build_folder == 'relative/dir'
+
+
+def test_pathlib_build_folder_serializes_as_string():
+    sdfg = _make_sdfg()
+    sdfg.build_folder = pathlib.Path('/pathlib/folder')
+
+    # As an implementation detail it is transformed into a string.
+    assert isinstance(sdfg._build_folder, str)
+
+    j = sdfg.to_json()
+    assert j['attributes']['build_folder'] == '/pathlib/folder'
+    assert isinstance(j['attributes']['build_folder'], str)
+    assert dace.SDFG.from_json(j)._build_folder == '/pathlib/folder'
+
+
+def test_derived_build_folder_not_serialized():
+    # No explicit folder: the key is absent - the serialized form of such
+    # SDFGs is byte-identical to before the feature (no hash/cache impact).
+    sdfg = _make_sdfg()
+    j = sdfg.to_json()
+    assert 'build_folder' not in j['attributes']
+
+    restored = dace.SDFG.from_json(j)
+    assert restored._build_folder is None
+
+
+def test_legacy_json_without_key_restores_derived():
+    # Files written before the key existed: nothing to consume, folder derives.
+    sdfg = _make_sdfg()
+    sdfg.build_folder = '/explicit/but/stripped'
+    j = sdfg.to_json()
+    del j['attributes']['build_folder']
+
+    restored = dace.SDFG.from_json(j)
+    assert restored._build_folder is None
+
+
+def test_explicit_build_folder_enters_hash():
+    # Pinning a build location is content: it changes the SDFG hash.
+    plain = _make_sdfg().hash_sdfg()
+    pinned = _make_sdfg()
+    pinned.build_folder = '/some/explicit/folder'
+    assert pinned.hash_sdfg() != plain
+
+
+if __name__ == '__main__':
+    test_explicit_build_folder_roundtrip()
+    test_relative_explicit_build_folder_kept_verbatim()
+    test_pathlib_build_folder_serializes_as_string()
+    test_derived_build_folder_not_serialized()
+    test_legacy_json_without_key_restores_derived()
+    test_explicit_build_folder_enters_hash()

tests/custom_build_folder_test.py → tests/sdfg/custom_build_folder_test.py

@@ -13,29 +13,72 @@ def customprog(A: dace.float64[20]):
     return A + 1
 
 
-def test_custom_build_folder():
+@pytest.fixture
+def unlaunched(monkeypatch):
+    """Drop the rank and cache settings the surrounding environment exports, which override config."""
+    for var in sdfg_module.LAUNCHER_RANK_VARS:
+        monkeypatch.delenv(var, raising=False)
+    for var in ('DACE_cache', 'DACE_cache_distaware', 'DACE_default_build_folder'):
+        monkeypatch.delenv(var, raising=False)
+    return monkeypatch
+
+
+def test_default_build_folder(unlaunched):
+    """Tests if the `default_build_folder` configuration key is respected.
+    """
     with tempfile.TemporaryDirectory() as tmpdir:
-        with dace.config.set_temporary('default_build_folder', value=tmpdir):
+        with dace.config.set_temporary('default_build_folder', value=tmpdir), dace.config.set_temporary('cache',
+                                                                                                        value='single'):
             # Ensure build folder matches
             sdfg = customprog.to_sdfg()
-            assert tmpdir in sdfg.build_folder
+            assert str(sdfg.build_folder).startswith(tmpdir)
+            assert str(sdfg.build_folder).endswith("/single_cache")
+
+            # Ensure that `build_folder` is not serialized if it was not specified.
+            json_dump = sdfg.to_json()
+            assert 'build_folder' not in json_dump['attributes']
+
+            sdfg_restore = dace.SDFG.from_json(json_dump)
+            assert sdfg_restore._build_folder is None
+
             csdfg = sdfg.compile()
+            assert sdfg._build_folder is None
 
             # Ensure files were generated in the right folder
-            assert os.path.isfile(os.path.join(sdfg.build_folder, 'program.sdfgz'))
+            sdfg_dump_path = os.path.join(sdfg.build_folder, 'program.sdfgz')
+            assert os.path.isfile(sdfg_dump_path)
+
+            # Because the build folder was explicitly set during compilation, it should be dumped.
+            assert csdfg._sdfg._build_folder is not None
+            assert csdfg._sdfg._build_folder == sdfg.build_folder
+
+            # Also test if it is was stored in the dump.
+            prog_sdfg = dace.SDFG.from_file(sdfg_dump_path)
+            assert prog_sdfg._build_folder == sdfg.build_folder
 
             # Ensure file is closed so it can be deleted
             del csdfg
 
 
-@pytest.fixture
-def unlaunched(monkeypatch):
-    """Drop the rank and cache settings the surrounding environment exports, which override config."""
-    for var in sdfg_module.LAUNCHER_RANK_VARS:
-        monkeypatch.delenv(var, raising=False)
-    for var in ('DACE_cache', 'DACE_cache_distaware', 'DACE_default_build_folder'):
-        monkeypatch.delenv(var, raising=False)
-    return monkeypatch
+def test_explicitly_set_build_folder(unlaunched):
+    with tempfile.TemporaryDirectory() as tmpdir_def, tempfile.TemporaryDirectory() as tmpdir_used:
+        with dace.config.set_temporary('default_build_folder',
+                                       value=tmpdir_def), dace.config.set_temporary('cache', value='single'):
+
+            # Ensure build folder matches
+            sdfg = customprog.to_sdfg()
+            sdfg.build_folder = tmpdir_used
+
+            # Because the build folder was set explicitly it is used exactly, i.e. the `cache` mode is ignored.
+            assert str(sdfg.build_folder) == tmpdir_used
+            assert str(sdfg._build_folder) == tmpdir_used
+
+            # Ensure that `build_folder` is serialized because it was set explicitly.
+            json_dump = sdfg.to_json()
+            assert json_dump['attributes']['build_folder'] == tmpdir_used
+
+            sdfg_restore = dace.SDFG.from_json(json_dump)
+            assert sdfg_restore._build_folder == tmpdir_used
 
 
 @pytest.mark.parametrize('rank_var', sdfg_module.LAUNCHER_RANK_VARS)
@@ -153,4 +196,4 @@ def test_distributed_compile_puts_every_rank_in_rank_0_folder(unlaunched, tmp_pa
 
 
 if __name__ == '__main__':
-    test_custom_build_folder()
+    print("Must be called using `pytest`.")