From 8d3823f6d5ae427074fcd699e3e9d6246caee97b Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Fri, 17 Jul 2026 12:36:56 +0530 Subject: [PATCH 1/4] Embed metadata in each reference results archive Store generation metadata inside each archive's case directory instead of writing one shared file per tutorial. --- changelog-entries/811.md | 1 + tools/tests/README.md | 6 ++- ...docker-compose.field_compare.template.yaml | 2 +- tools/tests/generate_reference_results.py | 47 +++++-------------- .../tests/reference_results.metadata.template | 17 ++++--- 5 files changed, 28 insertions(+), 45 deletions(-) create mode 100644 changelog-entries/811.md diff --git a/changelog-entries/811.md b/changelog-entries/811.md new file mode 100644 index 000000000..99b1d87ab --- /dev/null +++ b/changelog-entries/811.md @@ -0,0 +1 @@ +- Store each `reference_results.metadata` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial ([#811](https://github.com/precice/tutorials/issues/811)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 3321ed7d5..f2e93b9fd 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -176,6 +176,10 @@ The two options cannot be combined: defining any overrides to `reference_version The results will be added to a Git LFS, but you will need special push access: just use the aforementioned GitHub Actions workflow, instead. +Each generated reference archive contains a `reference_results.metadata` file +inside its case directory. This file records the component versions and machine +information used to generate that archive. + #### External test sources Local test cases are defined in the `tutorials:` list. For test cases maintained in other git repositories or available as archives, use the `external:` list. A test suite may define both lists so that external cases can be referenced via YAML anchors alongside local tutorials (for example in the `release` suite). @@ -284,7 +288,7 @@ Metadata and workflow/script files: - `docker-compose.template.yaml`: Describes how to prepare each test (Docker Compose service template) - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template) - `components.yaml`: Declares the available components and their parameters/options - - `reference_results.metadata.template`: Template for reporting the versions used to generate the reference results + - `reference_results.metadata.template`: Template for reporting the versions and machine used to generate each reference results archive - `reference_versions.yaml`: List of arguments to use for generating the reference results - `tests.yaml`: Declares the available tests, grouped in test suites diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 65590ab61..8a2ea4b2d 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol {{ tolerance }} --ignore-missing-reference-files --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --ignore-unsupported-file-formats --diff" diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 0787cc6e4..6904c4c56 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -1,5 +1,5 @@ import argparse -from metadata_parser.metdata import Tutorials, ReferenceResult +from metadata_parser.metdata import Tutorials from systemtests.TestSuite import TestSuites from systemtests.SystemtestArguments import SystemtestArguments from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, ITERATIONS_LOGS_DIR @@ -7,7 +7,6 @@ from typing import List import shutil from paths import PRECICE_TESTS_DIR, PRECICE_TUTORIAL_DIR -import hashlib from jinja2 import Environment, FileSystemLoader import tarfile import subprocess @@ -23,8 +22,9 @@ def create_reference_tar_gz( exports_dir: Path, output_filename: Path, iterations_logs: List[tuple[str, Path]], + metadata: str, ) -> None: - """Archive precice-exports and optional iterations logs as separate top-level tar members.""" + """Archive precice-exports, metadata, and optional iterations logs.""" stem = output_filename.name.replace(".tar.gz", "") exports_staging = system_test_dir / f".{stem}_reference_exports_staging" logs_staging = system_test_dir / f".{stem}_reference_logs_staging" @@ -33,6 +33,7 @@ def create_reference_tar_gz( shutil.rmtree(staging) shutil.copytree(exports_dir, exports_staging) try: + (exports_staging / "reference_results.metadata").write_text(metadata) with tarfile.open(output_filename, "w:gz") as tar: tar.add(exports_staging, arcname=stem) if iterations_logs: @@ -75,30 +76,14 @@ def command_is_avail(command: str): def render_reference_results_info( - reference_results: List[ReferenceResult], + archive_name: str, arguments_used: SystemtestArguments, time: str): - def sha256sum(filename): - # Implementation from https://stackoverflow.com/a/44873382/2254346, - # compatible with Python 3.10. - h = hashlib.sha256() - mv = memoryview(bytearray(128 * 1024)) - with open(filename, 'rb', buffering=0) as f: - while n := f.readinto(mv): - h.update(mv[:n]) - return h.hexdigest() - - files = [] - for reference_result in reference_results: - files.append({ - 'sha256': sha256sum(reference_result.path), - 'time': time, - 'name': reference_result.path.name, - }) uname, lscpu = get_machine_informations() render_dict = { 'arguments': arguments_used.arguments, - 'files': files, + 'archive_name': archive_name, + 'time': time, 'uname': uname, 'lscpu': lscpu, } @@ -197,7 +182,6 @@ def main(): max_time=max_time, max_time_windows=max_time_windows, timeout=timeout, run_before=run_before, run_after=run_after)) - reference_result_per_tutorial = {} current_time_string = datetime.now().strftime('%Y-%m-%d %H:%M:%S') logging.info(f"About to run the following tests {systemtests_to_run}") @@ -209,21 +193,25 @@ def main(): logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds") if not result.success: raise RuntimeError(f"Failed to execute {systemtest}") - reference_result_per_tutorial[systemtest.tutorial] = [] # Put the tar.gz in there for systemtest in systemtests_to_run: reference_result_folder = systemtest.get_system_test_dir() / PRECICE_REL_OUTPUT_DIR - reference_result_per_tutorial[systemtest.tutorial].append(systemtest.reference_result) # create folder if needed systemtest.reference_result.path.parent.mkdir(parents=True, exist_ok=True) if reference_result_folder.exists(): collected = systemtest._collect_iterations_logs(systemtest.get_system_test_dir()) + metadata = render_reference_results_info( + systemtest.reference_result.path.name, + build_args, + current_time_string, + ) create_reference_tar_gz( systemtest.get_system_test_dir(), reference_result_folder, systemtest.reference_result.path, collected, + metadata, ) if collected: logging.info( @@ -235,15 +223,6 @@ def main(): raise RuntimeError( f"Error executing: \n {systemtest} \n Could not find result folder {reference_result_folder}\n Probably the tutorial did not run through properly. Please check corresponding logs") - # write readme - for tutorial in reference_result_per_tutorial.keys(): - reference_results_dir = tutorial.path / "reference-results" - reference_results_dir.mkdir(parents=True, exist_ok=True) - with open(reference_results_dir / "reference_results.metadata", 'w') as file: - ref_results_info = render_reference_results_info( - reference_result_per_tutorial[tutorial], build_args, current_time_string) - logging.info(f"Writing results for {tutorial.name}") - file.write(ref_results_info) logging.info(f"Done. Please make sure to manually have a look into the reference results before making a PR.") diff --git a/tools/tests/reference_results.metadata.template b/tools/tests/reference_results.metadata.template index d7f189fe2..2e196c865 100644 --- a/tools/tests/reference_results.metadata.template +++ b/tools/tests/reference_results.metadata.template @@ -4,17 +4,16 @@ This File has been generated by the generate_reference_results.py and should not # Reference Results -This file contains an overview of the results over the reference results as well as the arguments used to generate them. -We also include some information on the machine used to generate them +This file describes the reference results in `{{ archive_name }}` and includes +the arguments and machine used to generate them. -## List of files +## Archive -| name | time | sha256 | -|------|------|-------| -{% for file in files -%} -| {{ file.name }} | {{ file.time }} | {{ file.sha256 }} | -{% endfor %} -## List of arguments used to generate the files +| name | generated at | +|------|--------------| +| {{ archive_name }} | {{ time }} | + +## Arguments used to generate the results | name | value | |------|------| From 8740f97fc12e97bfac66f79fc1effbb7fe8afe51 Mon Sep 17 00:00:00 2001 From: Pranjal Date: Fri, 17 Jul 2026 12:53:30 +0530 Subject: [PATCH 2/4] Update 811.md --- changelog-entries/811.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog-entries/811.md b/changelog-entries/811.md index 99b1d87ab..173958741 100644 --- a/changelog-entries/811.md +++ b/changelog-entries/811.md @@ -1 +1 @@ -- Store each `reference_results.metadata` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial ([#811](https://github.com/precice/tutorials/issues/811)). +- Store each `reference_results.metadata` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial [#882](https://github.com/precice/tutorials/pull/882). From 3130da7d865f0b9762cb3e3150b65c7199caa14c Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sat, 18 Jul 2026 14:13:31 +0530 Subject: [PATCH 3/4] Rename embedded metadata and ignore it in fieldcompare Rename the per-archive metadata file to reference-results-metadata.txt and add --ignore-missing-source-files so fieldcompare does not fail on the metadata when comparing against regenerated archives. --- changelog-entries/811.md | 2 +- tools/tests/README.md | 5 +++-- tools/tests/docker-compose.field_compare.template.yaml | 2 +- tools/tests/generate_reference_results.py | 4 ++-- ...data.template => reference-results-metadata.txt.template} | 0 5 files changed, 7 insertions(+), 6 deletions(-) rename tools/tests/{reference_results.metadata.template => reference-results-metadata.txt.template} (100%) diff --git a/changelog-entries/811.md b/changelog-entries/811.md index 173958741..8a8bfc6ae 100644 --- a/changelog-entries/811.md +++ b/changelog-entries/811.md @@ -1 +1 @@ -- Store each `reference_results.metadata` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial [#882](https://github.com/precice/tutorials/pull/882). +- Store each `reference-results-metadata.txt` inside the case directory of its reference results archive instead of sharing one metadata file between all archives of a tutorial. The SHA256 checksum of the outer archive is no longer included, because the metadata now lives inside that archive [#882](https://github.com/precice/tutorials/pull/882). diff --git a/tools/tests/README.md b/tools/tests/README.md index f2e93b9fd..75c547c93 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -107,6 +107,7 @@ To reproduce the comparison locally, use the [same fieldcompare command](https:/ ```bash fieldcompare dir precice-exports/ reference-results-unpacked// \ --ignore-missing-reference-files \ + --ignore-missing-source-files \ --ignore-unsupported-file-formats \ -rtol 3e-7 ``` @@ -176,7 +177,7 @@ The two options cannot be combined: defining any overrides to `reference_version The results will be added to a Git LFS, but you will need special push access: just use the aforementioned GitHub Actions workflow, instead. -Each generated reference archive contains a `reference_results.metadata` file +Each generated reference archive contains a `reference-results-metadata.txt` file inside its case directory. This file records the component versions and machine information used to generate that archive. @@ -288,7 +289,7 @@ Metadata and workflow/script files: - `docker-compose.template.yaml`: Describes how to prepare each test (Docker Compose service template) - `docker-compose.field_compare.template.yaml`: Describes how to compare results with fieldcompare (Docker Compose service template) - `components.yaml`: Declares the available components and their parameters/options - - `reference_results.metadata.template`: Template for reporting the versions and machine used to generate each reference results archive + - `reference-results-metadata.txt.template`: Template for reporting the versions and machine used to generate each reference results archive - `reference_versions.yaml`: List of arguments to use for generating the reference results - `tests.yaml`: Declares the available tests, grouped in test suites diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 8a2ea4b2d..5aeba51ce 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol {{ tolerance }} --ignore-missing-reference-files --ignore-unsupported-file-formats --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --ignore-missing-source-files --ignore-unsupported-file-formats --diff" diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index 6904c4c56..c2f3b4425 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -33,7 +33,7 @@ def create_reference_tar_gz( shutil.rmtree(staging) shutil.copytree(exports_dir, exports_staging) try: - (exports_staging / "reference_results.metadata").write_text(metadata) + (exports_staging / "reference-results-metadata.txt").write_text(metadata) with tarfile.open(output_filename, "w:gz") as tar: tar.add(exports_staging, arcname=stem) if iterations_logs: @@ -89,7 +89,7 @@ def render_reference_results_info( } jinja_env = Environment(loader=FileSystemLoader(PRECICE_TESTS_DIR)) - template = jinja_env.get_template("reference_results.metadata.template") + template = jinja_env.get_template("reference-results-metadata.txt.template") return template.render(render_dict) diff --git a/tools/tests/reference_results.metadata.template b/tools/tests/reference-results-metadata.txt.template similarity index 100% rename from tools/tests/reference_results.metadata.template rename to tools/tests/reference-results-metadata.txt.template From a35869399d9bc2cb67a9023b2ed832ea12f98748 Mon Sep 17 00:00:00 2001 From: PranjalManhgaye Date: Sun, 19 Jul 2026 09:30:19 +0530 Subject: [PATCH 4/4] Exclude embedded metadata from fieldcompare precisely Replace --ignore-missing-source-files with --exclude-files reference-results-metadata.txt so only the embedded metadata is skipped and missing timesteps still fail comparison. --- tools/tests/README.md | 2 +- tools/tests/docker-compose.field_compare.template.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/tests/README.md b/tools/tests/README.md index 75c547c93..f0e44adb8 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -107,7 +107,7 @@ To reproduce the comparison locally, use the [same fieldcompare command](https:/ ```bash fieldcompare dir precice-exports/ reference-results-unpacked// \ --ignore-missing-reference-files \ - --ignore-missing-source-files \ + --exclude-files reference-results-metadata.txt \ --ignore-unsupported-file-formats \ -rtol 3e-7 ``` diff --git a/tools/tests/docker-compose.field_compare.template.yaml b/tools/tests/docker-compose.field_compare.template.yaml index 5aeba51ce..8a789b109 100644 --- a/tools/tests/docker-compose.field_compare.template.yaml +++ b/tools/tests/docker-compose.field_compare.template.yaml @@ -6,4 +6,4 @@ services: command: - /runs/{{ tutorial_folder }}/{{ precice_output_folder }} - /runs/{{ tutorial_folder }}/{{ reference_output_folder }} - - "-rtol {{ tolerance }} --ignore-missing-reference-files --ignore-missing-source-files --ignore-unsupported-file-formats --diff" + - "-rtol {{ tolerance }} --ignore-missing-reference-files --exclude-files reference-results-metadata.txt --ignore-unsupported-file-formats --diff"