Skip to content

Commit c90e233

Browse files
authored
Merge pull request #387 from uriahf/feature/public-browser-summary-report
Add opt-in canonical browser summary reports
2 parents 6afd8fd + fa1ced8 commit c90e233

4 files changed

Lines changed: 301 additions & 34 deletions

File tree

‎great-docs.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,10 +101,14 @@ reference:
101101
- create_calibration_curve
102102
- create_calibration_curve_times
103103

104+
- title: Summary Reports
105+
desc: Create historical R-backed reports or explicitly opt into canonical browser ReportSpec rendering.
106+
contents:
107+
- create_summary_report
108+
104109
- title: Utility
105110
desc: Decision-curve analysis for classification and time-to-event models.
106111
contents:
107112
- create_decision_curve
108113
- create_decision_curve_times
109114
- plot_decision_curve
110-

‎src/rtichoke/summary_report/summary_report.py‎

Lines changed: 96 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,72 +1,135 @@
1-
"""
2-
A module for Summary Report
3-
"""
1+
"""Public summary-report entry points."""
42

3+
from __future__ import annotations
4+
5+
import subprocess
6+
from pathlib import Path
7+
from typing import Any, Dict, Literal, Union, cast
8+
9+
import numpy as np
10+
11+
from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list
12+
from rtichoke._performance_table_spec import (
13+
_performance_table_spec_from_performance_data,
14+
)
15+
from rtichoke._report_browser import RtichokeBrowserReport
16+
from rtichoke._report_spec import _report_spec_from_components
17+
from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data
18+
from rtichoke.calibration.calibration import _create_calibration_curve_list
19+
from rtichoke.performance_data.performance_data import prepare_performance_data
20+
from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata
521
from rtichoke.processing.send_post_request_to_r_rtichoke import (
622
send_requests_to_rtichoke_r,
723
)
8-
from rtichoke.processing.transforms import (
9-
_create_list_data_to_adjust,
10-
)
11-
import subprocess
24+
from rtichoke.processing.transforms import _create_list_data_to_adjust
25+
26+
SummaryReportRenderer = Literal["r", "browser"]
27+
28+
29+
def create_summary_report(
30+
probs: Dict[str, np.ndarray],
31+
reals: Union[np.ndarray, Dict[str, np.ndarray]],
32+
url_api: str = "http://localhost:4242/",
33+
*,
34+
renderer: SummaryReportRenderer = "r",
35+
output_file: str | Path = "summary_report.html",
36+
) -> Path | None:
37+
"""Create an rtichoke model-performance summary report.
1238
39+
The default ``renderer="r"`` preserves the historical public behavior and
40+
delegates to the R rtichoke backend at ``url_api``. ``renderer="browser"``
41+
is an explicit opt-in path that uses Python's existing production
42+
calculations, canonical standalone component builders, canonical ReportSpec
43+
assembly, and the vendored ``rtichoke_viz`` ``renderReport()`` composer.
1344
14-
def create_summary_report(probs, reals, url_api="http://localhost:4242/"):
15-
"""Creates a summary report for rtichoke model performance.
45+
The first browser report contains a canonical PerformanceTable, ROC-v2, and
46+
calibration-v2 component, in that order. The browser renderer writes an HTML
47+
file plus the vendored ``rtichoke-viz.js`` and ``rtichoke-viz.css`` assets
48+
beside it, and returns the written HTML path. The historical R path retains
49+
its existing return behavior (``None``).
1650
1751
Parameters
1852
----------
1953
probs : Dict[str, np.ndarray]
20-
A dictionary mapping model names to predicted probabilities.
54+
A dictionary mapping model or population names to predicted probabilities.
2155
reals : Union[np.ndarray, Dict[str, np.ndarray]]
22-
The true outcome labels (0 or 1).
56+
The true binary outcome labels.
2357
url_api : str, optional
24-
The API endpoint URL of the R rtichoke backend.
25-
Defaults to ``"http://localhost:4242/"``.
58+
The API endpoint URL of the historical R rtichoke backend. Used only by
59+
``renderer="r"``. Defaults to ``"http://localhost:4242/"``.
60+
renderer : {"r", "browser"}, optional
61+
Summary-report backend. Defaults to ``"r"`` for backward compatibility.
62+
output_file : str or pathlib.Path, optional
63+
HTML destination for ``renderer="browser"``. Defaults to
64+
``"summary_report.html"``.
65+
66+
Returns
67+
-------
68+
pathlib.Path or None
69+
The generated HTML path for ``renderer="browser"``; ``None`` for the
70+
historical R backend.
2671
"""
72+
if renderer == "browser":
73+
return _create_browser_summary_report(probs, reals, output_file=output_file)
74+
if renderer != "r":
75+
raise ValueError("renderer must be either 'r' or 'browser'")
76+
2777
rtichoke_response = send_requests_to_rtichoke_r(
2878
dictionary_to_send={"probs": probs, "reals": reals},
2979
url_api=url_api,
3080
endpoint="create_summary_report",
3181
)
3282
print(rtichoke_response.json()[0].keys())
83+
return None
3384

3485

35-
def render_summary_report():
36-
"""
37-
Render the rtichoke Summary Report using Quarto.
86+
def _create_browser_summary_report(
87+
probs: Dict[str, np.ndarray],
88+
reals: Union[np.ndarray, Dict[str, np.ndarray]],
89+
*,
90+
output_file: str | Path,
91+
) -> Path:
92+
"""Build the first public canonical browser report from production outputs."""
93+
performance_data = prepare_performance_data(probs, reals)
94+
metadata = _build_evaluation_metadata(probs, reals, np.array([]))
95+
calibration_curve_list = _create_calibration_curve_list(probs, reals)
96+
97+
performance_table = _performance_table_spec_from_performance_data(
98+
performance_data, metadata
99+
)
100+
roc = _roc_v2_spec_from_performance_data(performance_data, metadata)
101+
calibration = _calibration_v2_spec_from_curve_list(calibration_curve_list, metadata)
38102

39-
Args:
40-
probs (list): A list of probabilities.
41-
reals (list): A list of real values.
42-
times (list): A list of absolute numbers representing timestamps.
103+
report = _report_spec_from_components(
104+
[
105+
{"title": "Performance", "spec": performance_table},
106+
{"title": "ROC", "spec": roc},
107+
{"title": "Calibration", "spec": calibration},
108+
],
109+
title="rtichoke summary report",
110+
)
111+
return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file)
43112

44-
Example:
45-
probs = [0.1, 0.4, 0.8]
46-
reals = [0, 1, 1]
47-
times = [1, 3, 5]
48-
render_summary_report(probs, reals, times)
49113

50-
This will generate a `summary_report.html` file based on the `summary_report_template.qmd`.
114+
def render_summary_report():
115+
"""Render the historical rtichoke Summary Report using Quarto.
116+
117+
This function is unchanged by the canonical browser-report opt-in path. It
118+
renders ``aj_estimate_summary_report.qmd`` to ``summary_report.html`` using
119+
the local Quarto executable.
51120
"""
52-
# Define the path to the template and output file
53121
template_path = "aj_estimate_summary_report.qmd"
54122
output_path = "summary_report.html"
55123

56-
# Prepare the command to render the Quarto document
57124
command = [
58125
"quarto",
59126
"render",
60127
template_path,
61128
"--to",
62129
"html",
63130
"--output",
64-
output_path, # ,
65-
# "--execute-params",
66-
# f"probs={probs},reals={reals},times={times}",
131+
output_path,
67132
]
68-
69-
# Execute the command
70133
subprocess.run(command, check=True)
71134

72135

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
import json
2+
from pathlib import Path
3+
from typing import Any, cast
4+
5+
import numpy as np
6+
7+
from rtichoke.summary_report import summary_report as summary_report_module
8+
from rtichoke.summary_report.summary_report import create_summary_report
9+
10+
11+
def _inputs():
12+
probs = {
13+
"Model A": np.array(
14+
[
15+
0.03,
16+
0.08,
17+
0.12,
18+
0.18,
19+
0.25,
20+
0.32,
21+
0.40,
22+
0.50,
23+
0.62,
24+
0.75,
25+
0.88,
26+
0.96,
27+
]
28+
)
29+
}
30+
reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1])
31+
return probs, reals
32+
33+
34+
def _embedded_report(html: str) -> dict[str, Any]:
35+
start = html.index('<script id="rtichoke-report-spec" type="application/json">')
36+
start = html.index(">", start) + 1
37+
end = html.index("</script>", start)
38+
return cast(dict[str, Any], json.loads(html[start:end]))
39+
40+
41+
def test_default_summary_report_keeps_historical_r_backend(monkeypatch, capsys):
42+
probs, reals = _inputs()
43+
calls = []
44+
45+
class Response:
46+
def json(self):
47+
return [{"historical": True}]
48+
49+
def fake_send_requests_to_rtichoke_r(**kwargs):
50+
calls.append(kwargs)
51+
return Response()
52+
53+
def fail_browser(*args, **kwargs):
54+
raise AssertionError("default path must not invoke RtichokeBrowserReport")
55+
56+
monkeypatch.setattr(
57+
summary_report_module,
58+
"send_requests_to_rtichoke_r",
59+
fake_send_requests_to_rtichoke_r,
60+
)
61+
monkeypatch.setattr(
62+
summary_report_module.RtichokeBrowserReport,
63+
"write_html",
64+
fail_browser,
65+
)
66+
67+
result = create_summary_report(probs, reals)
68+
69+
assert result is None
70+
assert len(calls) == 1
71+
assert calls[0]["dictionary_to_send"]["probs"] is probs
72+
assert calls[0]["dictionary_to_send"]["reals"] is reals
73+
assert calls[0]["url_api"] == "http://localhost:4242/"
74+
assert calls[0]["endpoint"] == "create_summary_report"
75+
assert "dict_keys(['historical'])" in capsys.readouterr().out
76+
77+
78+
def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components(
79+
tmp_path,
80+
):
81+
probs, reals = _inputs()
82+
output = tmp_path / "canonical-summary.html"
83+
84+
result = create_summary_report(
85+
probs,
86+
reals,
87+
renderer="browser",
88+
output_file=output,
89+
)
90+
91+
assert result == output
92+
assert output.exists()
93+
assert (tmp_path / "rtichoke-viz.js").exists()
94+
assert (tmp_path / "rtichoke-viz.css").exists()
95+
96+
html = output.read_text(encoding="utf-8")
97+
report = _embedded_report(html)
98+
assert [component["id"] for component in report["components"]] == [
99+
"performance-table",
100+
"roc",
101+
"calibration",
102+
]
103+
assert [component["spec"]["type"] for component in report["components"]] == [
104+
"performance_table",
105+
"roc",
106+
"calibration",
107+
]
108+
assert report["components"][1]["spec"]["schemaVersion"] == "2.0"
109+
assert report["components"][2]["spec"]["schemaVersion"] == "2.0"
110+
assert 'import { renderReport } from "./rtichoke-viz.js";' in html
111+
assert "append(renderReport(spec))" in html
112+
assert "renderPerformanceTable" not in html
113+
assert "renderRocV2" not in html
114+
assert "renderCalibrationV2" not in html
115+
116+
117+
def test_browser_summary_report_preserves_component_local_identity(tmp_path):
118+
probs = {
119+
"Population A": np.array([0.05, 0.2, 0.7, 0.95]),
120+
"Population B": np.array([0.1, 0.4, 0.6, 0.9]),
121+
}
122+
reals = {
123+
"Population A": np.array([0, 0, 1, 1]),
124+
"Population B": np.array([0, 1, 0, 1]),
125+
}
126+
127+
output = create_summary_report(
128+
probs,
129+
reals,
130+
renderer="browser",
131+
output_file=tmp_path / "populations.html",
132+
)
133+
assert isinstance(output, Path)
134+
report = _embedded_report(output.read_text(encoding="utf-8"))
135+
136+
assert "evaluations" not in report
137+
assert "models" not in report
138+
assert "populations" not in report
139+
assert "horizon" not in report
140+
141+
table, roc, calibration = [component["spec"] for component in report["components"]]
142+
assert table["evaluations"][0]["id"] == "evaluation-1"
143+
assert roc["evaluations"][0]["id"] == "evaluation-1"
144+
assert calibration["evaluations"][0]["id"] == "evaluation-1"
145+
assert [item["population"] for item in calibration["evaluations"]] == [
146+
"Population A",
147+
"Population B",
148+
]
149+
assert all("model" not in item for item in calibration["evaluations"])
150+
assert [item["display"]["role"] for item in calibration["series"]] == [
151+
"population",
152+
"population",
153+
]
154+
155+
156+
def test_browser_summary_report_rejects_unknown_renderer():
157+
probs, reals = _inputs()
158+
159+
try:
160+
create_summary_report(probs, reals, renderer="unknown") # type: ignore[arg-type]
161+
except ValueError as exc:
162+
assert str(exc) == "renderer must be either 'r' or 'browser'"
163+
else:
164+
raise AssertionError("unknown renderer should fail")

‎user_guide/06-summary-reports.qmd‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
---
2+
title: "Summary reports"
3+
---
4+
5+
`create_summary_report()` keeps the historical R-backed report path as its default. The canonical browser report is available only when explicitly requested with `renderer="browser"`.
6+
7+
```python
8+
import numpy as np
9+
from rtichoke import create_summary_report
10+
11+
probs = {
12+
"Model A": np.array(
13+
[0.03, 0.08, 0.12, 0.18, 0.25, 0.32, 0.40, 0.50, 0.62, 0.75, 0.88, 0.96]
14+
)
15+
}
16+
reals = np.array([0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 1, 1])
17+
18+
create_summary_report(
19+
probs,
20+
reals,
21+
renderer="browser",
22+
output_file="summary_report.html",
23+
)
24+
```
25+
26+
The browser path uses the same production calculations as the standalone Python components, converts those results with the existing canonical component builders, assembles a canonical ReportSpec, and delegates report composition to the vendored immutable `rtichoke_viz v0.5.0` `renderReport()` implementation.
27+
28+
The first public browser report contains, in deterministic order:
29+
30+
1. PerformanceTable;
31+
2. ROC-v2;
32+
3. calibration-v2.
33+
34+
The generated HTML is accompanied by `rtichoke-viz.js` and `rtichoke-viz.css` in the same directory, so those three files should be kept together when moving the report. The browser backend returns the generated HTML `pathlib.Path`; the default historical R backend retains its existing `None` return behavior.
35+
36+
The browser backend does not replace Quarto or the historical R backend, and it is not the default. Existing Plotly, Matplotlib, table, standalone browser-chart, and time-dependent APIs are unchanged by this opt-in report path.

0 commit comments

Comments
 (0)