diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 67890a85..af257bb4 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -52,14 +52,15 @@ jobs: prefix = "rtichoke/_vendor/rtichoke_viz/" required = { f"{prefix}VENDORED_FROM", - f"{prefix}rtichoke-viz-0.10.0.tar.gz", + f"{prefix}rtichoke-viz-0.14.0.tar.gz", f"{prefix}rtichoke-viz.js", f"{prefix}rtichoke-viz.css", f"{prefix}rtichoke-viz.schema.json", f"{prefix}rtichoke-viz-v2.schema.json", + f"{prefix}rtichoke-viz-report.schema.json", } assert required <= names - assert f"{prefix}rtichoke-viz-0.9.0.tar.gz" not in names + assert f"{prefix}rtichoke-viz-0.10.0.tar.gz" not in names PY - name: Run tests diff --git a/pyproject.toml b/pyproject.toml index efa25ca0..c98d32ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "polars>=1.31.0", "reactable>=0.1.5", "great-tables>=0.18.0", + "scikit-learn>=1.6.1", ] name = "rtichoke" version = "0.1.36" @@ -59,7 +60,6 @@ dev = [ "pre-commit>=4.2.0", "dcurves>=1.1.5", "ty>=0.0.1a12", - "scikit-learn>=1.6.1", ] docs = [ "great-docs==0.15.0", diff --git a/src/rtichoke/_calibration_viz_spec_v2.py b/src/rtichoke/_calibration_viz_spec_v2.py index 29af0d15..9a5d4e15 100644 --- a/src/rtichoke/_calibration_viz_spec_v2.py +++ b/src/rtichoke/_calibration_viz_spec_v2.py @@ -151,6 +151,10 @@ def _calibration_v2_spec_from_curve_list( } ) + y_axis: dict[str, object] = {"label": "Observed probability"} + if calibration_type == "discrete": + y_axis["domain"] = [0, 1] + return { "schemaVersion": "2.0", "type": "calibration", @@ -161,7 +165,7 @@ def _calibration_v2_spec_from_curve_list( "x": "predicted", "y": "observed", "xAxis": {"label": "Predicted probability", "domain": [0, 1]}, - "yAxis": {"label": "Observed probability", "domain": [0, 1]}, + "yAxis": y_axis, "references": [{"type": "identity", "scope": "global"}], } diff --git a/src/rtichoke/_report_browser.py b/src/rtichoke/_report_browser.py index c3693502..385fa71f 100644 --- a/src/rtichoke/_report_browser.py +++ b/src/rtichoke/_report_browser.py @@ -41,7 +41,10 @@ def write_html(self, path: str | Path) -> Path: const spec = JSON.parse( document.querySelector("#rtichoke-report-spec").textContent ); - document.querySelector("#rtichoke-report").append(renderReport(spec)); + document.querySelector("#rtichoke-report").append(renderReport(spec, {{ + sectionGroupPresentation: "tabs", + groupPresentation: "stacked" + }})); diff --git a/src/rtichoke/_report_spec.py b/src/rtichoke/_report_spec.py index 9602fc0f..2f65082e 100644 --- a/src/rtichoke/_report_spec.py +++ b/src/rtichoke/_report_spec.py @@ -1,102 +1,210 @@ -"""Internal assembler for canonical ``rtichoke_viz`` ReportSpec objects. +"""Internal assembler for canonical ``rtichoke_viz`` ReportSpec v1.1 objects. -The assembler composes complete standalone canonical component specs. It does -not calculate statistics, normalize component specs, hoist evaluations, or +The assembler composes complete standalone canonical component specs into a structured +ReportSpec v1.1 object hierarchy with sections, items (components and groups), and title. +It does not calculate statistics, normalize component specs, hoist evaluations, or create report-global semantic registries. - -This path is intentionally separate from the existing public summary-report -API, which currently delegates to the historical R backend and Quarto -composition. Complete ReportSpec values can now use the internal shared-browser -path backed by the immutable vendored ``rtichoke_viz`` release, while the -existing public report behavior remains unchanged. """ from __future__ import annotations from collections.abc import Mapping, Sequence -from typing import TypedDict, cast +from typing import Any, TypedDict, cast + +_V10_SCHEMA_TYPES = { + "summary_metrics", +} -_SUPPORTED_COMPONENT_TYPES = { +_V20_SCHEMA_TYPES = { "performance_table", "roc", "calibration", "precision_recall", "gains", "lift", + "decision_curve", + "interventions_avoided", } -_COMPONENT_ID_BASES = { - "performance_table": "performance-table", - "roc": "roc", - "calibration": "calibration", - "precision_recall": "precision-recall", - "gains": "gains", - "lift": "lift", -} + +_ALL_SUPPORTED_TYPES = _V10_SCHEMA_TYPES | _V20_SCHEMA_TYPES -class _ReportComponent(TypedDict, total=False): +def _validate_spec_schema_version(spec: Mapping[str, object]) -> None: + """Validate that spec schemaVersion strictly matches type requirements. + + summary_metrics -> "1.0" + all v2 component types -> "2.0" + """ + spec_type = spec.get("type") + if not isinstance(spec_type, str): + raise ValueError("Report component spec is missing a string type") + if spec_type not in _ALL_SUPPORTED_TYPES: + raise ValueError(f"Unsupported ReportSpec component type: {spec_type}") + + schema_version = spec.get("schemaVersion") + if not isinstance(schema_version, str): + raise ValueError("Report component spec is missing a string schemaVersion") + + if spec_type in _V10_SCHEMA_TYPES: + if schema_version != "1.0": + raise ValueError( + f"Component type {spec_type!r} requires schemaVersion '1.0', got {schema_version!r}" + ) + elif spec_type in _V20_SCHEMA_TYPES: + if schema_version != "2.0": + raise ValueError( + f"Component type {spec_type!r} requires schemaVersion '2.0', got {schema_version!r}" + ) + + +class _ReportComponentV11(TypedDict, total=False): + type: str id: str title: str spec: Mapping[str, object] -class _ReportSpec(TypedDict, total=False): +class _ReportGroupV11(TypedDict, total=False): + type: str + id: str + title: str + components: list[_ReportComponentV11] + + +class _ReportSectionV11(TypedDict, total=False): + id: str + title: str + items: list[_ReportComponentV11 | _ReportGroupV11] + + +class _ReportSpecV11(TypedDict, total=False): schemaVersion: str type: str title: str - components: list[_ReportComponent] + sections: list[_ReportSectionV11] -def _report_spec_from_components( - components: Sequence[Mapping[str, object]], +def _build_report_spec_v11( + sections: Sequence[Mapping[str, Any]], *, title: str | None = None, -) -> _ReportSpec: - """Compose complete canonical component specs into a ReportSpec. - - Component order is preserved exactly. Component IDs are deterministic and - live in a report-local identity domain: the first component of a type gets - its boring base ID (for example ``roc``), and repeats get ``-2``, ``-3``, - and so on. Embedded specs are retained as-is, so evaluation IDs remain - component-local even when equal strings occur in multiple components. +) -> _ReportSpecV11: + """Compose structured ReportSpec v1.1 hierarchy. + + Validates type-aware schemaVersion for every embedded component spec. """ - if not components: - raise ValueError("ReportSpec requires at least one component") - - type_counts: dict[str, int] = {} - report_components: list[_ReportComponent] = [] - for component in components: - raw_spec = component.get("spec") - if not isinstance(raw_spec, Mapping): - raise ValueError("Report component is missing spec") - spec = cast(Mapping[str, object], raw_spec) - - component_type = spec.get("type") - if not isinstance(component_type, str): - raise ValueError("Report component spec is missing a string type") - if component_type not in _SUPPORTED_COMPONENT_TYPES: - raise ValueError(f"Unsupported ReportSpec component type: {component_type}") - - count = type_counts.get(component_type, 0) + 1 - type_counts[component_type] = count - base_id = _COMPONENT_ID_BASES[component_type] - component_id = base_id if count == 1 else f"{base_id}-{count}" - - assembled: _ReportComponent = { - "id": component_id, - "spec": spec, + if not sections: + raise ValueError("ReportSpec v1.1 requires at least one section") + + assembled_sections: list[_ReportSectionV11] = [] + + for section in sections: + sec_id = section.get("id") + if not isinstance(sec_id, str) or not sec_id: + raise ValueError("Report section must have a non-empty string id") + + sec_title = section.get("title") + if sec_title is None: + sec_title = sec_id + elif not isinstance(sec_title, str) or not sec_title: + raise ValueError("Report section must have a non-empty string title") + + items_assembled: list[_ReportComponentV11 | _ReportGroupV11] = [] + + components_raw = section.get("components") + if components_raw is not None: + if not isinstance(components_raw, Sequence): + raise ValueError("Section components must be a sequence") + for comp in components_raw: + if not isinstance(comp, Mapping): + raise ValueError("Report component must be a Mapping") + comp_id = comp.get("id") + raw_spec = comp.get("spec") + if not isinstance(comp_id, str) or not comp_id: + raise ValueError("Report component must have a non-empty string id") + if not isinstance(raw_spec, Mapping): + raise ValueError("Report component is missing spec") + _validate_spec_schema_version(cast(Mapping[str, object], raw_spec)) + assembled_comp: _ReportComponentV11 = { + "type": "component", + "id": comp_id, + "spec": raw_spec, + } + comp_title = comp.get("title") + if comp_title is not None: + if not isinstance(comp_title, str): + raise ValueError("Report component title must be a string") + assembled_comp["title"] = comp_title + items_assembled.append(assembled_comp) + + groups_raw = section.get("groups") + if groups_raw is not None: + if not isinstance(groups_raw, Sequence): + raise ValueError("Section groups must be a sequence") + for group in groups_raw: + if not isinstance(group, Mapping): + raise ValueError("Report group must be a Mapping") + group_id = group.get("id") + group_title = group.get("title") + if not isinstance(group_id, str) or not group_id: + raise ValueError("Report group must have a non-empty string id") + if not isinstance(group_title, str) or not group_title: + raise ValueError("Report group must have a non-empty string title") + + grp_comps_raw = group.get("components") + if not isinstance(grp_comps_raw, Sequence) or not grp_comps_raw: + raise ValueError("Group components must be a non-empty sequence") + + grp_components: list[_ReportComponentV11] = [] + for comp in grp_comps_raw: + if not isinstance(comp, Mapping): + raise ValueError("Report component must be a Mapping") + comp_id = comp.get("id") + raw_spec = comp.get("spec") + if not isinstance(comp_id, str) or not comp_id: + raise ValueError( + "Report component must have a non-empty string id" + ) + if not isinstance(raw_spec, Mapping): + raise ValueError("Report component is missing spec") + _validate_spec_schema_version(cast(Mapping[str, object], raw_spec)) + assembled_comp = cast( + _ReportComponentV11, + { + "type": "component", + "id": comp_id, + "spec": raw_spec, + }, + ) + comp_title = comp.get("title") + if comp_title is not None: + if not isinstance(comp_title, str): + raise ValueError("Report component title must be a string") + assembled_comp["title"] = comp_title + grp_components.append(assembled_comp) + + grp_dict: _ReportGroupV11 = { + "type": "group", + "id": group_id, + "title": group_title, + "components": grp_components, + } + items_assembled.append(grp_dict) + + if not items_assembled: + raise ValueError(f"Section {sec_id!r} has no components or groups") + + assembled_sec: _ReportSectionV11 = { + "id": sec_id, + "title": sec_title, + "items": items_assembled, } - component_title = component.get("title") - if component_title is not None: - if not isinstance(component_title, str): - raise ValueError("Report component title must be a string") - assembled["title"] = component_title - report_components.append(assembled) - - report: _ReportSpec = { - "schemaVersion": "1.0", + assembled_sections.append(assembled_sec) + + report: _ReportSpecV11 = { + "schemaVersion": "1.1", "type": "report", - "components": report_components, + "sections": assembled_sections, } if title is not None: report["title"] = title diff --git a/src/rtichoke/_summary_metrics_spec.py b/src/rtichoke/_summary_metrics_spec.py new file mode 100644 index 00000000..589c53f1 --- /dev/null +++ b/src/rtichoke/_summary_metrics_spec.py @@ -0,0 +1,241 @@ +"""Internal builder for canonical SummaryMetricsSpec v1.0 objects. + +Calculates SummaryMetricsSpec components (Prevalence and AUROC) from production +inputs and prepared performance data. Does not perform statistical rendering or +alter global evaluation identity. +""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any, TypedDict, cast + +import numpy as np +import polars as pl +from sklearn.metrics import roc_auc_score + +from rtichoke.performance_data.performance_data import _validate_and_align_binary_inputs +from rtichoke.processing.evaluation_semantics import ( + _EvaluationMetadata, + _SHARED_POPULATION, +) + + +class _PopulationSpec(TypedDict): + id: str + label: str + + +class _EvaluationSpec(TypedDict, total=False): + id: str + model: str + population: str + label: str + + +class _PopulationOwner(TypedDict): + type: str + populationId: str + + +class _EvaluationOwner(TypedDict): + type: str + evaluationId: str + + +class _PrevalenceMetric(TypedDict): + metric: str + owner: _PopulationOwner + estimate: float | None + + +class _AurocMetric(TypedDict): + metric: str + owner: _EvaluationOwner + estimate: float | None + + +class _SummaryMetricsSpec(TypedDict, total=False): + schemaVersion: str + type: str + title: str + evaluations: list[_EvaluationSpec] + populations: list[_PopulationSpec] + metrics: list[dict[str, Any]] + + +def _compute_auroc_proc_compatible( + reals: np.ndarray, + probs: np.ndarray, +) -> float | None: + """Compute AUROC with historical pROC auto-direction compatibility. + + If outcomes contain only a single class (all 0s or all 1s), returns None. + Otherwise, compares the median score of controls (y==0) and cases (y==1). + If median(controls) <= median(cases), computes roc_auc_score with original + scores. If median(controls) > median(cases), reverses ranking using -scores. + """ + reals_arr = np.asarray(reals) + probs_arr = np.asarray(probs) + + unique_reals = np.unique(reals_arr) + if not set(unique_reals).issubset({0, 1}) or len(unique_reals) < 2: + return None + + controls = probs_arr[reals_arr == 0] + cases = probs_arr[reals_arr == 1] + if len(controls) == 0 or len(cases) == 0: + return None + + med_controls = np.median(controls) + med_cases = np.median(cases) + + scores = probs_arr if med_controls <= med_cases else -probs_arr + + score_val = float(roc_auc_score(reals_arr, scores)) + if math.isnan(score_val) or not math.isfinite(score_val): + return None + return score_val + + +def _prevalence_summary_metrics_spec( + performance_data: pl.DataFrame, + evaluation_metadata: Mapping[str, _EvaluationMetadata], +) -> _SummaryMetricsSpec: + """Build canonical population-owned Prevalence SummaryMetricsSpec v1.0.""" + populations_list: list[_PopulationSpec] = [] + metrics_list: list[dict[str, Any]] = [] + + seen_populations: dict[tuple[str, str], str] = {} + pop_counter = 1 + + for group, metadata in evaluation_metadata.items(): + pop_name = metadata.population + label = "Population" if pop_name == _SHARED_POPULATION else pop_name + # Keep distinct populations with identical display labels distinct by keying on (group, pop_name) or pop_name + pop_key = ( + (group, pop_name) + if pop_name != _SHARED_POPULATION + and list(evaluation_metadata.values())[0].model is None + else (pop_name, pop_name) + ) + if pop_key not in seen_populations: + pop_id = f"population-{pop_counter}" + pop_counter += 1 + seen_populations[pop_key] = pop_id + populations_list.append({"id": pop_id, "label": label}) + + # Compute population prevalence from performance data + group_prevalence: dict[str, float] = {} + for row in ( + performance_data.select( + "reference_group", + (pl.col("real_positives") / pl.col("n")).alias("prevalence"), + ) + .unique() + .to_dicts() + ): + group = str(row["reference_group"]) + prev_val = row["prevalence"] + if prev_val is not None and math.isfinite(float(prev_val)): + group_prevalence[group] = float(prev_val) + + pop_estimates: dict[tuple[str, str], set[float]] = {} + for group, metadata in evaluation_metadata.items(): + if group in group_prevalence: + pop_name = metadata.population + pop_key = ( + (group, pop_name) + if pop_name != _SHARED_POPULATION and metadata.model is None + else (pop_name, pop_name) + ) + pop_estimates.setdefault(pop_key, set()).add(group_prevalence[group]) + + for pop_spec in populations_list: + pop_id = pop_spec["id"] + pop_key = next(key for key, pid in seen_populations.items() if pid == pop_id) + estimates = pop_estimates.get(pop_key, set()) + estimate: float | None = next(iter(estimates)) if len(estimates) == 1 else None + + metric_item: _PrevalenceMetric = { + "metric": "prevalence", + "owner": { + "type": "population", + "populationId": pop_id, + }, + "estimate": estimate, + } + metrics_list.append(cast(dict[str, Any], metric_item)) + + return { + "schemaVersion": "1.0", + "type": "summary_metrics", + "title": "Prevalence summary", + "evaluations": [], + "populations": populations_list, + "metrics": metrics_list, + } + + +def _auroc_summary_metrics_spec( + probs: dict[str, np.ndarray], + reals: np.ndarray | dict[str, np.ndarray], + evaluation_metadata: Mapping[str, _EvaluationMetadata], +) -> _SummaryMetricsSpec: + """Build canonical evaluation-owned AUROC SummaryMetricsSpec v1.0.""" + reals_normalized = _validate_and_align_binary_inputs(probs, reals) + + evaluations_list: list[_EvaluationSpec] = [] + metrics_list: list[dict[str, Any]] = [] + + seen_populations: dict[str, str] = {} + pop_counter = 1 + + for index, (group, metadata) in enumerate(evaluation_metadata.items(), start=1): + pop_name = metadata.population + if pop_name not in seen_populations: + seen_populations[pop_name] = f"population-{pop_counter}" + pop_counter += 1 + pop_id = seen_populations[pop_name] + + eval_id = f"evaluation-{index}" + eval_spec: _EvaluationSpec = { + "id": eval_id, + "population": pop_id, + } + if metadata.model is not None: + eval_spec["model"] = metadata.model + eval_spec["label"] = metadata.model + else: + label = "Population" if pop_name == _SHARED_POPULATION else pop_name + eval_spec["label"] = label + evaluations_list.append(eval_spec) + + # Get raw arrays for this evaluation + probs_arr = np.asarray(probs[group]) + if isinstance(reals_normalized, dict): + reals_arr = np.asarray(reals_normalized[group]) + else: + reals_arr = np.asarray(reals_normalized) + + auroc_est = _compute_auroc_proc_compatible(reals_arr, probs_arr) + + metric_item: _AurocMetric = { + "metric": "auroc", + "owner": { + "type": "evaluation", + "evaluationId": eval_id, + }, + "estimate": auroc_est, + } + metrics_list.append(cast(dict[str, Any], metric_item)) + + return { + "schemaVersion": "1.0", + "type": "summary_metrics", + "title": "AUROC", + "evaluations": evaluations_list, + "populations": [], + "metrics": metrics_list, + } diff --git a/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM b/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM index abc0d7db..81b9d552 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM +++ b/src/rtichoke/_vendor/rtichoke_viz/VENDORED_FROM @@ -1,5 +1,5 @@ repository=https://github.com/uriahf/rtichoke_viz -release=v0.10.0 -source_commit=b65b903e9456b5eb323dbe0bac823e9ff0c1bd01 -archive=rtichoke-viz-0.10.0.tar.gz -sha256=030a9e58c6367147b4904d73cf641d8ef3990d95e70b66022a99d6c8d37b3233 +release=v0.14.0 +source_commit=ec3a382656ce5b1c735175d5a8fd1ceb4f153eaf +archive=rtichoke-viz-0.14.0.tar.gz +sha256=b7c30d12db4b3f8008ef035bd4f7e04c09d599ced5bc8606a4a4141c5cdf48ee diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.10.0.tar.gz b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.10.0.tar.gz deleted file mode 100644 index 87d00a4c..00000000 Binary files a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.10.0.tar.gz and /dev/null differ diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.14.0.tar.gz b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.14.0.tar.gz new file mode 100644 index 00000000..0754908c Binary files /dev/null and b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-0.14.0.tar.gz differ diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json new file mode 100644 index 00000000..c763e815 --- /dev/null +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-report.schema.json @@ -0,0 +1,14989 @@ +{ + "$id": "https://rtichoke.dev/schema/viz/report.json", + "title": "rtichoke report specification", + "anyOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "components" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "report", + "type": "string" + }, + "title": { + "type": "string" + }, + "components": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "spec" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "spec": { + "anyOf": [ + { + "$id": "https://rtichoke.dev/schema/viz/2.0.json", + "title": "rtichoke visualization specification v2", + "anyOf": [ + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "roc", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "specificity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "specificity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "false_positive_rate", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "calibration", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "method": { + "const": "discrete", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "type": "number" + }, + "method": { + "const": "smooth", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + }, + "distribution": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "midpoint", + "count", + "binWidth" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "midpoint": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "count": { + "minimum": 0, + "type": "number" + }, + "binWidth": { + "exclusiveMinimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "predicted", + "type": "string" + }, + "y": { + "const": "observed", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "precision_recall", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "ppv" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "ppv": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "sensitivity", + "type": "string" + }, + "y": { + "const": "ppv", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "gains", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "sensitivity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "lift", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "lift" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "lift": { + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "lift", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "decision_curve", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "netBenefit" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "netBenefit": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "netBenefit", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "interventions_avoided", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "interventionsAvoided" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "interventionsAvoided": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "interventionsAvoided", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + } + ] + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "metrics", + "rows" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "type": { + "const": "performance_table", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "label": { + "type": "string" + } + } + } + }, + "rows": { + "type": "array", + "items": { + "type": "object", + "required": [ + "evaluationId", + "operatingPoint", + "values" + ], + "properties": { + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "operatingPoint": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "probability_threshold", + "type": "string" + }, + "value": { + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "ppcr", + "type": "string" + }, + "value": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + ] + }, + "context": { + "type": "object", + "properties": { + "censoringHeuristic": { + "type": "string" + }, + "competingEventHeuristic": { + "type": "string" + } + } + }, + "values": { + "type": "array", + "items": { + "type": "object", + "required": [ + "metricId", + "estimate" + ], + "properties": { + "metricId": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + } + } + } + } + } + }, + { + "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", + "title": "rtichoke summary metrics specification", + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "populations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "auroc", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + ] + } + } + } + } + ] + } + } + } + } + } + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "sections" + ], + "properties": { + "schemaVersion": { + "const": "1.1", + "type": "string" + }, + "type": { + "const": "report", + "type": "string" + }, + "title": { + "type": "string" + }, + "sections": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "title", + "items" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "items": { + "minItems": 1, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "id", + "spec" + ], + "properties": { + "type": { + "const": "component", + "type": "string" + }, + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "spec": { + "anyOf": [ + { + "$id": "https://rtichoke.dev/schema/viz/2.0.json", + "title": "rtichoke visualization specification v2", + "anyOf": [ + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "roc", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "specificity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "specificity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "false_positive_rate", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "calibration", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "method": { + "const": "discrete", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "type": "number" + }, + "method": { + "const": "smooth", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + }, + "distribution": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "midpoint", + "count", + "binWidth" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "midpoint": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "count": { + "minimum": 0, + "type": "number" + }, + "binWidth": { + "exclusiveMinimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "predicted", + "type": "string" + }, + "y": { + "const": "observed", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "precision_recall", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "ppv" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "ppv": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "sensitivity", + "type": "string" + }, + "y": { + "const": "ppv", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "gains", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "sensitivity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "lift", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "lift" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "lift": { + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "lift", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "decision_curve", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "netBenefit" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "netBenefit": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "netBenefit", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "interventions_avoided", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "interventionsAvoided" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "interventionsAvoided": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "interventionsAvoided", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + } + ] + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "metrics", + "rows" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "type": { + "const": "performance_table", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "label": { + "type": "string" + } + } + } + }, + "rows": { + "type": "array", + "items": { + "type": "object", + "required": [ + "evaluationId", + "operatingPoint", + "values" + ], + "properties": { + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "operatingPoint": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "probability_threshold", + "type": "string" + }, + "value": { + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "ppcr", + "type": "string" + }, + "value": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + ] + }, + "context": { + "type": "object", + "properties": { + "censoringHeuristic": { + "type": "string" + }, + "competingEventHeuristic": { + "type": "string" + } + } + }, + "values": { + "type": "array", + "items": { + "type": "object", + "required": [ + "metricId", + "estimate" + ], + "properties": { + "metricId": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + } + } + } + } + } + }, + { + "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", + "title": "rtichoke summary metrics specification", + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "populations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "auroc", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + ] + } + } + } + } + ] + } + } + }, + { + "type": "object", + "required": [ + "type", + "id", + "title", + "components" + ], + "properties": { + "type": { + "const": "group", + "type": "string" + }, + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "components": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "required": [ + "type", + "id", + "spec" + ], + "properties": { + "type": { + "const": "component", + "type": "string" + }, + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "spec": { + "anyOf": [ + { + "$id": "https://rtichoke.dev/schema/viz/2.0.json", + "title": "rtichoke visualization specification v2", + "anyOf": [ + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "roc", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "specificity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "specificity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "false_positive_rate", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "calibration", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "method": { + "const": "discrete", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "type": "number" + }, + "method": { + "const": "smooth", + "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + }, + "distribution": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "midpoint", + "count", + "binWidth" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "midpoint": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "count": { + "minimum": 0, + "type": "number" + }, + "binWidth": { + "exclusiveMinimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "predicted", + "type": "string" + }, + "y": { + "const": "observed", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "precision_recall", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "sensitivity", + "ppv" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "ppv": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "sensitivity", + "type": "string" + }, + "y": { + "const": "ppv", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "gains", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "sensitivity" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "sensitivity": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "sensitivity", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "data", + "x", + "y" + ], + "properties": { + "type": { + "const": "lift", + "type": "string" + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "cutoff", + "ppcr", + "lift" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "cutoff": { + "type": "number" + }, + "ppcr": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "lift": { + "type": "number" + } + } + } + }, + "x": { + "const": "ppcr", + "type": "string" + }, + "y": { + "const": "lift", + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "decision_curve", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "netBenefit" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "netBenefit": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "netBenefit", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + }, + { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "schemaVersion", + "evaluations", + "series", + "xAxis", + "yAxis" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "series": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + } + }, + "xAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "yAxis": { + "type": "object", + "required": [ + "label" + ], + "properties": { + "label": { + "type": "string" + }, + "domain": { + "type": "array", + "items": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "additionalItems": false, + "minItems": 2, + "maxItems": 2 + } + } + }, + "references": { + "type": "array", + "items": { + "anyOf": [ + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope" + ], + "properties": { + "scope": { + "const": "global", + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population" + ], + "properties": { + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + } + ] + }, + { + "allOf": [ + { + "anyOf": [ + { + "type": "object", + "required": [ + "type" + ], + "properties": { + "type": { + "const": "identity", + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "vertical", + "type": "string" + }, + "value": { + "type": "number" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + } + } + } + ] + }, + { + "type": "object", + "required": [ + "scope", + "population", + "horizon" + ], + "properties": { + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + }, + { + "type": "object", + "required": [ + "type", + "evaluations", + "series", + "data", + "x", + "y", + "references" + ], + "properties": { + "type": { + "const": "interventions_avoided", + "type": "string" + }, + "evaluations": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "series": { + "minItems": 1, + "type": "array", + "items": { + "type": "object", + "allOf": [ + { + "type": "object", + "required": [ + "id", + "evaluationId", + "display" + ], + "properties": { + "id": { + "type": "string" + }, + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "display": { + "type": "object", + "required": [ + "label", + "group", + "role" + ], + "properties": { + "label": { + "type": "string" + }, + "group": { + "type": "string" + }, + "role": { + "anyOf": [ + { + "const": "model", + "type": "string" + }, + { + "const": "population", + "type": "string" + }, + { + "const": "evaluation", + "type": "string" + }, + { + "const": "context", + "type": "string" + } + ] + } + } + } + } + }, + { + "type": "object", + "required": [ + "id", + "evaluationId" + ], + "properties": { + "id": { + "pattern": "^series-[1-9][0-9]*$", + "type": "string" + }, + "evaluationId": { + "pattern": "^evaluation-[1-9][0-9]*$", + "type": "string" + } + } + } + ] + } + }, + "data": { + "type": "array", + "items": { + "type": "object", + "required": [ + "seriesId", + "threshold", + "interventionsAvoided" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "threshold": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "interventionsAvoided": { + "type": "number" + } + } + } + }, + "x": { + "const": "threshold", + "type": "string" + }, + "y": { + "const": "interventionsAvoided", + "type": "string" + }, + "references": { + "minItems": 2, + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value", + "scope", + "benchmark" + ], + "properties": { + "type": { + "const": "horizontal", + "type": "string" + }, + "value": { + "const": 0, + "type": "number" + }, + "label": { + "type": "string" + }, + "scope": { + "const": "global", + "type": "string" + }, + "benchmark": { + "const": "treat_all", + "type": "string" + } + } + }, + { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population", + "type": "string" + }, + "population": { + "type": "string" + } + } + }, + { + "type": "object", + "required": [ + "type", + "points", + "benchmark", + "scope", + "population", + "horizon" + ], + "properties": { + "type": { + "const": "path", + "type": "string" + }, + "points": { + "minItems": 2, + "type": "array", + "items": { + "type": "object", + "required": [ + "x", + "y" + ], + "properties": { + "x": { + "type": "number" + }, + "y": { + "type": "number" + } + } + } + }, + "label": { + "type": "string" + }, + "benchmark": { + "const": "treat_none", + "type": "string" + }, + "scope": { + "const": "population_horizon", + "type": "string" + }, + "population": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + } + } + } + ] + } + ] + } + } + } + } + ] + } + ] + }, + { + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "metrics", + "rows" + ], + "properties": { + "schemaVersion": { + "const": "2.0", + "type": "string" + }, + "type": { + "const": "performance_table", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "label": { + "type": "string" + } + } + } + }, + "rows": { + "type": "array", + "items": { + "type": "object", + "required": [ + "evaluationId", + "operatingPoint", + "values" + ], + "properties": { + "evaluationId": { + "type": "string" + }, + "horizon": { + "minimum": 0, + "type": "number" + }, + "operatingPoint": { + "anyOf": [ + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "probability_threshold", + "type": "string" + }, + "value": { + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "ppcr", + "type": "string" + }, + "value": { + "minimum": 0, + "maximum": 1, + "type": "number" + } + } + } + ] + }, + "context": { + "type": "object", + "properties": { + "censoringHeuristic": { + "type": "string" + }, + "competingEventHeuristic": { + "type": "string" + } + } + }, + "values": { + "type": "array", + "items": { + "type": "object", + "required": [ + "metricId", + "estimate" + ], + "properties": { + "metricId": { + "anyOf": [ + { + "const": "true_positives", + "type": "string" + }, + { + "const": "true_negatives", + "type": "string" + }, + { + "const": "false_positives", + "type": "string" + }, + { + "const": "false_negatives", + "type": "string" + }, + { + "const": "sensitivity", + "type": "string" + }, + { + "const": "specificity", + "type": "string" + }, + { + "const": "false_positive_rate", + "type": "string" + }, + { + "const": "ppv", + "type": "string" + }, + { + "const": "npv", + "type": "string" + }, + { + "const": "lift", + "type": "string" + }, + { + "const": "predicted_positives", + "type": "string" + }, + { + "const": "ppcr", + "type": "string" + }, + { + "const": "net_benefit", + "type": "string" + }, + { + "const": "net_benefit_interventions_avoided", + "type": "string" + } + ] + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "lower": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + }, + "upper": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + } + } + } + } + } + }, + { + "$id": "https://rtichoke.dev/schema/viz/summary-metrics.json", + "title": "rtichoke summary metrics specification", + "type": "object", + "required": [ + "schemaVersion", + "type", + "evaluations", + "populations", + "metrics" + ], + "properties": { + "schemaVersion": { + "const": "1.0", + "type": "string" + }, + "type": { + "const": "summary_metrics", + "type": "string" + }, + "title": { + "type": "string" + }, + "evaluations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "population" + ], + "properties": { + "id": { + "type": "string" + }, + "model": { + "type": "string" + }, + "population": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "populations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "label" + ], + "properties": { + "id": { + "type": "string" + }, + "label": { + "type": "string" + } + } + } + }, + "metrics": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "auroc", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "evaluationId" + ], + "properties": { + "type": { + "const": "evaluation", + "type": "string" + }, + "evaluationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + }, + { + "type": "object", + "required": [ + "metric", + "owner", + "estimate" + ], + "properties": { + "metric": { + "const": "prevalence", + "type": "string" + }, + "owner": { + "type": "object", + "required": [ + "type", + "populationId" + ], + "properties": { + "type": { + "const": "population", + "type": "string" + }, + "populationId": { + "type": "string" + } + } + }, + "estimate": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ] + } + } + } + ] + } + } + } + } + ] + } + } + } + } + } + } + ] + } + } + } + } + } + } + } + ] +} diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-v2.schema.json b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-v2.schema.json index f5113042..ba024b43 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-v2.schema.json +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz-v2.schema.json @@ -1071,48 +1071,78 @@ "data": { "type": "array", "items": { - "type": "object", - "required": [ - "seriesId", - "predicted", - "observed", - "method" - ], - "properties": { - "seriesId": { - "type": "string" - }, - "predicted": { - "minimum": 0, - "maximum": 1, - "type": "number" - }, - "observed": { - "minimum": 0, - "maximum": 1, - "type": "number" - }, - "method": { - "anyOf": [ - { + "anyOf": [ + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "method": { "const": "discrete", "type": "string" }, - { + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" + } + } + }, + { + "type": "object", + "required": [ + "seriesId", + "predicted", + "observed", + "method" + ], + "properties": { + "seriesId": { + "type": "string" + }, + "predicted": { + "minimum": 0, + "maximum": 1, + "type": "number" + }, + "observed": { + "type": "number" + }, + "method": { "const": "smooth", "type": "string" + }, + "events": { + "minimum": 0, + "type": "number" + }, + "total": { + "minimum": 0, + "type": "number" } - ] - }, - "events": { - "minimum": 0, - "type": "number" - }, - "total": { - "minimum": 0, - "type": "number" + } } - } + ] } }, "distribution": { diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.css b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.css index 422303db..d57025dc 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.css +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.css @@ -11,13 +11,211 @@ display: flex; flex-direction: column; gap: 2.5rem; + max-width: 1200px; + margin: 0 auto; + box-sizing: border-box; + font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; + color: #1a1a1a; +} + +.rtichoke-report__header { + border-bottom: 2px solid #e5e7eb; + padding-bottom: 1rem; +} + +.rtichoke-report__title { + margin: 0; + font-size: 2rem; + font-weight: 700; + color: #111827; +} + +.rtichoke-report__nav { + background-color: #f9fafb; + border: 1px solid #e5e7eb; + border-radius: 6px; + padding: 1rem 1.5rem; +} + +.rtichoke-report__nav-list, +.rtichoke-report__nav-sublist { + list-style-type: none; + margin: 0; + padding: 0; +} + +.rtichoke-report__nav-list { + display: flex; + flex-wrap: wrap; + gap: 1rem 1.5rem; +} + +.rtichoke-report__nav-item { + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.rtichoke-report__nav-link { + color: #2563eb; + text-decoration: none; + font-weight: 600; + font-size: 0.95rem; +} + +.rtichoke-report__nav-link:hover, +.rtichoke-report__nav-link:focus { + text-decoration: underline; +} + +.rtichoke-report__nav-sublist { + padding-left: 0.75rem; + border-left: 2px solid #d1d5db; + margin-top: 0.25rem; + display: flex; + flex-direction: column; + gap: 0.25rem; +} + +.rtichoke-report__nav-subitem .rtichoke-report__nav-link { + font-weight: 400; + font-size: 0.875rem; + color: #4b5563; +} + +.rtichoke-report__section { + display: flex; + flex-direction: column; + gap: 1.75rem; + padding-top: 0.5rem; +} + +.rtichoke-report__section-title { + margin: 0; + font-size: 1.5rem; + font-weight: 600; + color: #111827; + border-bottom: 1px solid #e5e7eb; + padding-bottom: 0.5rem; +} + +.rtichoke-report__group { + display: flex; + flex-direction: column; + gap: 1.25rem; + padding-left: 0.5rem; +} + +.rtichoke-report__group-title { + margin: 0; + font-size: 1.25rem; + font-weight: 600; + color: #374151; +} + +.rtichoke-report__group-title--tab-panel { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} + +.rtichoke-report__component { + display: flex; + flex-direction: column; + gap: 0.75rem; } -.rtichoke-report__title, .rtichoke-report__component-title { - margin: 0 0 1rem; + margin: 0; + font-size: 1.1rem; + font-weight: 600; + color: #4b5563; } .rtichoke-report__component-content { overflow-x: auto; } + +.rtichoke-report__tablist { + display: flex; + flex-wrap: wrap; + gap: 0.25rem; + border-bottom: 2px solid #e5e7eb; + margin-bottom: 0.5rem; +} + +.rtichoke-report__tab { + padding: 0.5rem 1rem; + font-family: inherit; + font-size: 0.9rem; + font-weight: 500; + color: #4b5563; + background-color: transparent; + border: none; + border-bottom: 2px solid transparent; + margin-bottom: -2px; + cursor: pointer; + border-radius: 4px 4px 0 0; + transition: color 0.15s ease-in-out, border-color 0.15s ease-in-out, background-color 0.15s ease-in-out; +} + +.rtichoke-report__tab:hover { + color: #111827; + background-color: #f3f4f6; +} + +.rtichoke-report__tab[aria-selected="true"] { + color: #2563eb; + border-bottom-color: #2563eb; + font-weight: 600; + background-color: #eff6ff; +} + +.rtichoke-report__tab:focus-visible { + outline: 2px solid #2563eb; + outline-offset: -2px; +} + +.rtichoke-report__tabpanel[hidden] { + display: none !important; +} + +.rtichoke-report__tabpanel { + display: flex; + flex-direction: column; + gap: 0.75rem; + outline: none; +} + +@media (max-width: 640px) { + .rtichoke-report__nav-list { + flex-direction: column; + gap: 0.75rem; + } + + .rtichoke-report__tablist { + flex-direction: column; + border-bottom: none; + gap: 0.25rem; + } + + .rtichoke-report__tab { + width: 100%; + text-align: left; + border-bottom: none; + border-left: 3px solid transparent; + margin-bottom: 0; + border-radius: 4px; + } + + .rtichoke-report__tab[aria-selected="true"] { + border-bottom-color: transparent; + border-left-color: #2563eb; + } +} diff --git a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js index 915f87d5..7abfda3b 100644 --- a/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js +++ b/src/rtichoke/_vendor/rtichoke_viz/rtichoke-viz.js @@ -1053,7 +1053,7 @@ function Literal(value, options) { } // node_modules/@sinclair/typebox/build/esm/type/boolean/boolean.mjs -function Boolean(options) { +function Boolean2(options) { return CreateType({ [Kind]: "Boolean", type: "boolean" }, options); } @@ -1075,7 +1075,7 @@ function String2(options) { // node_modules/@sinclair/typebox/build/esm/type/template-literal/syntax.mjs function* FromUnion(syntax) { const trim = syntax.trim().replace(/"|'/g, ""); - return trim === "boolean" ? yield Boolean() : trim === "number" ? yield Number2() : trim === "bigint" ? yield BigInt2() : trim === "string" ? yield String2() : yield (() => { + return trim === "boolean" ? yield Boolean2() : trim === "number" ? yield Number2() : trim === "bigint" ? yield BigInt2() : trim === "string" ? yield String2() : yield (() => { const literals = trim.split("|").map((literal) => Literal(literal.trim())); return literals.length === 0 ? Never() : literals.length === 1 ? literals[0] : UnionEvaluated(literals); })(); @@ -2691,7 +2691,7 @@ __export(type_exports3, { AsyncIterator: () => AsyncIterator, Awaited: () => Awaited, BigInt: () => BigInt2, - Boolean: () => Boolean, + Boolean: () => Boolean2, Capitalize: () => Capitalize, Composite: () => Composite, Const: () => Const, @@ -2917,14 +2917,26 @@ var BaseChartV2SpecSchema = Type.Object({ }); // src/spec/v2/calibration.ts -var CalibrationV2DatumSchema = Type.Object({ +var DiscreteCalibrationV2DatumSchema = Type.Object({ seriesId: Type.String(), predicted: Type.Number({ minimum: 0, maximum: 1 }), observed: Type.Number({ minimum: 0, maximum: 1 }), - method: Type.Union([Type.Literal("discrete"), Type.Literal("smooth")]), + method: Type.Literal("discrete"), + events: Type.Optional(Type.Number({ minimum: 0 })), + total: Type.Optional(Type.Number({ minimum: 0 })) +}); +var SmoothCalibrationV2DatumSchema = Type.Object({ + seriesId: Type.String(), + predicted: Type.Number({ minimum: 0, maximum: 1 }), + observed: Type.Number(), + method: Type.Literal("smooth"), events: Type.Optional(Type.Number({ minimum: 0 })), total: Type.Optional(Type.Number({ minimum: 0 })) }); +var CalibrationV2DatumSchema = Type.Union([ + DiscreteCalibrationV2DatumSchema, + SmoothCalibrationV2DatumSchema +]); var CalibrationV2DistributionDatumSchema = Type.Object({ seriesId: Type.String(), midpoint: Type.Number({ minimum: 0, maximum: 1 }), @@ -3210,27 +3222,130 @@ var PerformanceTableSpecSchema = Type.Object({ rows: Type.Array(PerformanceTableRowSchema) }); +// src/spec/v2/summary-metrics.ts +var PopulationSummaryOwnerSpecSchema = Type.Object({ + id: Type.String(), + label: Type.String() +}); +var AUROCSummaryMetricSchema = Type.Object({ + metric: Type.Literal("auroc"), + owner: Type.Object({ + type: Type.Literal("evaluation"), + evaluationId: Type.String() + }), + estimate: Type.Union([Type.Number(), Type.Null()]) +}); +var PrevalenceSummaryMetricSchema = Type.Object({ + metric: Type.Literal("prevalence"), + owner: Type.Object({ + type: Type.Literal("population"), + populationId: Type.String() + }), + estimate: Type.Union([Type.Number(), Type.Null()]) +}); +var SummaryMetricSchema = Type.Union([ + AUROCSummaryMetricSchema, + PrevalenceSummaryMetricSchema +]); +var SummaryMetricsSpecSchema = Type.Object( + { + schemaVersion: Type.Literal("1.0"), + type: Type.Literal("summary_metrics"), + title: Type.Optional(Type.String()), + evaluations: Type.Array(EvaluationSpecSchema), + populations: Type.Array(PopulationSummaryOwnerSpecSchema), + metrics: Type.Array(SummaryMetricSchema) + }, + { + $id: "https://rtichoke.dev/schema/viz/summary-metrics.json", + title: "rtichoke summary metrics specification" + } +); + // src/spec/report.ts +var StandaloneCanonicalSpecSchema = Type.Union([ + RtichokeChartSpecV2Schema, + PerformanceTableSpecSchema, + SummaryMetricsSpecSchema +]); var ReportComponentSchema = Type.Object({ id: Type.String(), title: Type.Optional(Type.String()), - spec: Type.Union([RtichokeChartSpecV2Schema, PerformanceTableSpecSchema]) + spec: StandaloneCanonicalSpecSchema }); -var ReportSpecSchema = Type.Object({ +var ReportSpecV1_0Schema = Type.Object({ schemaVersion: Type.Literal("1.0"), type: Type.Literal("report"), title: Type.Optional(Type.String()), components: Type.Array(ReportComponentSchema, { minItems: 1 }) }); +var ReportComponentV1_1Schema = Type.Object({ + type: Type.Literal("component"), + id: Type.String(), + title: Type.Optional(Type.String()), + spec: StandaloneCanonicalSpecSchema +}); +var ReportGroupSchema = Type.Object({ + type: Type.Literal("group"), + id: Type.String(), + title: Type.String(), + components: Type.Array(ReportComponentV1_1Schema, { minItems: 1 }) +}); +var ReportSectionSchema = Type.Object({ + id: Type.String(), + title: Type.String(), + items: Type.Array( + Type.Union([ReportComponentV1_1Schema, ReportGroupSchema]), + { minItems: 1 } + ) +}); +var ReportSpecV1_1Schema = Type.Object({ + schemaVersion: Type.Literal("1.1"), + type: Type.Literal("report"), + title: Type.Optional(Type.String()), + sections: Type.Array(ReportSectionSchema, { minItems: 1 }) +}); +var ReportSpecSchema = Type.Union( + [ReportSpecV1_0Schema, ReportSpecV1_1Schema], + { + $id: "https://rtichoke.dev/schema/viz/report.json", + title: "rtichoke report specification" + } +); // src/spec/validate-report.ts function assertReportReferentialIntegrity(spec) { const componentIds = /* @__PURE__ */ new Set(); - for (const component of spec.components) { + const assertUniqueComponent = (component) => { if (componentIds.has(component.id)) { throw new Error(`duplicate component id: ${component.id}`); } componentIds.add(component.id); + }; + if (spec.schemaVersion === "1.0") { + for (const component of spec.components) assertUniqueComponent(component); + return; + } + const sectionIds = /* @__PURE__ */ new Set(); + const groupIds = /* @__PURE__ */ new Set(); + for (const section of spec.sections) { + if (sectionIds.has(section.id)) { + throw new Error(`duplicate section id: ${section.id}`); + } + sectionIds.add(section.id); + for (const item of section.items) { + if (item.type === "component") { + assertUniqueComponent(item); + continue; + } + if (groupIds.has(item.id)) { + throw new Error(`duplicate group id: ${item.id}`); + } + groupIds.add(item.id); + for (const component of item.components) { + assertUniqueComponent(component); + } + } } } @@ -3262,6 +3377,53 @@ function assertPerformanceTableReferentialIntegrity(spec) { } } +// src/spec/v2/validate-summary-metrics.ts +function assertSummaryMetricsReferentialIntegrity(spec) { + const evaluationIds = /* @__PURE__ */ new Set(); + for (const evaluation of spec.evaluations) { + if (evaluationIds.has(evaluation.id)) { + throw new Error(`duplicate evaluation id: ${evaluation.id}`); + } + evaluationIds.add(evaluation.id); + } + const populationIds = /* @__PURE__ */ new Set(); + for (const population of spec.populations) { + if (populationIds.has(population.id)) { + throw new Error(`duplicate population id: ${population.id}`); + } + populationIds.add(population.id); + } + const seenMetrics = /* @__PURE__ */ new Set(); + for (const item of spec.metrics) { + if (item.estimate !== null && !Number.isFinite(item.estimate)) { + throw new Error(`non-finite metric estimate: ${item.estimate}`); + } + if (item.metric === "auroc") { + if (!evaluationIds.has(item.owner.evaluationId)) { + throw new Error(`unknown evaluation id: ${item.owner.evaluationId}`); + } + const key = `auroc:${item.owner.evaluationId}`; + if (seenMetrics.has(key)) { + throw new Error( + `duplicate metric ownership: auroc for evaluation ${item.owner.evaluationId}` + ); + } + seenMetrics.add(key); + } else if (item.metric === "prevalence") { + if (!populationIds.has(item.owner.populationId)) { + throw new Error(`unknown population id: ${item.owner.populationId}`); + } + const key = `prevalence:${item.owner.populationId}`; + if (seenMetrics.has(key)) { + throw new Error( + `duplicate metric ownership: prevalence for population ${item.owner.populationId}` + ); + } + seenMetrics.add(key); + } + } +} + // src/spec/v2/validate.ts function assertV2ReferentialIntegrity(spec) { const evaluationIds = new Set(spec.evaluations.map((evaluation) => evaluation.id)); @@ -3541,6 +3703,9 @@ function rocV2SpecFromRtichokePython(rows, context) { } function calibrationV2SpecFromRtichokeRows(rows, method, context, distributionRows) { const byGroup = identities(rows.map((row) => row.reference_group), context); + const observedValues = rows.map((row) => row.y).filter(Number.isFinite); + const minY = Math.min(0, ...observedValues); + const maxY = Math.max(1, ...observedValues); return { schemaVersion: "2.0", type: "calibration", @@ -3569,7 +3734,7 @@ function calibrationV2SpecFromRtichokeRows(rows, method, context, distributionRo x: "predicted", y: "observed", xAxis: { label: "Predicted probability", domain: [0, 1] }, - yAxis: { label: "Observed probability", domain: [0, 1] }, + yAxis: { label: "Observed probability", domain: [minY, maxY] }, references: [{ type: "identity", scope: "global", label: "Perfectly Calibrated" }] }; } @@ -19364,6 +19529,11 @@ function renderCalibrationV2(spec, options = {}) { ); const hasDistribution = (spec.distribution?.length ?? 0) > 0; const mainHeight = hasDistribution ? Math.round(theme.height * 0.8) : theme.height; + const observedValues = data.map((datum2) => datum2.observed).filter(Number.isFinite); + const yDomain = spec.yAxis.domain ?? [ + Math.min(0, ...observedValues), + Math.max(1, ...observedValues) + ]; const calibration = themedPlot( { ...basePlotOptions(resolved, spec), @@ -19374,7 +19544,7 @@ function renderCalibrationV2(spec, options = {}) { axis: null, label: null } : axisOptions2(theme, spec.xAxis.label, spec.xAxis.domain), - y: axisOptions2(theme, spec.yAxis.label, spec.yAxis.domain), + y: axisOptions2(theme, spec.yAxis.label, yDomain), marks: finishMarks(marks2, theme) }, theme @@ -19720,6 +19890,89 @@ function renderPerformanceTable(spec, document2 = globalThis.document) { return root2; } +// src/render/summary-metrics.ts +var MISSING2 = "\u2014"; +function cell2(document2, text2, className) { + const element = document2.createElement("td"); + element.textContent = text2; + if (className) element.className = className; + return element; +} +function header2(document2, text2) { + const element = document2.createElement("th"); + element.scope = "col"; + element.textContent = text2; + return element; +} +function formatEstimate(value) { + if (value === null) return MISSING2; + return value.toFixed(2); +} +function renderSummaryMetrics(spec, document2 = globalThis.document) { + assertSummaryMetricsReferentialIntegrity(spec); + const root2 = document2.createElement("div"); + root2.className = "rtichoke-summary-metrics"; + if (spec.title) { + const title = document2.createElement("div"); + title.className = "rtichoke-summary-metrics__title"; + title.textContent = spec.title; + root2.append(title); + } + const table = document2.createElement("table"); + table.className = "rtichoke-summary-metrics__table"; + const head = document2.createElement("thead"); + const headRow = document2.createElement("tr"); + for (const label of ["Owner", "Metric", "Estimate"]) { + headRow.append(header2(document2, label)); + } + head.append(headRow); + table.append(head); + const evaluations = new Map( + spec.evaluations.map((evaluation) => [evaluation.id, evaluation]) + ); + const populations = new Map( + spec.populations.map((population) => [population.id, population]) + ); + const body = document2.createElement("tbody"); + for (const item of spec.metrics) { + const tr = document2.createElement("tr"); + let ownerLabel = ""; + let metricLabel = ""; + if (item.metric === "auroc") { + const evaluation = evaluations.get(item.owner.evaluationId); + ownerLabel = evaluation.label ?? evaluation.model ?? evaluation.population ?? evaluation.id; + metricLabel = "AUROC"; + tr.dataset.metric = "auroc"; + tr.dataset.evaluationId = item.owner.evaluationId; + } else if (item.metric === "prevalence") { + const population = populations.get(item.owner.populationId); + ownerLabel = population.label; + metricLabel = "Prevalence"; + tr.dataset.metric = "prevalence"; + tr.dataset.populationId = item.owner.populationId; + } + tr.append( + cell2(document2, ownerLabel, "rtichoke-summary-metrics__owner"), + cell2(document2, metricLabel, "rtichoke-summary-metrics__metric") + ); + const estimateCell = cell2( + document2, + formatEstimate(item.estimate), + "rtichoke-summary-metrics__estimate" + ); + if (item.estimate === null) { + estimateCell.dataset.unavailable = "true"; + } else { + estimateCell.dataset.estimate = String(item.estimate); + } + tr.append(estimateCell); + body.append(tr); + } + table.append(body); + root2.append(table); + return root2; +} + // node_modules/@sinclair/typebox/build/esm/errors/function.mjs function DefaultErrorFunction(error) { switch (error.errorType) { @@ -23055,8 +23308,80 @@ __export(value_exports2, { }); // src/render/report.ts -function renderReport(spec) { - if (!value_exports2.Check(ReportSpecSchema, spec)) throw new Error("Invalid ReportSpec"); +function renderStandaloneComponentContent(spec) { + switch (spec.type) { + case "summary_metrics": + return renderSummaryMetrics(spec); + case "performance_table": + return renderPerformanceTable(spec); + case "roc": + return renderRocV2(spec); + case "calibration": + return renderCalibrationV2(spec); + case "precision_recall": + return renderPrecisionRecallV2(spec); + case "gains": + return renderGainsV2(spec); + case "lift": + return renderLiftV2(spec); + case "decision_curve": + return renderDecisionCurveV2(spec); + case "interventions_avoided": + return renderInterventionsAvoidedV2(spec); + } +} +function generateUniqueDomId(rawId, used) { + let base = rawId.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "id"; + if (!/^[a-z]/i.test(base)) { + base = `id-${base}`; + } + let candidate = base; + let counter = 1; + while (used.has(candidate)) { + candidate = `${base}-${counter}`; + counter++; + } + used.add(candidate); + return candidate; +} +function wireTabInteraction(tabs, panels) { + const activateTab = (index2, setFocus = true) => { + tabs.forEach((tab, tabIndex) => { + const isSelected = tabIndex === index2; + tab.setAttribute("aria-selected", isSelected ? "true" : "false"); + tab.tabIndex = isSelected ? 0 : -1; + panels[tabIndex].hidden = !isSelected; + }); + if (setFocus) { + tabs[index2].focus(); + } + }; + tabs.forEach((tab, index2) => { + tab.addEventListener("click", () => { + activateTab(index2, false); + }); + tab.addEventListener("keydown", (event) => { + let targetIndex = null; + if (event.key === "ArrowRight") { + targetIndex = (index2 + 1) % tabs.length; + } else if (event.key === "ArrowLeft") { + targetIndex = (index2 - 1 + tabs.length) % tabs.length; + } else if (event.key === "Home") { + targetIndex = 0; + } else if (event.key === "End") { + targetIndex = tabs.length - 1; + } + if (targetIndex !== null) { + event.preventDefault(); + activateTab(targetIndex, true); + } + }); + }); +} +function renderReportV1_0(spec) { + if (!value_exports2.Check(ReportSpecV1_0Schema, spec)) { + throw new Error("Invalid ReportSpec"); + } assertReportReferentialIntegrity(spec); const root2 = document.createElement("div"); root2.className = "rtichoke-report"; @@ -23078,37 +23403,286 @@ function renderReport(spec) { } const content = document.createElement("div"); content.className = "rtichoke-report__component-content"; - switch (component.spec.type) { - case "performance_table": - content.append(renderPerformanceTable(component.spec)); - break; - case "roc": - content.append(renderRocV2(component.spec)); - break; - case "calibration": - content.append(renderCalibrationV2(component.spec)); - break; - case "precision_recall": - content.append(renderPrecisionRecallV2(component.spec)); - break; - case "gains": - content.append(renderGainsV2(component.spec)); - break; - case "lift": - content.append(renderLiftV2(component.spec)); - break; - case "decision_curve": - content.append(renderDecisionCurveV2(component.spec)); - break; - case "interventions_avoided": - content.append(renderInterventionsAvoidedV2(component.spec)); - break; - } + content.append(renderStandaloneComponentContent(component.spec)); container.append(content); root2.append(container); } return root2; } +function renderReportV1_1(spec, options) { + if (!value_exports2.Check(ReportSpecV1_1Schema, spec)) { + throw new Error("Invalid ReportSpec"); + } + assertReportReferentialIntegrity(spec); + const hasTitle = Boolean(spec.title); + const usedDomIds = /* @__PURE__ */ new Set(); + const root2 = document.createElement("article"); + root2.className = "rtichoke-report"; + if (spec.title) { + const header3 = document.createElement("header"); + header3.className = "rtichoke-report__header"; + const title = document.createElement("h1"); + title.className = "rtichoke-report__title"; + title.textContent = spec.title; + header3.append(title); + root2.append(header3); + } + const navSections = []; + let totalNavigableTargets = 0; + for (const section of spec.sections) { + const secDomId = generateUniqueDomId(section.id, usedDomIds); + totalNavigableTargets++; + const navGroups = []; + for (const item of section.items) { + if (item.type === "group") { + const grpDomId = generateUniqueDomId(item.id, usedDomIds); + totalNavigableTargets++; + navGroups.push({ + rawId: item.id, + title: item.title, + domId: grpDomId + }); + } + } + navSections.push({ + rawId: section.id, + title: section.title, + domId: secDomId, + groups: navGroups + }); + } + if (totalNavigableTargets > 1) { + const nav = document.createElement("nav"); + nav.className = "rtichoke-report__nav"; + nav.setAttribute("aria-label", "Report sections"); + const navList = document.createElement("ul"); + navList.className = "rtichoke-report__nav-list"; + for (const sec of navSections) { + const navItem = document.createElement("li"); + navItem.className = "rtichoke-report__nav-item"; + const navLink = document.createElement("a"); + navLink.className = "rtichoke-report__nav-link"; + navLink.href = `#${sec.domId}`; + navLink.textContent = sec.title; + navItem.append(navLink); + if (sec.groups.length > 0) { + const subList = document.createElement("ul"); + subList.className = "rtichoke-report__nav-sublist"; + for (const grp of sec.groups) { + const subItem = document.createElement("li"); + subItem.className = "rtichoke-report__nav-subitem"; + const subLink = document.createElement("a"); + subLink.className = "rtichoke-report__nav-link"; + subLink.href = `#${grp.domId}`; + subLink.textContent = grp.title; + subItem.append(subLink); + subList.append(subItem); + } + navItem.append(subList); + } + navList.append(navItem); + } + nav.append(navList); + root2.append(nav); + } + const sectionHeadingTag = hasTitle ? "h2" : "h1"; + const groupHeadingTag = hasTitle ? "h3" : "h2"; + const directCompHeadingTag = hasTitle ? "h3" : "h2"; + const groupCompHeadingTag = hasTitle ? "h4" : "h3"; + const renderComponent = (comp, headingTag) => { + const container = document.createElement("section"); + container.className = "rtichoke-report__component"; + container.dataset.componentId = comp.id; + if (comp.title) { + const compHeadingDomId = generateUniqueDomId( + `heading-${comp.id}`, + usedDomIds + ); + const titleEl = document.createElement(headingTag); + titleEl.className = "rtichoke-report__component-title"; + titleEl.id = compHeadingDomId; + titleEl.textContent = comp.title; + container.setAttribute("aria-labelledby", compHeadingDomId); + container.append(titleEl); + } + const content = document.createElement("div"); + content.className = "rtichoke-report__component-content"; + content.append(renderStandaloneComponentContent(comp.spec)); + container.append(content); + return container; + }; + const renderGroup = (item, grpNav, sectionPanelTabId) => { + const grpSection = document.createElement("section"); + grpSection.className = "rtichoke-report__group"; + grpSection.id = grpNav.domId; + grpSection.dataset.groupId = item.id; + const grpHeadingDomId = generateUniqueDomId( + `heading-${item.id}`, + usedDomIds + ); + const grpHeading = document.createElement(groupHeadingTag); + grpHeading.className = "rtichoke-report__group-title"; + if (sectionPanelTabId) { + grpHeading.classList.add("rtichoke-report__group-title--tab-panel"); + } + grpHeading.id = grpHeadingDomId; + grpHeading.textContent = item.title; + grpSection.setAttribute( + "aria-labelledby", + sectionPanelTabId ?? grpHeadingDomId + ); + grpSection.append(grpHeading); + if (options.groupPresentation === "tabs") { + const tablist = document.createElement("div"); + tablist.className = "rtichoke-report__tablist"; + tablist.setAttribute("role", "tablist"); + tablist.setAttribute("aria-labelledby", grpHeadingDomId); + const tabs = []; + const panels = []; + item.components.forEach((comp, compIdx) => { + const tabDomId = generateUniqueDomId( + `tab-${item.id}-${comp.id}`, + usedDomIds + ); + const panelDomId = generateUniqueDomId( + `panel-${item.id}-${comp.id}`, + usedDomIds + ); + const tab = document.createElement("button"); + tab.className = "rtichoke-report__tab"; + tab.type = "button"; + tab.id = tabDomId; + tab.setAttribute("role", "tab"); + tab.setAttribute("aria-controls", panelDomId); + tab.setAttribute("aria-selected", compIdx === 0 ? "true" : "false"); + tab.tabIndex = compIdx === 0 ? 0 : -1; + tab.textContent = comp.title || comp.id; + const panel = document.createElement("section"); + panel.className = "rtichoke-report__component rtichoke-report__tabpanel"; + panel.id = panelDomId; + panel.setAttribute("role", "tabpanel"); + panel.setAttribute("aria-labelledby", tabDomId); + panel.tabIndex = 0; + panel.dataset.componentId = comp.id; + panel.hidden = compIdx !== 0; + const content = document.createElement("div"); + content.className = "rtichoke-report__component-content"; + content.append(renderStandaloneComponentContent(comp.spec)); + panel.append(content); + tabs.push(tab); + panels.push(panel); + tablist.append(tab); + }); + wireTabInteraction(tabs, panels); + grpSection.append(tablist, ...panels); + } else { + for (const comp of item.components) { + grpSection.append(renderComponent(comp, groupCompHeadingTag)); + } + } + return grpSection; + }; + for (let i = 0; i < spec.sections.length; i++) { + const sectionSpec = spec.sections[i]; + const secNav = navSections[i]; + const secSection = document.createElement("section"); + secSection.className = "rtichoke-report__section"; + secSection.id = secNav.domId; + secSection.dataset.sectionId = sectionSpec.id; + const secHeadingDomId = generateUniqueDomId( + `heading-${sectionSpec.id}`, + usedDomIds + ); + const secHeading = document.createElement(sectionHeadingTag); + secHeading.className = "rtichoke-report__section-title"; + secHeading.id = secHeadingDomId; + secHeading.textContent = sectionSpec.title; + secSection.setAttribute("aria-labelledby", secHeadingDomId); + secSection.append(secHeading); + const groups2 = sectionSpec.items.filter( + (item) => item.type === "group" + ); + const useSectionGroupTabs = options.sectionGroupPresentation === "tabs" && groups2.length > 1; + let groupIndex2 = 0; + let renderedSectionGroupTabs = false; + for (const item of sectionSpec.items) { + if (item.type === "component") { + secSection.append(renderComponent(item, directCompHeadingTag)); + } else if (item.type === "group") { + if (useSectionGroupTabs) { + if (renderedSectionGroupTabs) { + continue; + } + renderedSectionGroupTabs = true; + const tablist = document.createElement("div"); + tablist.className = "rtichoke-report__tablist rtichoke-report__section-group-tablist"; + tablist.setAttribute("role", "tablist"); + tablist.setAttribute("aria-labelledby", secHeadingDomId); + const tabs = []; + const panels = []; + groups2.forEach((group2, tabIndex) => { + const groupNav = secNav.groups[tabIndex]; + const tabDomId = generateUniqueDomId( + `section-group-tab-${sectionSpec.id}-${group2.id}`, + usedDomIds + ); + const tab = document.createElement("button"); + tab.className = "rtichoke-report__tab"; + tab.type = "button"; + tab.id = tabDomId; + tab.setAttribute("role", "tab"); + tab.setAttribute("aria-controls", groupNav.domId); + tab.setAttribute("aria-selected", tabIndex === 0 ? "true" : "false"); + tab.tabIndex = tabIndex === 0 ? 0 : -1; + tab.textContent = group2.title; + const panel = renderGroup(group2, groupNav, tabDomId); + panel.classList.add("rtichoke-report__tabpanel"); + panel.setAttribute("role", "tabpanel"); + panel.setAttribute("aria-labelledby", tabDomId); + panel.tabIndex = 0; + panel.hidden = tabIndex !== 0; + tabs.push(tab); + panels.push(panel); + tablist.append(tab); + }); + wireTabInteraction(tabs, panels); + secSection.append(tablist, ...panels); + } else { + secSection.append(renderGroup(item, secNav.groups[groupIndex2++])); + } + } + } + root2.append(secSection); + } + return root2; +} +function renderReport(spec, options) { + if (!spec || typeof spec !== "object") { + throw new Error("Invalid ReportSpec"); + } + const groupPresentation = options?.groupPresentation ?? "stacked"; + const sectionGroupPresentation = options?.sectionGroupPresentation ?? "stacked"; + if (options !== void 0 && (typeof options !== "object" || options === null || options.groupPresentation !== void 0 && options.groupPresentation !== "stacked" && options.groupPresentation !== "tabs")) { + throw new Error( + "Invalid render options: groupPresentation must be 'stacked' or 'tabs'" + ); + } + if (options !== void 0 && options.sectionGroupPresentation !== void 0 && options.sectionGroupPresentation !== "stacked" && options.sectionGroupPresentation !== "tabs") { + throw new Error( + "Invalid render options: sectionGroupPresentation must be 'stacked' or 'tabs'" + ); + } + if (spec.schemaVersion === "1.0") { + return renderReportV1_0(spec); + } + if (spec.schemaVersion === "1.1") { + return renderReportV1_1(spec, { + groupPresentation, + sectionGroupPresentation + }); + } + throw new Error("Invalid ReportSpec"); +} // src/render/roc.ts var RTICHOKE_COLORS3 = [ @@ -23176,13 +23750,16 @@ function renderRoc(spec) { }); } export { + AUROCSummaryMetricSchema, CalibrationSpecSchema, + CalibrationV2DatumSchema, CalibrationV2SpecSchema, DecisionCurveV2DatumSchema, DecisionCurveV2EvaluationSchema, DecisionCurveV2ReferenceSchema, DecisionCurveV2SeriesSchema, DecisionCurveV2SpecSchema, + DiscreteCalibrationV2DatumSchema, DisplayGroupingSpecSchema, DisplayRoleSchema, EvaluationSpecSchema, @@ -23202,21 +23779,33 @@ export { PerformanceMetricValueSchema, PerformanceTableRowSchema, PerformanceTableSpecSchema, + PopulationSummaryOwnerSpecSchema, PrecisionRecallV2SpecSchema, + PrevalenceSummaryMetricSchema, RTICHOKE_BROWSER_THEME, RTICHOKE_COLORS2 as RTICHOKE_COLORS, ReferenceLineV2SpecSchema, ReportComponentSchema, + ReportComponentV1_1Schema, + ReportGroupSchema, + ReportSectionSchema, ReportSpecSchema, + ReportSpecV1_0Schema, + ReportSpecV1_1Schema, RocSpecSchema, RocV2SpecSchema, RtichokeChartSpecSchema, RtichokeChartSpecV2Schema, SeriesSpecSchema, + SmoothCalibrationV2DatumSchema, + StandaloneCanonicalSpecSchema, + SummaryMetricSchema, + SummaryMetricsSpecSchema, TreatAllReferenceSchema, TreatNoneReferenceSchema, assertPerformanceTableReferentialIntegrity, assertReportReferentialIntegrity, + assertSummaryMetricsReferentialIntegrity, assertV2ReferentialIntegrity, calibrationSpecFromRtichokeRows, calibrationV2SpecFromRtichokeRows, @@ -23231,6 +23820,7 @@ export { renderReport, renderRoc, renderRocV2, + renderSummaryMetrics, resolveV2RenderOptions, rocSpecFromRtichokePython, rocSpecFromRtichokeR, diff --git a/src/rtichoke/_viz_spec_v2.py b/src/rtichoke/_viz_spec_v2.py index 3098b3a5..48ff236d 100644 --- a/src/rtichoke/_viz_spec_v2.py +++ b/src/rtichoke/_viz_spec_v2.py @@ -262,8 +262,13 @@ def _lift_v2_spec_from_performance_data( evaluation_metadata: Mapping[str, _EvaluationMetadata], ) -> dict[str, object]: """Build a canonical lift-v2 spec from production performance quantities.""" + finite_rows = performance_data.filter( + pl.col("chosen_cutoff").is_finite() + & pl.col("ppcr").is_finite() + & pl.col("lift").is_finite() + ) spec = _curve_v2_spec_from_performance_data( - performance_data, + finite_rows, evaluation_metadata, chart_type="lift", ) diff --git a/src/rtichoke/summary_report/summary_report.py b/src/rtichoke/summary_report/summary_report.py index 156587e7..9d58af54 100644 --- a/src/rtichoke/summary_report/summary_report.py +++ b/src/rtichoke/summary_report/summary_report.py @@ -9,12 +9,27 @@ import numpy as np from rtichoke._calibration_viz_spec_v2 import _calibration_v2_spec_from_curve_list +from rtichoke._decision_curve_viz_spec_v2 import ( + _decision_curve_v2_spec_from_performance_data, +) +from rtichoke._interventions_avoided_viz_spec_v2 import ( + _interventions_avoided_v2_spec_from_performance_data, +) from rtichoke._performance_table_spec import ( _performance_table_spec_from_performance_data, ) from rtichoke._report_browser import RtichokeBrowserReport -from rtichoke._report_spec import _report_spec_from_components -from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data +from rtichoke._report_spec import _build_report_spec_v11 +from rtichoke._summary_metrics_spec import ( + _auroc_summary_metrics_spec, + _prevalence_summary_metrics_spec, +) +from rtichoke._viz_spec_v2 import ( + _gains_v2_spec_from_performance_data, + _lift_v2_spec_from_performance_data, + _precision_recall_v2_spec_from_performance_data, + _roc_v2_spec_from_performance_data, +) from rtichoke.calibration.calibration import _create_calibration_curve_list from rtichoke.performance_data.performance_data import prepare_performance_data from rtichoke.processing.evaluation_semantics import _build_evaluation_metadata @@ -42,12 +57,6 @@ def create_summary_report( calculations, canonical standalone component builders, canonical ReportSpec assembly, and the vendored ``rtichoke_viz`` ``renderReport()`` composer. - The first browser report contains a canonical PerformanceTable, ROC-v2, and - calibration-v2 component, in that order. The browser renderer writes an HTML - file plus the vendored ``rtichoke-viz.js`` and ``rtichoke-viz.css`` assets - beside it, and returns the written HTML path. The historical R path retains - its existing return behavior (``None``). - Parameters ---------- probs : Dict[str, np.ndarray] @@ -89,25 +98,183 @@ def _create_browser_summary_report( *, output_file: str | Path, ) -> Path: - """Build the first public canonical browser report from production outputs.""" - performance_data = prepare_performance_data(probs, reals) + """Build the canonical static ReportSpec v1.1 browser summary report.""" metadata = _build_evaluation_metadata(probs, reals, np.array([])) + + # Stratified by probability threshold + perf_data_thresh = prepare_performance_data( + probs, reals, stratified_by=("probability_threshold",), by=0.01 + ) + # Stratified by PPCR + perf_data_ppcr = prepare_performance_data( + probs, reals, stratified_by=("ppcr",), by=0.01 + ) + calibration_curve_list = _create_calibration_curve_list(probs, reals) - performance_table = _performance_table_spec_from_performance_data( - performance_data, metadata + # Specs + prev_spec = _prevalence_summary_metrics_spec(perf_data_thresh, metadata) + auroc_spec = _auroc_summary_metrics_spec(probs, reals, metadata) + + calib_smooth_spec = _calibration_v2_spec_from_curve_list( + calibration_curve_list, metadata, calibration_type="smooth" + ) + calib_discrete_spec = _calibration_v2_spec_from_curve_list( + calibration_curve_list, metadata, calibration_type="discrete" ) - roc = _roc_v2_spec_from_performance_data(performance_data, metadata) - calibration = _calibration_v2_spec_from_curve_list(calibration_curve_list, metadata) - - report = _report_spec_from_components( - [ - {"title": "Performance", "spec": performance_table}, - {"title": "ROC", "spec": roc}, - {"title": "Calibration", "spec": calibration}, - ], - title="rtichoke summary report", + + roc_thresh_spec = _roc_v2_spec_from_performance_data(perf_data_thresh, metadata) + pr_thresh_spec = _precision_recall_v2_spec_from_performance_data( + perf_data_thresh, metadata ) + gains_thresh_spec = _gains_v2_spec_from_performance_data(perf_data_thresh, metadata) + lift_thresh_spec = _lift_v2_spec_from_performance_data(perf_data_thresh, metadata) + + roc_ppcr_spec = _roc_v2_spec_from_performance_data(perf_data_ppcr, metadata) + pr_ppcr_spec = _precision_recall_v2_spec_from_performance_data( + perf_data_ppcr, metadata + ) + gains_ppcr_spec = _gains_v2_spec_from_performance_data(perf_data_ppcr, metadata) + lift_ppcr_spec = _lift_v2_spec_from_performance_data(perf_data_ppcr, metadata) + + decision_curve_spec = _decision_curve_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + interventions_avoided_spec = _interventions_avoided_v2_spec_from_performance_data( + perf_data_thresh, metadata + ) + + table_thresh_spec = _performance_table_spec_from_performance_data( + perf_data_thresh, metadata + ) + table_ppcr_spec = _performance_table_spec_from_performance_data( + perf_data_ppcr, metadata + ) + + sections = [ + { + "id": "prevalence", + "title": "Prevalence", + "components": [ + { + "id": "prevalence-summary", + "title": "Prevalence summary", + "spec": prev_spec, + } + ], + }, + { + "id": "calibration", + "title": "Calibration", + "components": [ + { + "id": "calibration-smooth", + "title": "Smooth", + "spec": calib_smooth_spec, + }, + { + "id": "calibration", + "title": "Discrete", + "spec": calib_discrete_spec, + }, + ], + }, + { + "id": "discrimination", + "title": "Discrimination", + "components": [ + { + "id": "auroc", + "title": "AUROC", + "spec": auroc_spec, + } + ], + "groups": [ + { + "id": "discrimination-probability-threshold", + "title": "By Probability Threshold", + "components": [ + {"id": "roc", "title": "ROC", "spec": roc_thresh_spec}, + { + "id": "precision-recall", + "title": "Precision-Recall", + "spec": pr_thresh_spec, + }, + { + "id": "gains", + "title": "Gains", + "spec": gains_thresh_spec, + }, + {"id": "lift", "title": "Lift", "spec": lift_thresh_spec}, + ], + }, + { + "id": "discrimination-ppcr", + "title": "By PPCR", + "components": [ + {"id": "roc-2", "title": "ROC", "spec": roc_ppcr_spec}, + { + "id": "precision-recall-2", + "title": "Precision-Recall", + "spec": pr_ppcr_spec, + }, + { + "id": "gains-2", + "title": "Gains", + "spec": gains_ppcr_spec, + }, + {"id": "lift-2", "title": "Lift", "spec": lift_ppcr_spec}, + ], + }, + ], + }, + { + "id": "utility", + "title": "Utility", + "components": [ + { + "id": "decision-curve", + "title": "Decision Curve", + "spec": decision_curve_spec, + }, + { + "id": "interventions-avoided", + "title": "Interventions Avoided", + "spec": interventions_avoided_spec, + }, + ], + }, + { + "id": "performance-table", + "title": "Performance Table", + "groups": [ + { + "id": "performance-table-probability-threshold", + "title": "By Probability Threshold", + "components": [ + { + "id": "performance-table", + "title": "Performance Table", + "spec": table_thresh_spec, + } + ], + }, + { + "id": "performance-table-ppcr", + "title": "By PPCR", + "components": [ + { + "id": "performance-table-2", + "title": "Performance Table", + "spec": table_ppcr_spec, + } + ], + }, + ], + }, + ] + + report = _build_report_spec_v11(sections, title="rtichoke summary report") return RtichokeBrowserReport(cast(dict[str, Any], report)).write_html(output_file) diff --git a/tests/test_calibration_v2.py b/tests/test_calibration_v2.py index 3efa3988..25d3b34e 100644 --- a/tests/test_calibration_v2.py +++ b/tests/test_calibration_v2.py @@ -9,7 +9,7 @@ _performance_table_spec_from_performance_data, ) from rtichoke._report_browser import RtichokeBrowserReport -from rtichoke._report_spec import _report_spec_from_components +from rtichoke._report_spec import _build_report_spec_v11 from rtichoke._viz_browser import _calibration_spec_from_curve_list from rtichoke._viz_spec_v2 import _roc_v2_spec_from_performance_data from rtichoke.calibration.calibration import ( @@ -244,11 +244,13 @@ def test_calibration_v2_complete_spec_embeds_unchanged_in_report(): calibration = _calibration_v2_spec_from_curve_list( _create_calibration_curve_list(probs, reals), _metadata(probs, reals) ) - report = _report_spec_from_components([{"spec": calibration}]) + report = _build_report_spec_v11( + [{"id": "sec-1", "components": [{"id": "calibration", "spec": calibration}]}] + ) - assert report["components"][0]["id"] == "calibration" - assert report["components"][0]["spec"] is calibration - assert report["components"][0]["spec"] == calibration + assert report["sections"][0]["items"][0]["id"] == "calibration" + assert report["sections"][0]["items"][0]["spec"] is calibration + assert report["sections"][0]["items"][0]["spec"] == calibration def test_real_performance_table_roc_calibration_report_uses_shared_renderer(tmp_path): @@ -261,11 +263,16 @@ def test_real_performance_table_roc_calibration_report_uses_shared_renderer(tmp_ calibration = _calibration_v2_spec_from_curve_list( _create_calibration_curve_list(probs, reals), metadata ) - report = _report_spec_from_components( + report = _build_report_spec_v11( [ - {"title": "Performance", "spec": table}, - {"title": "ROC", "spec": roc}, - {"title": "Calibration", "spec": calibration}, + { + "id": "sec-1", + "components": [ + {"id": "performance-table", "title": "Performance", "spec": table}, + {"id": "roc", "title": "ROC", "spec": roc}, + {"id": "calibration", "title": "Calibration", "spec": calibration}, + ], + } ] ) @@ -275,16 +282,16 @@ def test_real_performance_table_roc_calibration_report_uses_shared_renderer(tmp_ html = output.read_text(encoding="utf-8") viz_js = (tmp_path / "rtichoke-viz.js").read_text(encoding="utf-8") - assert [component["id"] for component in report["components"]] == [ + assert [item["id"] for item in report["sections"][0]["items"]] == [ "performance-table", "roc", "calibration", ] - assert report["components"][2]["spec"] is calibration + assert report["sections"][0]["items"][2]["spec"] is calibration assert _embedded_report(html) == report assert 'import { renderReport } from "./rtichoke-viz.js";' not in html assert viz_js in html - assert "append(renderReport(spec))" in html + assert 'sectionGroupPresentation: "tabs"' in html def test_duplicate_evaluation_ids_remain_component_local_in_real_report(): @@ -301,17 +308,26 @@ def test_duplicate_evaluation_ids_remain_component_local_in_real_report(): _create_calibration_curve_list(probs, reals), metadata ), ) - report = _report_spec_from_components( - [{"spec": table}, {"spec": roc}, {"spec": calibration}] + report = _build_report_spec_v11( + [ + { + "id": "sec-1", + "components": [ + {"id": "performance-table", "spec": table}, + {"id": "roc", "spec": roc}, + {"id": "calibration", "spec": calibration}, + ], + } + ] ) assert table["evaluations"][0]["id"] == "evaluation-1" assert roc["evaluations"][0]["id"] == "evaluation-1" assert calibration["evaluations"][0]["id"] == "evaluation-1" assert "evaluations" not in report - assert report["components"][0]["spec"] is table - assert report["components"][1]["spec"] is roc - assert report["components"][2]["spec"] is calibration + assert report["sections"][0]["items"][0]["spec"] is table + assert report["sections"][0]["items"][1]["spec"] is roc + assert report["sections"][0]["items"][2]["spec"] is calibration def test_existing_v1_calibration_adapter_and_public_plotly_behavior_are_unchanged(): diff --git a/tests/test_quarto_summary_report_browser.py b/tests/test_quarto_summary_report_browser.py index 3878798f..daa80e23 100644 --- a/tests/test_quarto_summary_report_browser.py +++ b/tests/test_quarto_summary_report_browser.py @@ -137,7 +137,9 @@ def test_quarto_single_browser_summary_report(tmp_path): assert "ROC" in frame.content() assert "Calibration" in frame.content() - tbl_text = frame.locator("table").inner_text() + tbl_text = frame.locator( + ".rtichoke-performance-table__table" + ).first.inner_text() assert "Model" in tbl_text or "True Positives" in tbl_text assert frame.locator("svg").count() >= 2 assert len(errors) == 0, f"Console errors found: {errors}" @@ -230,7 +232,9 @@ def test_quarto_two_browser_summary_reports(tmp_path): assert "ROC" in frame.content() assert "Calibration" in frame.content() - tbl_text = frame.locator("table").inner_text() + tbl_text = frame.locator( + ".rtichoke-performance-table__table" + ).first.inner_text() assert "Model" in tbl_text or "True Positives" in tbl_text assert frame.locator("svg").count() >= 2 diff --git a/tests/test_report_browser.py b/tests/test_report_browser.py index 5874a3bc..ca41d2e5 100644 --- a/tests/test_report_browser.py +++ b/tests/test_report_browser.py @@ -8,7 +8,7 @@ _performance_table_times_spec_from_performance_data, ) from rtichoke._report_browser import RtichokeBrowserReport -from rtichoke._report_spec import _report_spec_from_components +from rtichoke._report_spec import _build_report_spec_v11 from rtichoke._viz_spec_v2 import ( _gains_v2_spec_from_performance_data, _roc_v2_spec_from_performance_data, @@ -36,11 +36,16 @@ def test_browser_report_uses_real_producers_and_only_shared_render_report(tmp_pa table = _performance_table_spec_from_performance_data(performance_data, metadata) roc = _roc_v2_spec_from_performance_data(performance_data, metadata) gains = _gains_v2_spec_from_performance_data(performance_data, metadata) - report = _report_spec_from_components( + report = _build_report_spec_v11( [ - {"title": "Performance", "spec": table}, - {"title": "ROC", "spec": roc}, - {"title": "Gains", "spec": gains}, + { + "id": "sec-1", + "components": [ + {"id": "performance-table", "title": "Performance", "spec": table}, + {"id": "roc", "title": "ROC", "spec": roc}, + {"id": "gains", "title": "Gains", "spec": gains}, + ], + } ], title="Canonical report", ) @@ -53,9 +58,10 @@ def test_browser_report_uses_real_producers_and_only_shared_render_report(tmp_pa assert 'import { renderReport } from "./rtichoke-viz.js";' not in html assert viz_js in html - assert "append(renderReport(spec))" in html + assert 'sectionGroupPresentation: "tabs"' in html + assert 'groupPresentation: "stacked"' in html assert _embedded_report(html) == report - assert [component["id"] for component in report["components"]] == [ + assert [item["id"] for item in report["sections"][0]["items"]] == [ "performance-table", "roc", "gains", @@ -93,12 +99,21 @@ def test_browser_report_preserves_component_local_evaluation_ids_and_time_contex static_data, _build_evaluation_metadata(probs, reals, np.array([])), ) - report = _report_spec_from_components( - [{"spec": time_table}, {"spec": roc}, {"spec": roc}] + report = _build_report_spec_v11( + [ + { + "id": "sec-1", + "components": [ + {"id": "performance-table", "spec": time_table}, + {"id": "roc", "spec": roc}, + {"id": "roc-2", "spec": roc}, + ], + } + ] ) assert "horizon" not in report - assert [component["id"] for component in report["components"]] == [ + assert [item["id"] for item in report["sections"][0]["items"]] == [ "performance-table", "roc", "roc-2", diff --git a/tests/test_report_spec.py b/tests/test_report_spec.py index 3ba52b0a..efb78e99 100644 --- a/tests/test_report_spec.py +++ b/tests/test_report_spec.py @@ -4,7 +4,7 @@ import pytest -from rtichoke._report_spec import _report_spec_from_components +from rtichoke._report_spec import _build_report_spec_v11 from rtichoke.summary_report import summary_report as legacy_summary_report @@ -26,7 +26,6 @@ def _curve_spec( chart_type: str, *, evaluations: list[dict[str, object]] | None = None, - horizon: float | None = None, ) -> dict[str, Any]: evaluations = evaluations or [ _evaluation("evaluation-1", "population-a", "model-a") @@ -41,8 +40,6 @@ def _curve_spec( "evaluationId": "evaluation-1", "display": display, } - if horizon is not None: - series["horizon"] = horizon return { "schemaVersion": "2.0", "type": chart_type, @@ -57,204 +54,87 @@ def _curve_spec( } -def _performance_table_spec(*, time_dependent: bool = False) -> dict[str, Any]: - row: dict[str, object] = { - "evaluationId": "evaluation-1", - "operatingPoint": { - "type": "probability_threshold", - "value": 0.5, - }, - "values": [ - { - "metricId": "sensitivity", - "estimate": 0.8, - } - ], - } - if time_dependent: - row["horizon"] = 365.0 - row["context"] = { - "censoringHeuristic": "include", - "competingEventHeuristic": "exclude", - } +def _summary_metrics_spec(metric_type: str) -> dict[str, Any]: return { - "schemaVersion": "2.0", - "type": "performance_table", - "evaluations": [_evaluation("evaluation-1", "population-a", "model-a")], + "schemaVersion": "1.0", + "type": "summary_metrics", + "title": "Summary", + "evaluations": [], + "populations": [{"id": "population-1", "label": "Population"}], "metrics": [ { - "id": "sensitivity", - "label": "Sensitivity", + "metric": metric_type, + "owner": {"type": "population", "populationId": "population-1"}, + "estimate": 0.25, } ], - "rows": [row], } -def test_core_composition_and_order() -> None: - performance_table = _performance_table_spec() - roc = _curve_spec("roc") - calibration = _curve_spec("calibration") +def test_build_report_spec_v11_structure() -> None: + prev_spec = _summary_metrics_spec("prevalence") + roc_spec = _curve_spec("roc") - report = _report_spec_from_components( - [ - { - "spec": performance_table, - "title": "Performance", - }, - {"spec": roc}, - { - "spec": calibration, - "title": "Calibration", - }, - ], - title="Model report", - ) - - assert report["schemaVersion"] == "1.0" - assert report["type"] == "report" - assert report["title"] == "Model report" - component_ids = [component["id"] for component in report["components"]] - component_types = [component["spec"]["type"] for component in report["components"]] - assert component_ids == [ - "performance-table", - "roc", - "calibration", - ] - assert component_types == [ - "performance_table", - "roc", - "calibration", - ] - - -def test_component_ids_are_deterministic_and_unique() -> None: - first_roc = _curve_spec("roc") - second_roc = _curve_spec("roc") - inputs = [ - {"spec": first_roc}, - {"spec": second_roc}, - {"spec": _curve_spec("lift")}, + sections = [ + { + "id": "prevalence", + "title": "Prevalence", + "components": [ + {"id": "prevalence-summary", "title": "Summary", "spec": prev_spec} + ], + }, + { + "id": "discrimination", + "title": "Discrimination", + "groups": [ + { + "id": "discrimination-probability-threshold", + "title": "By Threshold", + "components": [{"id": "roc", "title": "ROC", "spec": roc_spec}], + } + ], + }, ] - first = _report_spec_from_components(inputs) - second = _report_spec_from_components(inputs) - - ids = [component["id"] for component in first["components"]] - second_ids = [component["id"] for component in second["components"]] - assert ids == ["roc", "roc-2", "lift"] - assert ids == second_ids - assert len(ids) == len(set(ids)) - assert "series-1" not in ids - - -def test_specs_remain_complete_and_component_local() -> None: - roc = _curve_spec("roc") - calibration = _curve_spec("calibration") - - report = _report_spec_from_components( - [ - {"spec": roc}, - {"spec": calibration}, - ] - ) - - assert report["components"][0]["spec"] is roc - assert report["components"][1]["spec"] is calibration - assert roc["evaluations"] == calibration["evaluations"] - assert roc["evaluations"][0]["id"] == "evaluation-1" - assert calibration["evaluations"][0]["id"] == "evaluation-1" - - -def test_semantics_pass_through_unchanged() -> None: - known = _curve_spec( - "roc", - evaluations=[ - _evaluation("evaluation-1", "population-a", "model-a"), - _evaluation("evaluation-2", "population-b", "model-b"), - ], - ) - unknown = _curve_spec( - "calibration", - evaluations=[_evaluation("evaluation-1", "population-c")], - ) - - report = _report_spec_from_components( - [ - {"spec": known}, - {"spec": unknown}, - ] - ) - - assert report["components"][0]["spec"] is known - assert report["components"][1]["spec"] is unknown - assert known["evaluations"][0]["model"] == "model-a" - populations = {item["population"] for item in known["evaluations"]} - assert populations == { - "population-a", - "population-b", - } - assert "model" not in unknown["evaluations"][0] - assert unknown["evaluations"][0]["population"] == "population-c" - - -def test_time_dependent_specs_pass_through_unchanged() -> None: - table = _performance_table_spec(time_dependent=True) - gains = _curve_spec("gains", horizon=365.0) - - report = _report_spec_from_components( - [ - {"spec": table}, - {"spec": gains}, - ] + report = _build_report_spec_v11(sections, title="Test Report") + assert report["schemaVersion"] == "1.1" + assert report["type"] == "report" + assert report["title"] == "Test Report" + assert len(report["sections"]) == 2 + assert report["sections"][0]["id"] == "prevalence" + assert report["sections"][0]["items"][0]["id"] == "prevalence-summary" + assert report["sections"][1]["id"] == "discrimination" + assert ( + report["sections"][1]["items"][0]["id"] + == "discrimination-probability-threshold" ) + assert report["sections"][1]["items"][0]["components"][0]["id"] == "roc" - assert report["components"][0]["spec"] is table - assert report["components"][1]["spec"] is gains - assert table["rows"][0]["horizon"] == 365.0 - assert gains["series"][0]["horizon"] == 365.0 +def test_type_aware_schema_version_validation() -> None: + bad_v1_spec = _summary_metrics_spec("prevalence") + bad_v1_spec["schemaVersion"] = "2.0" # Invalid: summary_metrics must be 1.0 -def test_first_report_component_types_are_supported() -> None: - specs = [ - _performance_table_spec(), - _curve_spec("roc"), - _curve_spec("calibration"), - _curve_spec("precision_recall"), - _curve_spec("gains"), - _curve_spec("lift"), + sections = [ + { + "id": "prevalence", + "components": [{"id": "prevalence-summary", "spec": bad_v1_spec}], + } ] + with pytest.raises(ValueError, match="requires schemaVersion '1.0'"): + _build_report_spec_v11(sections) - report = _report_spec_from_components([{"spec": spec} for spec in specs]) + bad_v2_spec = _curve_spec("roc") + bad_v2_spec["schemaVersion"] = "1.0" # Invalid: roc must be 2.0 - component_ids = [component["id"] for component in report["components"]] - assert component_ids == [ - "performance-table", - "roc", - "calibration", - "precision-recall", - "gains", - "lift", + sections_v2 = [ + { + "id": "discrimination", + "components": [{"id": "roc", "spec": bad_v2_spec}], + } ] - - -def test_invalid_report_components_are_rejected() -> None: - with pytest.raises(ValueError, match="at least one component"): - _report_spec_from_components([]) - - with pytest.raises( - ValueError, - match="Unsupported ReportSpec component type", - ): - _report_spec_from_components( - [ - { - "spec": { - "type": "decision_curve", - } - } - ] - ) + with pytest.raises(ValueError, match="requires schemaVersion '2.0'"): + _build_report_spec_v11(sections_v2) def test_existing_summary_report_still_uses_r_backend( diff --git a/tests/test_rtichoke_viz_vendor.py b/tests/test_rtichoke_viz_vendor.py index 1c8c6cc4..43cf75fa 100644 --- a/tests/test_rtichoke_viz_vendor.py +++ b/tests/test_rtichoke_viz_vendor.py @@ -3,19 +3,19 @@ from pathlib import Path _VENDOR = Path(__file__).parents[1] / "src" / "rtichoke" / "_vendor" / "rtichoke_viz" -_RELEASE_DIR = "rtichoke-viz-0.10.0" -_SHA256 = "030a9e58c6367147b4904d73cf641d8ef3990d95e70b66022a99d6c8d37b3233" -_SOURCE_COMMIT = "b65b903e9456b5eb323dbe0bac823e9ff0c1bd01" +_RELEASE_DIR = "rtichoke-viz-0.14.0" +_SHA256 = "b7c30d12db4b3f8008ef035bd4f7e04c09d599ced5bc8606a4a4141c5cdf48ee" +_SOURCE_COMMIT = "ec3a382656ce5b1c735175d5a8fd1ceb4f153eaf" -def test_vendored_rtichoke_viz_v0100_provenance_archive_and_schemas(): +def test_vendored_rtichoke_viz_v0140_provenance_archive_and_schemas(): provenance = (_VENDOR / "VENDORED_FROM").read_text() - assert "release=v0.10.0" in provenance + assert "release=v0.14.0" in provenance assert f"source_commit={_SOURCE_COMMIT}" in provenance - assert "archive=rtichoke-viz-0.10.0.tar.gz" in provenance + assert "archive=rtichoke-viz-0.14.0.tar.gz" in provenance assert f"sha256={_SHA256}" in provenance - archive = _VENDOR / "rtichoke-viz-0.10.0.tar.gz" + archive = _VENDOR / "rtichoke-viz-0.14.0.tar.gz" assert hashlib.sha256(archive.read_bytes()).hexdigest() == _SHA256 with tarfile.open(archive, "r:gz") as release: assert set(release.getnames()) == { @@ -25,35 +25,40 @@ def test_vendored_rtichoke_viz_v0100_provenance_archive_and_schemas(): f"{_RELEASE_DIR}/rtichoke-viz.js", f"{_RELEASE_DIR}/rtichoke-viz.schema.json", f"{_RELEASE_DIR}/rtichoke-viz-v2.schema.json", + f"{_RELEASE_DIR}/rtichoke-viz-report.schema.json", } manifest = release.extractfile(f"{_RELEASE_DIR}/MANIFEST") assert manifest is not None assert manifest.read().decode() == ( - f"version=0.10.0\ncommit={_SOURCE_COMMIT}\n" + f"version=0.14.0\ncommit={_SOURCE_COMMIT}\n" ) for filename in ( "rtichoke-viz.css", "rtichoke-viz.js", "rtichoke-viz.schema.json", "rtichoke-viz-v2.schema.json", + "rtichoke-viz-report.schema.json", ): packaged = release.extractfile(f"{_RELEASE_DIR}/{filename}") assert packaged is not None assert (_VENDOR / filename).read_bytes() == packaged.read() - assert not (_VENDOR / "rtichoke-viz-0.9.0.tar.gz").exists() + assert not (_VENDOR / "rtichoke-viz-0.10.0.tar.gz").exists() assert (_VENDOR / "rtichoke-viz.js").stat().st_size > 0 assert (_VENDOR / "rtichoke-viz.css").stat().st_size > 0 v1_schema = (_VENDOR / "rtichoke-viz.schema.json").read_text() v2_schema = (_VENDOR / "rtichoke-viz-v2.schema.json").read_text() + report_schema = (_VENDOR / "rtichoke-viz-report.schema.json").read_text() assert '"$id": "https://rtichoke.dev/schema/viz/1.0.json"' in v1_schema assert '"$id": "https://rtichoke.dev/schema/viz/2.0.json"' in v2_schema + assert '"$id": "https://rtichoke.dev/schema/viz/report.json"' in report_schema assert '"decision_curve"' in v2_schema assert '"interventions_avoided"' in v2_schema + assert '"summary_metrics"' in report_schema -def test_v0100_bundle_keeps_existing_exports_and_time_dependent_surfaces(): +def test_v0140_bundle_keeps_existing_exports_and_time_dependent_surfaces(): bundle = (_VENDOR / "rtichoke-viz.js").read_text(encoding="utf-8") for export_name in ( "renderRoc", @@ -70,6 +75,7 @@ def test_v0100_bundle_keeps_existing_exports_and_time_dependent_surfaces(): "RtichokeChartSpecV2Schema", "renderPerformanceTable", "renderReport", + "SummaryMetricsSpecSchema", ): assert export_name in bundle diff --git a/tests/test_summary_metrics_spec.py b/tests/test_summary_metrics_spec.py new file mode 100644 index 00000000..1de69cd2 --- /dev/null +++ b/tests/test_summary_metrics_spec.py @@ -0,0 +1,123 @@ +import numpy as np +import pytest +from sklearn.metrics import roc_auc_score + +from rtichoke._summary_metrics_spec import ( + _auroc_summary_metrics_spec, + _compute_auroc_proc_compatible, + _prevalence_summary_metrics_spec, +) +from rtichoke.performance_data.performance_data import prepare_performance_data +from rtichoke.processing.evaluation_semantics import ( + _EvaluationMetadata, + _build_evaluation_metadata, +) + + +def test_auroc_proc_compatible_direction(): + # Normal orientation: median(controls) <= median(cases) + reals = np.array([0, 0, 0, 1, 1, 1]) + probs = np.array([0.1, 0.2, 0.3, 0.7, 0.8, 0.9]) + score = _compute_auroc_proc_compatible(reals, probs) + assert score == pytest.approx(1.0) + assert score == pytest.approx(roc_auc_score(reals, probs)) + + # Reversed orientation: median(controls) > median(cases) + # pROC auto-direction flips -probs + reals_rev = np.array([0, 0, 0, 1, 1, 1]) + probs_rev = np.array([0.9, 0.8, 0.7, 0.3, 0.2, 0.1]) + score_rev = _compute_auroc_proc_compatible(reals_rev, probs_rev) + assert score_rev == pytest.approx(1.0) + # Standard sklearn without reversing gives 0.0, but pROC auto-direction gives 1.0! + assert roc_auc_score(reals_rev, probs_rev) == pytest.approx(0.0) + assert score_rev == pytest.approx(roc_auc_score(reals_rev, -probs_rev)) + + +def test_auroc_single_class_returns_null(): + reals_all_zeros = np.array([0, 0, 0, 0]) + probs = np.array([0.1, 0.2, 0.3, 0.4]) + assert _compute_auroc_proc_compatible(reals_all_zeros, probs) is None + + reals_all_ones = np.array([1, 1, 1, 1]) + assert _compute_auroc_proc_compatible(reals_all_ones, probs) is None + + +def test_auroc_ties(): + reals = np.array([0, 0, 1, 1]) + probs = np.array([0.5, 0.5, 0.5, 0.5]) + assert _compute_auroc_proc_compatible(reals, probs) == pytest.approx(0.5) + + +def test_summary_metrics_spec_structure(): + probs = {"Model A": np.array([0.8, 0.7, 0.2, 0.1])} + reals = np.array([1, 1, 0, 0]) + perf_data = prepare_performance_data(probs, reals) + metadata = _build_evaluation_metadata(probs, reals, np.array([])) + + prev_spec = _prevalence_summary_metrics_spec(perf_data, metadata) + assert prev_spec["schemaVersion"] == "1.0" + assert prev_spec["type"] == "summary_metrics" + assert prev_spec["title"] == "Prevalence summary" + assert prev_spec["evaluations"] == [] + assert len(prev_spec["populations"]) == 1 + assert prev_spec["populations"][0]["id"] == "population-1" + assert prev_spec["metrics"][0]["owner"]["type"] == "population" + assert prev_spec["metrics"][0]["owner"]["populationId"] == "population-1" + assert prev_spec["metrics"][0]["estimate"] == pytest.approx(0.5) + + auroc_spec = _auroc_summary_metrics_spec(probs, reals, metadata) + assert auroc_spec["schemaVersion"] == "1.0" + assert auroc_spec["type"] == "summary_metrics" + assert auroc_spec["title"] == "AUROC" + assert auroc_spec["populations"] == [] + assert len(auroc_spec["evaluations"]) == 1 + assert auroc_spec["evaluations"][0]["id"] == "evaluation-1" + assert auroc_spec["metrics"][0]["owner"]["type"] == "evaluation" + assert auroc_spec["metrics"][0]["owner"]["evaluationId"] == "evaluation-1" + assert auroc_spec["metrics"][0]["estimate"] == pytest.approx(1.0) + + +def test_prevalence_summary_metrics_spec_distinct_populations_same_label(): + probs = { + "Cohort 1": np.array([0.1, 0.2, 0.8, 0.9]), + "Cohort 2": np.array([0.2, 0.3, 0.7, 0.8]), + } + reals = { + "Cohort 1": np.array([0, 0, 1, 1]), + "Cohort 2": np.array([0, 1, 1, 1]), + } + perf_data = prepare_performance_data(probs, reals) + metadata = _build_evaluation_metadata(probs, reals, np.array([])) + + prev_spec = _prevalence_summary_metrics_spec(perf_data, metadata) + assert len(prev_spec["populations"]) == 2 + assert prev_spec["populations"][0]["id"] == "population-1" + assert prev_spec["populations"][1]["id"] == "population-2" + assert prev_spec["populations"][0]["label"] == "Cohort 1" + assert prev_spec["populations"][1]["label"] == "Cohort 2" + + # Distinct evaluation populations sharing identical display string ("Shared Label") + metadata_shared_label = { + "group-1": _EvaluationMetadata( + reference_group="group-1", + evaluation="group-1", + model=None, + population="Shared Label", + ), + "group-2": _EvaluationMetadata( + reference_group="group-2", + evaluation="group-2", + model=None, + population="Shared Label", + ), + } + + # Distinct populations remain distinct even if their display labels are identical + prev_spec_shared = _prevalence_summary_metrics_spec( + perf_data, metadata_shared_label + ) + assert len(prev_spec_shared["populations"]) == 2 + assert prev_spec_shared["populations"][0]["id"] == "population-1" + assert prev_spec_shared["populations"][1]["id"] == "population-2" + assert prev_spec_shared["populations"][0]["label"] == "Shared Label" + assert prev_spec_shared["populations"][1]["label"] == "Shared Label" diff --git a/tests/test_summary_report_browser.py b/tests/test_summary_report_browser.py index 27096ce8..12c69c2f 100644 --- a/tests/test_summary_report_browser.py +++ b/tests/test_summary_report_browser.py @@ -102,9 +102,11 @@ def _assert_report_rendered(browser: subprocess.CompletedProcess[str]) -> None: assert browser.returncode == 0, browser.stderr assert "INFO:CONSOLE" not in browser.stderr, browser.stderr rendered = _rendered_report_html(browser.stdout) - assert "Performance" in rendered, browser.stderr - assert "ROC" in rendered + assert "Prevalence" in rendered, browser.stderr assert "Calibration" in rendered + assert "Discrimination" in rendered + assert "Utility" in rendered + assert "Performance Table" in rendered assert "= 2 @@ -167,21 +169,19 @@ def test_browser_summary_report_is_opt_in_and_uses_real_canonical_components( html = output.read_text(encoding="utf-8") report = _embedded_report(html) - assert [component["id"] for component in report["components"]] == [ - "performance-table", - "roc", - "calibration", - ] - assert [component["spec"]["type"] for component in report["components"]] == [ - "performance_table", - "roc", + assert report["schemaVersion"] == "1.1" + assert report["type"] == "report" + section_ids = [section["id"] for section in report["sections"]] + assert section_ids == [ + "prevalence", "calibration", + "discrimination", + "utility", + "performance-table", ] - assert report["components"][1]["spec"]["schemaVersion"] == "2.0" - assert report["components"][2]["spec"]["schemaVersion"] == "2.0" assert 'import { renderReport } from "./rtichoke-viz.js";' not in html assert viz_js_path.read_text(encoding="utf-8") in html - assert "append(renderReport(spec))" in html + assert 'sectionGroupPresentation: "tabs"' in html def test_browser_summary_report_executes_when_opened_directly(tmp_path): @@ -239,19 +239,18 @@ def test_browser_summary_report_preserves_component_local_identity(tmp_path): assert "populations" not in report assert "horizon" not in report - table, roc, calibration = [component["spec"] for component in report["components"]] - assert table["evaluations"][0]["id"] == "evaluation-1" - assert roc["evaluations"][0]["id"] == "evaluation-1" - assert calibration["evaluations"][0]["id"] == "evaluation-1" - assert [item["population"] for item in calibration["evaluations"]] == [ - "Population A", - "Population B", - ] - assert all("model" not in item for item in calibration["evaluations"]) - assert [item["display"]["role"] for item in calibration["series"]] == [ - "population", - "population", - ] + prev_spec = report["sections"][0]["items"][0]["spec"] + calib_smooth_spec = report["sections"][1]["items"][0]["spec"] + calib_discrete_spec = report["sections"][1]["items"][1]["spec"] + + assert prev_spec["schemaVersion"] == "1.0" + assert prev_spec["type"] == "summary_metrics" + assert len(prev_spec["populations"]) == 2 + assert ( + "evaluations" not in calib_smooth_spec["yAxis"] + ) # check omit yAxis.domain on smooth + assert calib_smooth_spec["yAxis"].get("domain") is None + assert calib_discrete_spec["yAxis"]["domain"] == [0, 1] def test_browser_summary_report_rejects_unknown_renderer(): diff --git a/uv.lock b/uv.lock index d921c5a3..28e2e333 100644 --- a/uv.lock +++ b/uv.lock @@ -2583,6 +2583,7 @@ dependencies = [ { name = "polars" }, { name = "polarstate" }, { name = "reactable" }, + { name = "scikit-learn" }, { name = "smoothstate" }, ] @@ -2602,7 +2603,6 @@ dev = [ { name = "pytest" }, { name = "pytest-cov" }, { name = "ruff" }, - { name = "scikit-learn" }, { name = "ty" }, { name = "uv" }, ] @@ -2618,6 +2618,7 @@ requires-dist = [ { name = "polars", specifier = ">=1.31.0" }, { name = "polarstate", specifier = "==0.1.8" }, { name = "reactable", specifier = ">=0.1.5" }, + { name = "scikit-learn", specifier = ">=1.6.1" }, { name = "smoothstate", specifier = ">=0.1.1" }, ] provides-extras = ["matplotlib"] @@ -2633,7 +2634,6 @@ dev = [ { name = "pytest", specifier = ">=7.3.0,<8.0.0" }, { name = "pytest-cov", specifier = ">=4.0.0,<5.0.0" }, { name = "ruff", specifier = ">=0.11.0" }, - { name = "scikit-learn", specifier = ">=1.6.1" }, { name = "ty", specifier = ">=0.0.1a12" }, { name = "uv", specifier = ">=0.6.11" }, ]