diff --git a/Project.toml b/Project.toml index 8becaec..c127d7d 100644 --- a/Project.toml +++ b/Project.toml @@ -24,7 +24,7 @@ Weave = "44d3d7a6-8a23-5bf8-98c5-b353f8df5ec9" [extensions] CairoMakieExt = "CairoMakie" PlotlyLightExt = "PlotlyLight" -WeaveExt = ["PlotlyLight", "Weave"] +WeaveExt = "Weave" [compat] CSV = "~0.9, 0.10" diff --git a/README.md b/README.md index a368e93..2ff6f52 100644 --- a/README.md +++ b/README.md @@ -28,17 +28,26 @@ package extensions. Load the backend you want **before** (or alongside) - [PlotlyLight](https://github.com/JuliaComputing/PlotlyLight.jl): lightweight interactive HTML plots — `using PlotlyLight` +Every plot function takes a `backend` key word, defaulting to +`CairoMakieBackend()`: + ```julia using CairoMakie # or `using PlotlyLight` using PowerGraphics -using PowerAnalytics # where `res` is a PowerSimulations.SimulationResults object -gen = get_generation_data(res) -plot_powerdata(gen) # CairoMakie -# plot_powerdata_plotly(gen) # PlotlyLight (`_plotly`-suffixed API) +plot_fuel(res) # CairoMakie (default) +plot_fuel(res; backend = PlotlyLightBackend()) # PlotlyLight ``` +The `_plotly`-suffixed functions (`plot_fuel_plotly`, `plot_dataframe_plotly`, +…) are deprecated: they still work but emit a warning. Replace them with the +un-suffixed function plus `backend = PlotlyLightBackend()`. + +Every other public function returns a plot object. To get the demand *numbers* +behind `plot_demand` — as a `DataFrame` with a `DateTime` column — use +`get_demand_data(res)`. + If neither backend is loaded, `PowerGraphics.jl` prints a warning at load time and the plotting functions throw an `ArgumentError` when called. diff --git a/docs/make.jl b/docs/make.jl index c223507..e743897 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -24,7 +24,7 @@ pages = OrderedDict( ## TODO add additional pages here in the future and remove stubs "Tutorials" => Any["Examples"=>"tutorials/examples.md"], # TODO: make examples page "How to..." => Any["Change Backends"=>"how_to_guides/backends.md"], - # "Explanation" => Any["stub" => "explanation/stub.md"], + "Explanation" => Any["Backend Parity Contract"=>"explanation/backend_parity.md"], "Reference" => Any[ "Public API"=>"reference/public.md", "Developers"=>[ diff --git a/docs/src/explanation/backend_parity.md b/docs/src/explanation/backend_parity.md new file mode 100644 index 0000000..47805b1 --- /dev/null +++ b/docs/src/explanation/backend_parity.md @@ -0,0 +1,90 @@ +# Backend Parity Contract + +```@meta +CurrentModule = PowerGraphics +``` + +`PowerGraphics.jl` renders through two plotting backends, and they are not pixel-identical. +Some of what differs is a promise the package intends to keep, and some of it is an +unavoidable consequence of what CairoMakie and PlotlyLight each can do. This page draws +that line explicitly, so that neither users nor maintainers have to guess which is which. + +The distinction matters. When a divergence is undocumented, a bug fixed in one recipe +quietly stays broken in the other — which is exactly what happened to the bar-plot +stacking fix in +[PR #140](https://github.com/Sienna-Platform/PowerGraphics.jl/pull/140). + +## Choosing a backend + +Every `plot_*` function takes a `backend` key word: + +```julia +backend::PlottingBackend = CairoMakieBackend() +``` + + - [`CairoMakieBackend`](@ref)`()` — the default. Static, publication-quality figures + written as `png`, `pdf`, or `svg`. Requires `using CairoMakie`. + - [`PlotlyLightBackend`](@ref)`()` — lightweight interactive figures written as `html`. + Requires `using PlotlyLight`. + +The backend packages are weak dependencies loaded through Julia package extensions, so the +matching package must be `using`-loaded **before** any plot call. Otherwise the stubs in +`src/PowerGraphics.jl` throw an `ArgumentError` telling you which `using` is missing. + +```julia +using CairoMakie # or PlotlyLight +using PowerGraphics + +plot_fuel(res) # CairoMakie (default) +plot_fuel(res; backend = PlotlyLightBackend()) # PlotlyLight +``` + +## Guaranteed identical across backends + +The behaviors below are resolved **once** in `src/call_plots.jl` (and, for colors, +`src/definitions.jl`) before either recipe is reached. The recipes in `ext/` consume +already-decided values; they do not re-derive them. Treat this list as a stability +promise: **a change to any of these is a change to both backends by construction.** + +| Behavior | Where it is decided | The promise | +|:---------------------------- |:----------------------------------------- |:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Series draw order | `_series_draw_order` | On non-bar plots, series whose values sum to a net-negative total are drawn first, then the rest, each group keeping its original column order. Net-negative series (storage charging, source input) sit below the zero axis, so drawing them first leaves the positive bands on top. | +| Sign-aware stacking | `_series_is_negative` | A series is classified by the sign of its *total*, not per timestep, by the one helper that `_signed_stack_bounds`, `_series_draw_order` and the PlotlyLight `stackgroup` split all read. Positive-type series stack upward from 0; negative-type series stack downward from 0. A positive series keeps a zero-width band in place at timesteps where it is 0 (PV at night) rather than jumping to the negative baseline. | +| `nofill` default | `_PlotOptions` | `nofill = !bar && !stack`. A plain line plot draws no area fill; stacked and bar plots do. | +| `linestyle` / `linewidth` | `_resolve_linestyle`, `_PlotOptions` | `linestyle::Symbol` is the canonical spelling and defaults to `:solid`; the old PlotlyLight-only `line_dash` spelling is folded into it centrally. `linewidth` defaults to `1` and is converted to `Float64` once. | +| Title resolution | `_resolve_title` | `title` defaults to "no title"; the legacy `" "` (single-space) sentinel for "untitled" is normalized to `nothing` in one place. | +| Untitled-save filename | `_UNTITLED_SAVE_NAME` | A [`plot_dataframe`](@ref) save with no title lands at `dataframe.`. | +| Empty-`DataFrame` handling | `_plot_dataframe!` | An empty input warns `"Plot dataframe empty: skipping plot creation"` and returns the plot handle unchanged. Neither recipe is entered, so no labels, legend, or file are produced. | +| Default series color palette | `_PlotOptions`, `get_palette_seriescolor` | Both backends receive a finished `seriescolor` vector, one entry per drawn series and continuing the cycle past series already on the plot. Both select the *same* colors — the whole palette from [`load_palette`](@ref), so more series get a distinct color before the cycle repeats. The two backends differ only in the representation each library wants (`Colors.RGBA` objects vs. `"rgba(...)"` strings). | +| Label handling / `label_fn` | `_PlotOptions` | `label_fn` defaults to [`label_short`](@ref) and is applied in core; the recipes receive the finished legend text as `column_labels`. | + +## Deliberate, documented differences + +These differences are intentional. Each one exists because of a constraint in the +underlying library, and the "Why" column is the reason not to "fix" it. + +| Behavior | CairoMakie | PlotlyLight | Why the difference exists | +|:----------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |:---------------------------------------------------------------------------------------------------------------------------------------------------------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **Save formats** | `png`, `pdf`, `svg` via `CairoMakie.save`, defaulting to `png`. A `.html` filename throws an `ArgumentError` pointing at `PlotlyLightBackend()`. | `html` only, and the default. Any other extension emits a warning and is rewritten to `.html`; the rewritten path is returned. | PlotlyLight has no built-in image export — it serializes a plot to an HTML/JS payload. Rasterizing would require Kaleido/PlotlyBase, which the package deliberately does not depend on. CairoMakie is a vector/raster renderer with no HTML target. `_default_save_format` is therefore dispatched on the backend; an explicit `format` key word still wins. | +| **Time axis** | `DateTime`s are converted to unix floats (`Dates.datetime2unix`) and only the first and last timestamps are drawn as ticks. | Timestamps are passed through as a native Plotly datetime axis with full automatic tick control. | `CairoMakie.band!` — the primitive behind stacked areas — cannot take a `DateTime` axis. Every CairoMakie plot therefore uses a float axis so that stacked and non-stacked layers can share one `Axis`. Float ticks would render as raw unix seconds, so the axis is labeled explicitly at the endpoints. | +| **Bar-plot x-axis** | Grouped bars (`stack = false`) get one tick per category with the label rotated 45° and right/top-anchored. Stacked bars get a single unlabeled tick and are identified by legend only. | Tick labels are hidden for all bar plots (`showticklabels = !bar`); bars are identified by legend only. | Long category labels such as `RenewableDispatch__Curtailment` overlap when drawn horizontally, hence the rotation. CairoMakie stacked bars all sit at one x position (a single `barplot!` call with per-element stack ids), so there is no per-category tick to label; Plotly's `barmode` handles positioning itself and its legend is interactive, so tick labels are redundant. | +| **Y-limit anchoring** | `reset_limits!` on the axis; zero is *not* forced into range. | `yaxis.rangemode = "tozero"`. | Plotly's `rangemode` is a layout flag with no exact Makie equivalent. Makie's autolimits keep a tight fit around the data, which is usually the better default for a static figure; Plotly's zoom/pan makes an anchored baseline cheap to escape. | +| **Stacked-area band outline** | In the non-stair stacked branch the per-band outline is deliberately **omitted** — only the filled band is drawn. The stair branch does draw a `stairs!` outline. | Every trace is a `scatter` with `mode = "lines"`, so the outline is always drawn alongside the fill. | For intermittent series (PV at night, idle storage) a CairoMakie outline jumps between the stacked position and the zero anchor, drawing near-vertical streaks across the stack. Plotly's `stackgroup` machinery interpolates the line along the stacked baseline instead, so the same artifact does not appear. | +| **`save_plot` key words** | Accepted and ignored. | Filtered to a supported set and forwarded to the HTML writer: `autoplay`, `post_script`, `full_html`, `animation_opts`, `default_width`, `default_height`. | These are `PlotlyLight`'s HTML-serialization options; `CairoMakie.save` has no analogue. Unrecognized key words are dropped rather than erroring so that a single `save_plot` call can be written backend-agnostically. | +| **Figure size** | Hardcoded `1280 × 720` (16:9). | Plotly's own default. | Makie's 800×600 (4:3) default deforms time-series stack plots badly enough to be worth overriding; Plotly's default is responsive in the browser. Neither backend honors a `size` key word — see [issue #77](https://github.com/Sienna-Platform/PowerGraphics.jl/issues/77). | + +## Guidance for maintainers + +The recipes in `ext/plot_recipes.jl` and `ext/plotly_recipes.jl` are **drawing layers +only**: each reads a fully-resolved `_PlotOptions` — +scaled data, legend labels, colors, net-sign classification — and turns it into library +calls. Neither reads the raw `kwargs`. + +When you change plotting behavior, decide which kind of change it is: a guaranteed +behavior belongs in `src/`, once, and in the table above; a library-forced divergence +belongs in one recipe *and* in the differences table, naming the constraint. A +**user-visible rendering** difference that is in neither table is a bug, not a design +decision. Internal representation may differ freely and is deliberately not catalogued +here — the plot handle types and the mechanics of legend construction are two examples, +and neither changes what the reader sees. If a difference could be unified but is not, +unify it — the default answer is parity. diff --git a/docs/src/explanation/stub.md b/docs/src/explanation/stub.md deleted file mode 100644 index 979e4f0..0000000 --- a/docs/src/explanation/stub.md +++ /dev/null @@ -1 +0,0 @@ -Please refer to the [Explanation](https://diataxis.fr/explanation/) section of the diataxis framework. diff --git a/docs/src/how_to_guides/backends.md b/docs/src/how_to_guides/backends.md index 80f2218..dc2d517 100644 --- a/docs/src/how_to_guides/backends.md +++ b/docs/src/how_to_guides/backends.md @@ -13,5 +13,40 @@ using CairoMakie # or PlotlyLight using PowerGraphics ``` +## Pick the backend per plot + +The backend is a value, not a separate function: every `plot_*` function takes a +`backend` key word, defaulting to [`CairoMakieBackend`](@ref)`()`. Pass +[`PlotlyLightBackend`](@ref)`()` to render interactive HTML instead. + +```julia +plot_fuel(res) # CairoMakie (default) +plot_fuel(res; backend = PlotlyLightBackend()) # PlotlyLight + +# The same key word works for every family and its `!` form: +plot_demand(res; backend = PlotlyLightBackend()) +plot_dataframe!(p, df, time_range; backend = PlotlyLightBackend()) +``` + +`report` takes the same key word: `report(res, out_path, template; backend = PlotlyLightBackend())`. + +!!! warning "Deprecated: the `_plotly` suffix" + + The `_plotly`-suffixed functions — `plot_demand_plotly`, + `plot_dataframe_plotly`, `plot_results_plotly`, `plot_fuel_plotly`, + `plot_powerdata_plotly`, and their `!` forms — are deprecated. They still + work and forward to the un-suffixed function with + `backend = PlotlyLightBackend()`, but they emit a warning and will be + removed in a future breaking release. They do not accept a `backend` key + word; use the un-suffixed function if you need to choose the backend. + If neither backend is loaded, `PowerGraphics.jl` will print a warning and plotting functions will not be available. + +## Switching backends without surprises + +The two backends do not render identically. Before you swap one for the other — or before +you change plotting behavior — check the [Backend Parity Contract](@ref), which lists what +is guaranteed to match across backends and which differences are deliberate (save formats, +time-axis ticks, bar-plot tick labels, y-limit anchoring, and the `save_plot` key words +each backend accepts). diff --git a/docs/src/reference/public.md b/docs/src/reference/public.md index daa3778..dc0ddb7 100644 --- a/docs/src/reference/public.md +++ b/docs/src/reference/public.md @@ -4,4 +4,33 @@ Modules = [PowerGraphics] Public = true Private = false +Filter = t -> !( + startswith(string(nameof(t)), "plot_powerdata") || + occursin("_plotly", string(nameof(t))) +) +``` + +## Deprecated + +Two families are deprecated but still exported, so existing code keeps working: + + - The `_plotly`-suffixed functions. The backend is now a `backend` key word on + every plot function, so the suffix only doubled the API — write + [`plot_fuel`](@ref)`(res; backend = PlotlyLightBackend())` instead of + `plot_fuel_plotly(res)`. + - The `plot_powerdata` family, which takes a `PowerAnalytics.PowerData` — a + type that predates the PowerAnalytics 1.0 metrics API. Use + [`plot_results`](@ref) with a `Dict{String, DataFrame}` (or + [`plot_dataframe`](@ref) for a single `DataFrame`) instead. + +Both emit a deprecation warning and will be removed in a future breaking release. + +```@autodocs +Modules = [PowerGraphics] +Public = true +Private = false +Filter = t -> ( + startswith(string(nameof(t)), "plot_powerdata") || + occursin("_plotly", string(nameof(t))) +) ``` diff --git a/ext/plot_recipes.jl b/ext/plot_recipes.jl index 7b76f24..6438e59 100644 --- a/ext/plot_recipes.jl +++ b/ext/plot_recipes.jl @@ -16,67 +16,31 @@ function PowerGraphics._empty_plot(backend::PowerGraphics.CairoMakieBackend) return CairoMakiePlot(fig, ax, 0, false) end +PowerGraphics._drawn_series_count( + plot::CairoMakiePlot, + ::PowerGraphics.CairoMakieBackend, +) = plot.series_count + function PowerGraphics._dataframe_plots_internal( - plot::Union{CairoMakiePlot, Nothing}, - variable::DataFrames.DataFrame, + plot::CairoMakiePlot, time_range::Array, - backend::PowerGraphics.CairoMakieBackend; + backend::PowerGraphics.CairoMakieBackend, + opts::PowerGraphics._PlotOptions; kwargs..., ) - save_fig = get(kwargs, :save, nothing) - title = get(kwargs, :title, " ") - bar = get(kwargs, :bar, false) - stack = get(kwargs, :stack, false) - nofill = get(kwargs, :nofill, false) - stair = get(kwargs, :stair, false) - label_fn = get(kwargs, :label_fn, PowerGraphics.label_short) - linestyle = get(kwargs, :linestyle, :solid) - linewidth = get(kwargs, :linewidth, 1) - - time_interval = PowerGraphics.IS.convert_compound_period( - length(time_range) * (time_range[2] - time_range[1]), - ) - interval = - Dates.Millisecond(Dates.Hour(1)) / Dates.Millisecond(time_range[2] - time_range[1]) - - if isnothing(plot) - plot = PowerGraphics._empty_plot(backend) - end - - ndf = PowerGraphics.PA.no_datetime(variable) - column_names = DataFrames.names(ndf) - existing_series = plot.series_count - seriescolor = PowerGraphics.set_seriescolor( - get( - kwargs, - :seriescolor, - PowerGraphics.get_palette_cairomakie( - get(kwargs, :palette, PowerGraphics.PALETTE), - ), - ), - vcat(ones(existing_series), column_names), - )[(existing_series + 1):end] - - if isempty(variable) - @warn "Plot dataframe empty: skipping plot creation" - return plot - end + data = opts.data + labels = opts.column_labels + seriescolor = opts.seriescolor + interval = opts.interval # CairoMakie.band doesn't allow for DateTime axes. Every plot now gets # float axes instead so plots can be layered on the same Axis. time_range_float = Dates.datetime2unix.(time_range) - data = Matrix(ndf) - power_scale = get(kwargs, :power_scale, 1.0) - if power_scale != 1.0 - data = data ./ power_scale - end - labels = [label_fn(label) for label in column_names] - - plot.axis.xlabel = "$time_interval" - plot.axis.ylabel = get(kwargs, :y_label, "") - if title != " " # Only set title if not default - plot.axis.title = title + plot.axis.xlabel = opts.x_label + plot.axis.ylabel = opts.y_label + if !isnothing(opts.title) + plot.axis.title = opts.title end # For stacked bar plots CairoMakie's auto-legend extraction fails because a @@ -85,10 +49,10 @@ function PowerGraphics._dataframe_plots_internal( # manually with PolyElement below. bar_legend_entries = nothing - if bar + if opts.bar plot_data = sum(data; dims = 1) ./ interval - if stack + if opts.stack # CairoMakie stacks within a single barplot! call when given # per-element stack ids. Plotting one slice per call (each with # stack=[1]) just overlays bars at the same x — that's what the @@ -129,15 +93,13 @@ function PowerGraphics._dataframe_plots_internal( end plot.axis.xgridvisible = false else - if stack && !nofill + draw_order = PowerGraphics._series_draw_order(opts.series_negative) + if opts.stack && !opts.nofill # Sign-aware stacked area: positive series stack upward from 0, # negative series (e.g. storage charging) stack downward from 0 so # charging renders below the zero axis. - lower_b, upper_b = PowerGraphics._signed_stack_bounds(data) - # Draw negative (e.g. storage charging) series first so they sit at - # the back; positive generation bands/outlines render on top. - is_neg = [sum(view(data, :, ix)) < 0 for ix in 1:length(labels)] - draw_order = vcat(findall(is_neg), findall(.!is_neg)) + lower_b, upper_b = + PowerGraphics._signed_stack_bounds(data, opts.series_negative) for ix in draw_order lo = lower_b[:, ix] up = upper_b[:, ix] @@ -145,7 +107,7 @@ function PowerGraphics._dataframe_plots_internal( outer = ifelse.(data[:, ix] .>= 0, up, lo) color = seriescolor[ix] - if stair + if opts.stair CairoMakie.stairs!( plot.axis, time_range_float, @@ -153,8 +115,8 @@ function PowerGraphics._dataframe_plots_internal( color = color, label = string(labels[ix]), step = :post, - linestyle = linestyle, - linewidth = linewidth, + linestyle = opts.linestyle, + linewidth = opts.linewidth, ) CairoMakie.band!( plot.axis, @@ -179,16 +141,15 @@ function PowerGraphics._dataframe_plots_internal( ) end end - elseif stack && nofill + elseif opts.stack && opts.nofill # Sign-aware stacked lines: outer envelope of each band (positive # stacked up, negative stacked down). - lower_b, upper_b = PowerGraphics._signed_stack_bounds(data) - is_neg = [sum(view(data, :, ix)) < 0 for ix in 1:length(labels)] - draw_order = vcat(findall(is_neg), findall(.!is_neg)) + lower_b, upper_b = + PowerGraphics._signed_stack_bounds(data, opts.series_negative) for ix in draw_order outer = ifelse.(data[:, ix] .>= 0, upper_b[:, ix], lower_b[:, ix]) color = seriescolor[ix] - if stair + if opts.stair CairoMakie.stairs!( plot.axis, time_range_float, @@ -196,8 +157,8 @@ function PowerGraphics._dataframe_plots_internal( color = color, label = string(labels[ix]), step = :post, - linestyle = linestyle, - linewidth = linewidth, + linestyle = opts.linestyle, + linewidth = opts.linewidth, ) else CairoMakie.lines!( @@ -206,23 +167,23 @@ function PowerGraphics._dataframe_plots_internal( outer; color = color, label = string(labels[ix]), - linestyle = linestyle, - linewidth = linewidth, + linestyle = opts.linestyle, + linewidth = opts.linewidth, ) end end else - for ix in 1:length(labels) + for ix in draw_order color = seriescolor[ix] - if stair + if opts.stair CairoMakie.stairs!( plot.axis, time_range_float, data[:, ix]; color = color, label = string(labels[ix]), - linestyle = linestyle, - linewidth = linewidth, + linestyle = opts.linestyle, + linewidth = opts.linewidth, step = :post, ) else @@ -232,8 +193,8 @@ function PowerGraphics._dataframe_plots_internal( data[:, ix]; color = color, label = string(labels[ix]), - linestyle = linestyle, - linewidth = linewidth, + linestyle = opts.linestyle, + linewidth = opts.linewidth, ) end end @@ -259,14 +220,12 @@ function PowerGraphics._dataframe_plots_internal( end end - legend_position = get(kwargs, :legend_position, :right) - legend_font_size = get(kwargs, :legend_font_size, nothing) legend_kwargs = Dict{Symbol, Any}() - if !isnothing(legend_font_size) - legend_kwargs[:labelsize] = legend_font_size + if !isnothing(opts.legend_font_size) + legend_kwargs[:labelsize] = opts.legend_font_size end - if legend_position == :bottom + if opts.legend_position == :bottom if !isnothing(bar_legend_entries) bar_labels, bar_colors = bar_legend_entries elems = [CairoMakie.PolyElement(; color = c) for c in bar_colors] @@ -306,12 +265,10 @@ function PowerGraphics._dataframe_plots_internal( plot.has_legend = true end - get(kwargs, :set_display, true) && display(plot.figure) + opts.set_display && display(plot.figure) - title = title == " " ? "dataframe" : title - if !isnothing(save_fig) - format = get(kwargs, :format, "png") - save_plot(plot, joinpath(save_fig, "$title.$format"), backend; kwargs...) + if !isnothing(opts.save_file) + save_plot(plot, opts.save_file, backend; kwargs...) end return plot @@ -339,7 +296,7 @@ function PowerGraphics.save_plot( throw( ArgumentError( "HTML output is not supported by the CairoMakie backend; " * - "use a `_plotly` plot function (which uses PlotlyLight) or " * + "pass `backend = PlotlyLightBackend()` to the plot function or " * "choose a raster/vector format such as png, pdf, or svg.", ), ) diff --git a/ext/plotly_recipes.jl b/ext/plotly_recipes.jl index 304f0a3..33d669d 100644 --- a/ext/plotly_recipes.jl +++ b/ext/plotly_recipes.jl @@ -4,70 +4,38 @@ function PowerGraphics._empty_plot(backend::PowerGraphics.PlotlyLightBackend) return PlotlyLight.Plot() end +PowerGraphics._drawn_series_count( + plot::PlotlyLight.Plot, + ::PowerGraphics.PlotlyLightBackend, +) = length(plot.data) + function PowerGraphics._dataframe_plots_internal( - plot, - variable::DataFrames.DataFrame, + plot::PlotlyLight.Plot, time_range::Array, - backend::PowerGraphics.PlotlyLightBackend; + backend::PowerGraphics.PlotlyLightBackend, + opts::PowerGraphics._PlotOptions; kwargs..., ) - save_fig = get(kwargs, :save, nothing) - y_label = get(kwargs, :y_label, "") - title = get(kwargs, :title, " ") - stack = get(kwargs, :stack, false) - bar = get(kwargs, :bar, false) - nofill = get(kwargs, :nofill, !bar && !stack) - label_fn = get(kwargs, :label_fn, PowerGraphics.label_short) - - # Guard before any `plot.data` access — callers may pass `nothing` to ask - # for a fresh plot. - isnothing(plot) && (plot = PowerGraphics._empty_plot(backend)) - - ndf = PowerGraphics.PA.no_datetime(variable) - names = [label_fn(name) for name in DataFrames.names(ndf)] + names = opts.column_labels + seriescolor = opts.seriescolor + interval = opts.interval + plot_data = opts.data + # Plotly keys its stacking on a group name, so a `!` call layering new traces + # has to start its groups past the ones already on the plot. plot_length = length(plot.data) - seriescolor = permutedims( - PowerGraphics.set_seriescolor( - get( - kwargs, - :seriescolor, - PowerGraphics.get_palette_plotly( - get(kwargs, :palette, PowerGraphics.PALETTE), - ), - ), - vcat(ones(plot_length), names), - )[(plot_length + 1):end], - ) - time_interval = PowerGraphics.IS.convert_compound_period( - length(time_range) * (time_range[2] - time_range[1]), - ) - interval = - Dates.Millisecond(Dates.Hour(1)) / Dates.Millisecond(time_range[2] - time_range[1]) + line_shape = opts.stair ? "hv" : "linear" + # Plotly spells the canonical `linestyle::Symbol` as a string. + line_dash = string(opts.linestyle) - if isempty(variable) - @warn "Plot dataframe empty: skipping plot creation" - plot_data = Array{Float64}(undef, 0, 0) - else - plot_data = Matrix(ndf) - end - power_scale = get(kwargs, :power_scale, 1.0) - if power_scale != 1.0 && !isempty(plot_data) - plot_data = plot_data ./ power_scale - end - - plot_type = bar ? "bar" : "scatter" - line_shape = get(kwargs, :stair, false) ? "hv" : "linear" - line_dash = get(kwargs, :line_dash, "solid") - - if bar + if opts.bar plot_data = sum(plot_data; dims = 1) ./ interval - if nofill + if opts.nofill plot_data = [plot_data; plot_data] x_data = [-0.5, 0.5] for ix = 1:length(names) y_data = plot_data[:, ix] - sign_group = sum(y_data) >= 0 ? 0 : 10 + sign_group = opts.series_negative[ix] ? 10 : 0 trace_config = PlotlyLight.Config(; type = "scatter", @@ -78,12 +46,13 @@ function PowerGraphics._dataframe_plots_internal( line = PlotlyLight.Config(; color = seriescolor[ix], dash = line_dash, + width = opts.linewidth, shape = line_shape, ), showlegend = true, ) - if stack + if opts.stack trace_config.stackgroup = string(plot_length + 1 + sign_group) trace_config.fillcolor = "transparent" end @@ -93,7 +62,7 @@ function PowerGraphics._dataframe_plots_internal( else for ix = 1:length(names) y_data = vec(plot_data[:, ix]) - sign_group = sum(y_data) >= 0 ? 0 : 10 + sign_group = opts.series_negative[ix] ? 10 : 0 trace_config = PlotlyLight.Config(; type = "bar", @@ -103,7 +72,7 @@ function PowerGraphics._dataframe_plots_internal( showlegend = true, ) - if stack + if opts.stack trace_config.stackgroup = string(plot_length + 1 + sign_group) trace_config.fillcolor = seriescolor[ix] end @@ -112,13 +81,9 @@ function PowerGraphics._dataframe_plots_internal( end end else - # Scatter plot. Add negative (e.g. storage charging) series first so - # they sit at the back; positive generation renders on top. - is_neg = [sum(view(plot_data, :, ix)) < 0 for ix in 1:length(names)] - draw_order = vcat(findall(is_neg), findall(.!is_neg)) - for ix in draw_order + for ix in PowerGraphics._series_draw_order(opts.series_negative) data_to_plot = plot_data[:, ix] - sign_group = sum(data_to_plot) >= 0 ? 0 : 10 + sign_group = opts.series_negative[ix] ? 10 : 0 trace_config = PlotlyLight.Config(; type = "scatter", @@ -129,20 +94,21 @@ function PowerGraphics._dataframe_plots_internal( line = PlotlyLight.Config(; color = seriescolor[ix], dash = line_dash, + width = opts.linewidth, shape = line_shape, ), showlegend = true, ) - if stack + if opts.stack trace_config.stackgroup = string(plot_length + 1 + sign_group) - if nofill + if opts.nofill trace_config.fillcolor = "transparent" else trace_config.fill = "tonexty" trace_config.fillcolor = seriescolor[ix] end - elseif !nofill + elseif !opts.nofill trace_config.stackgroup = string(ix + plot_length) trace_config.fill = "tonexty" end @@ -153,16 +119,15 @@ function PowerGraphics._dataframe_plots_internal( plot.layout.yaxis.showticklabels = true plot.layout.yaxis.rangemode = "tozero" - plot.layout.yaxis.title.text = y_label - plot.layout.xaxis.showticklabels = !bar - plot.layout.xaxis.title.text = string(time_interval) - plot.layout.title.text = title - plot.layout.barmode = stack ? "relative" : "group" - - legend_position = get(kwargs, :legend_position, :right) - legend_font_size = get(kwargs, :legend_font_size, nothing) + plot.layout.yaxis.title.text = opts.y_label + plot.layout.xaxis.showticklabels = !opts.bar + plot.layout.xaxis.title.text = opts.x_label + if !isnothing(opts.title) + plot.layout.title.text = opts.title + end + plot.layout.barmode = opts.stack ? "relative" : "group" - if legend_position == :bottom + if opts.legend_position == :bottom plot.layout.legend = PlotlyLight.Config(; orientation = "h", x = 0, @@ -171,15 +136,13 @@ function PowerGraphics._dataframe_plots_internal( yanchor = "top", ) end - if !isnothing(legend_font_size) - plot.layout.legend.font = PlotlyLight.Config(; size = legend_font_size) + if !isnothing(opts.legend_font_size) + plot.layout.legend.font = PlotlyLight.Config(; size = opts.legend_font_size) end - get(kwargs, :set_display, true) && display(plot) - if !isnothing(save_fig) - title = title == " " ? "dataframe" : title - format = get(kwargs, :format, "png") - save_plot(plot, joinpath(save_fig, "$title.$format"), backend; kwargs...) + opts.set_display && display(plot) + if !isnothing(opts.save_file) + save_plot(plot, opts.save_file, backend; kwargs...) end return plot end @@ -207,7 +170,7 @@ function PowerGraphics.save_plot( save_kwargs = Dict{Symbol, Any}(((k, v) for (k, v) in kwargs if k in SUPPORTED_PLOTLY_SAVE_KWARGS)) @info "saving plot" filename - if last(splitext(filename)) == ".html" + if lowercase(last(splitext(filename))) == ".html" open(filename, "w") do io show(io, MIME("text/html"), plot; save_kwargs...) end diff --git a/report_templates/generic_report_template.jmd b/report_templates/generic_report_template.jmd index dc895ec..7d644b9 100644 --- a/report_templates/generic_report_template.jmd +++ b/report_templates/generic_report_template.jmd @@ -9,10 +9,11 @@ date : 1-14 ```julia; echo = false using PowerGraphics using PowerAnalytics +using PowerSystems -PowerGraphics._report_plot_fuel( - WEAVE_ARGS["backend"], +plot_fuel( WEAVE_ARGS["results"]; + backend = WEAVE_ARGS["backend"], bar = true, stack = true, ) @@ -21,31 +22,46 @@ PowerGraphics._report_plot_fuel( # Stack Plots ```julia; echo = false -PowerGraphics._report_plot_fuel(WEAVE_ARGS["backend"], WEAVE_ARGS["results"]) +plot_fuel(WEAVE_ARGS["results"]; backend = WEAVE_ARGS["backend"]) ``` # Tables ### Generation + ```julia; echo = false -for (k,v) in get_generation_data(WEAVE_ARGS["results"]).data - display(k) - display(v) +# Realized generation by fuel category, computed with the PowerAnalytics +# metrics API. Categories with no components in the system are skipped, as are +# categories whose components were not modeled with a dispatch variable (e.g. +# `FixedOutput` formulations) and therefore have no stored result. +results = WEAVE_ARGS["results"] +for (category, selector) in pairs(PowerAnalytics.Selectors.generator_categories) + isempty(PowerSystems.get_components(selector, results)) && continue + df = try + compute(PowerAnalytics.Metrics.calc_active_power, results, selector) + catch e + e isa PowerAnalytics.NoResultError && continue + e isa PowerGraphics.IS.InvalidValue && continue + rethrow() + end + display(category) + display(df) end ``` ### Load -```julia; echo = false -for (k,v) in get_load_data(WEAVE_ARGS["results"]).data - display(k) - display(v) -end -``` -### Services ```julia; echo = false -for (k,v) in get_service_data(WEAVE_ARGS["results"]).data - display(k) - display(v) -end +# `get_demand_data` is the public accessor behind `plot_demand`, so this table +# and the demand plot always report the same quantity. Reading +# `calc_load_forecast` (or `calc_system_load_forecast`) directly instead would +# report the *requested* rather than the served demand, and with the opposite +# sign for controllable load formulations — a private `_demand_data` call would +# get the numbers right but would not survive being copied into user code. +display(get_demand_data(WEAVE_ARGS["results"])) ``` + + diff --git a/src/PowerGraphics.jl b/src/PowerGraphics.jl index 075bb83..e0bd0a5 100644 --- a/src/PowerGraphics.jl +++ b/src/PowerGraphics.jl @@ -2,21 +2,27 @@ isdefined(Base, :__precompile__) && __precompile__() module PowerGraphics export load_palette -export plot_demand, plot_demand_plotly -export plot_dataframe, plot_dataframe_plotly -export plot_powerdata, plot_powerdata_plotly -export plot_results, plot_results_plotly -export plot_fuel, plot_fuel_plotly -export plot_demand!, plot_demand_plotly! -export plot_dataframe!, plot_dataframe_plotly! -export plot_powerdata!, plot_powerdata_plotly! -export plot_results!, plot_results_plotly! -export plot_fuel!, plot_fuel_plotly! +export PlottingBackend, CairoMakieBackend, PlotlyLightBackend +export plot_demand, plot_demand! +export plot_dataframe, plot_dataframe! +export plot_results, plot_results! +export plot_fuel, plot_fuel! +export get_demand_data export report export save_plot export label_component, label_variable, label_acronym, label_first_word export label_short, label_truncate +# Deprecated exports — kept so existing user code keeps working. The `_plotly` +# suffix has been replaced by the `backend` key word, and the `plot_powerdata` +# family by `plot_results`/`plot_dataframe`; see `src/deprecated.jl`. +export plot_powerdata, plot_powerdata! +export plot_demand_plotly, plot_demand_plotly! +export plot_dataframe_plotly, plot_dataframe_plotly! +export plot_results_plotly, plot_results_plotly! +export plot_fuel_plotly, plot_fuel_plotly! +export plot_powerdata_plotly, plot_powerdata_plotly! + #I/O Imports import Dates import TimeSeries @@ -37,32 +43,55 @@ include("backends.jl") include("definitions.jl") include("label_utils.jl") include("call_plots.jl") +include("deprecated.jl") # Methods for these are provided by package extensions: # - `_empty_plot(::PlottingBackend)` — CairoMakieExt / PlotlyLightExt -# - `_dataframe_plots_internal(p, df, time, ::PlottingBackend; kwargs...)` — same +# - `_drawn_series_count(plot, ::PlottingBackend)` — same +# - `_dataframe_plots_internal(p, time, ::PlottingBackend, ::_PlotOptions; kwargs...)` — same # - `save_plot(plot, filename, ::PlottingBackend; kwargs...)` — same # - `report(results, out_path, template; kwargs...)` — WeaveExt function report end -function _no_backend_loaded() +# Each stub names the package that its own backend needs, because `backend` +# defaults to `CairoMakieBackend()`: a user who loaded only PlotlyLight reaches +# the CairoMakie stub without having asked for CairoMakie, so a message naming +# both packages would point at the wrong remedy. The default backend's message +# also names the key word that selects the other one. +function _no_backend_loaded(::CairoMakieBackend) throw( ArgumentError( - "No plotting backend loaded. Run `using CairoMakie` or " * - "`using PlotlyLight` before calling PowerGraphics plot functions.", + "CairoMakie is not loaded. Run `using CairoMakie` before calling " * + "PowerGraphics plot functions, or pass `backend = PlotlyLightBackend()` " * + "to plot with PlotlyLight instead.", ), ) end -_empty_plot(::PlottingBackend) = _no_backend_loaded() +function _no_backend_loaded(::PlotlyLightBackend) + throw( + ArgumentError( + "PlotlyLight is not loaded. Run `using PlotlyLight` before calling " * + "PowerGraphics plot functions with `backend = PlotlyLightBackend()`.", + ), + ) +end + +_empty_plot(backend::PlottingBackend) = _no_backend_loaded(backend) + +# How many series a plot handle already carries, so that `_PlotOptions` can +# continue the color cycle instead of restarting it on a `!` call. Each backend +# answers from its own plot object. +_drawn_series_count(::Any, backend::PlottingBackend) = _no_backend_loaded(backend) + function _dataframe_plots_internal( ::Any, - ::DataFrames.DataFrame, ::Any, - ::PlottingBackend; + backend::PlottingBackend, + ::_PlotOptions; kwargs..., ) - return _no_backend_loaded() + return _no_backend_loaded(backend) end function set_seriescolor(seriescolor::Array, vars::Array) diff --git a/src/backends.jl b/src/backends.jl index 9c0fd81..b7b09b5 100644 --- a/src/backends.jl +++ b/src/backends.jl @@ -1,7 +1,30 @@ # Backend system for PowerGraphics.jl # Supports CairoMakie (default) and PlotlyLight (optional) +""" +Supertype of the plotting backends. A backend is a *value*, not a function name: +every `plot_*` function takes it as a `backend` key word and selects the drawing +code by dispatch on the concrete subtype. + +Subtypes: [`CairoMakieBackend`](@ref), [`PlotlyLightBackend`](@ref). +""" abstract type PlottingBackend end +""" +Render with [CairoMakie](https://docs.makie.org/stable/) — static, +publication-quality plots saved as `png`, `pdf`, or `svg`. This is the default +`backend` of every `plot_*` function. Requires `using CairoMakie`. +""" struct CairoMakieBackend <: PlottingBackend end + +""" +Render with [PlotlyLight](https://github.com/JuliaComputing/PlotlyLight.jl) — +lightweight interactive plots saved as `html`. Pass it as +`backend = PlotlyLightBackend()`. Requires `using PlotlyLight`. +""" struct PlotlyLightBackend <: PlottingBackend end + +# File extension used when the caller does not pass `format`; an explicit user +# `format` still wins. See the Backend Parity Contract for why it is dispatched. +_default_save_format(::CairoMakieBackend) = "png" +_default_save_format(::PlotlyLightBackend) = "html" diff --git a/src/call_plots.jl b/src/call_plots.jl index ace2451..61bb89b 100644 --- a/src/call_plots.jl +++ b/src/call_plots.jl @@ -1,15 +1,29 @@ -function _empty_plot() - return _empty_plot(CairoMakieBackend()) -end - -function _empty_plot_plotly() - return _empty_plot(PlotlyLightBackend()) -end - function popkwargs(kwargs, kwarg) return Dict{Symbol, Any}((k, v) for (k, v) in kwargs if k ≠ kwarg) end +# Key-word documentation every public plot function accepts, interpolated into +# each docstring rather than copied into it: the same twelve entries appeared in +# eight docstrings and could only rot independently. Function-specific key words +# stay written out at the call site. +const _COMMON_PLOT_KWARGS = """ +- `set_display::Bool = true`: set to false to prevent the plots from displaying +- `save::String = "file_path"`: set a file path to save the plots +- `format::String`: file extension for saved plots; defaults to `"png"` for the CairoMakie backend and `"html"` for the PlotlyLight backend. CairoMakie supports `"png"`, `"pdf"`, `"svg"`; PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). +- `seriescolor::Array`: Set different colors for the plots +- `title::String = "Title"`: Set a title for the plots +- `stack::Bool = true`: stack plot traces +- `bar::Bool` : create bar plot +- `nofill::Bool = !bar && !stack`: draw traces without an area fill +- `stair::Bool`: Make a stair plot instead of a stack plot +- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_results`; `plot_fuel` always aggregates), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. +- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` +- `legend_font_size::Number`: override the legend label font size""" + +# Documented last in every plot docstring, and in the deprecated shims too. +const _BACKEND_KWARG = """ +- `backend::PlottingBackend = CairoMakieBackend()`: plotting backend, `CairoMakieBackend()` (static png/pdf/svg) or `PlotlyLightBackend()` (interactive html). The matching backend package must be loaded with `using`.""" + # A CairoMakie plot is displayed through its `Figure`; a PlotlyLight plot is # displayed directly. Dispatching keeps the backend split out of plot bodies. _display_plot(::CairoMakieBackend, p) = display(p.figure) @@ -17,9 +31,10 @@ _display_plot(::PlotlyLightBackend, p) = display(p) # Translation table for the user-facing `aggregate::String` kwarg of # `plot_demand` to the typed `aggregation::Type` kwarg expected by -# `PowerAnalytics.get_load_data(::PSY.System; aggregation = …)`. The -# `IS.Results` branch of `get_load_data` ignores `aggregation` entirely, so -# the translation is a safe no-op there. +# `PowerAnalytics.get_load_data(::PSY.System; aggregation = …)`. This +# translation applies ONLY to the `PSY.System` path; the `IS.Results` path +# always aggregates to a single "Load" column and ignores `aggregate` +# entirely. const _AGGREGATE_STRING_TO_TYPE = Dict("System" => PSY.System, "Bus" => PSY.ACBus, "PowerLoad" => PSY.PowerLoad) @@ -47,6 +62,28 @@ function _translate_demand_aggregate(kwargs) return out end +# `start_time`/`len` are documented aliases of `initial_time`/`horizon`. This is +# the only place the two spellings are related; both the `IS.Results` row slicing +# in `_time_window_indices` and the `PSY.System` forwarding below read it. +const _WINDOW_ALIASES = (initial_time = :start_time, horizon = :len) + +function _window_kwarg(kwargs, canonical::Symbol) + return get(kwargs, canonical, get(kwargs, _WINDOW_ALIASES[canonical], nothing)) +end + +# `PowerAnalytics.get_load_data(::PSY.System)` reads only the canonical spellings, +# so an aliased window has to be normalized before forwarding or a `PSY.System` +# plot would silently ignore it. Returns a fresh `Dict{Symbol,Any}` regardless so +# callers can keep mutating it. +function _translate_demand_window(kwargs) + out = Dict{Symbol, Any}(kwargs) + for canonical in keys(_WINDOW_ALIASES) + value = _window_kwarg(out, canonical) + isnothing(value) || (out[canonical] = value) + end + return out +end + """ Pick a power unit and scaling divisor from the peak magnitude of the plotted totals (values are assumed to be in MW): `< 1e3 → MW`, `[1e3, 1e6) → GW`, @@ -83,16 +120,24 @@ function _resolve_power_units(df::DataFrames.DataFrame, kwargs) else # stacked plots: peak is the largest per-timestep positive total; # also guard against a single dominant (possibly negative) series. - max( - maximum(sum(x -> max(x, 0.0), mat; dims = 2)), - maximum(abs, mat), - ) + max(maximum(sum(x -> max(x, 0.0), mat; dims = 2)), maximum(abs, mat)) end divisor, unit = _auto_power_unit(peak) end return (something(user_ylabel, unit), divisor) end +""" +Per-series net-sign classification of a `time × series` matrix: `true` where the +series' values sum to a net-negative total. Every rule that decides which side of +the zero axis a series belongs on — `_signed_stack_bounds`, `_series_draw_order`, +and the PlotlyLight `stackgroup` split — reads this one answer, so the three +cannot disagree. +""" +function _series_is_negative(data::AbstractMatrix) + return [sum(view(data, :, ix)) < zero(eltype(data)) for ix in axes(data, 2)] +end + """ Per-series `(lower, upper)` envelopes for a sign-aware stacked-area/line plot. `data` is `time × series`. Positive values stack **upward** from 0, negative @@ -102,22 +147,23 @@ positive generation stack. Returns `(lower, upper)` matrices the same size as `data`; band `ix` is `[lower[:,ix], upper[:,ix]]`. """ function _signed_stack_bounds(data::AbstractMatrix) + return _signed_stack_bounds(data, _series_is_negative(data)) +end + +# Classification is by *series* (not by value): a positive-type series always +# stacks on the positive baseline — even at timesteps where it is 0 (e.g. PV at +# night) it keeps a zero-width band *in place* rather than jumping to the +# negative baseline, which left whitespace holes and slash lines. +function _signed_stack_bounds(data::AbstractMatrix, negative::AbstractVector{Bool}) nt, ns = size(data) lower = zeros(eltype(data), nt, ns) upper = zeros(eltype(data), nt, ns) pos = zeros(eltype(data), nt) neg = zeros(eltype(data), nt) - # Classify each *series* (not each value) by its net sign, matching the - # PlotlyLight backend's `sign_group`. A positive-type series always stacks - # on the positive baseline — even at timesteps where it is 0 (e.g. PV at - # night) it keeps a zero-width band *in place* rather than jumping to the - # negative baseline (which left whitespace holes / slash lines). Negative- - # type series (e.g. storage charging, source input) always stack downward from 0. for ix in 1:ns - series_negative = sum(@view data[:, ix]) < zero(eltype(data)) for t in 1:nt v = data[t, ix] - if series_negative + if negative[ix] upper[t, ix] = neg[t] lower[t, ix] = neg[t] + v neg[t] = lower[t, ix] @@ -131,6 +177,190 @@ function _signed_stack_bounds(data::AbstractMatrix) return lower, upper end +""" +Series indices in the order a non-bar plot must draw them: series whose values +sum to a net-negative total first, then all the others, each group keeping its +original column order. Net-negative series (storage charging, source input) +stack *below* the zero axis, so drawing them first leaves the positive +generation bands and lines on top of them instead of hidden behind their fill. +""" +function _series_draw_order(negative::AbstractVector{Bool}) + return vcat(findall(negative), findall(.!negative)) +end + +function _series_draw_order(data::AbstractMatrix) + return _series_draw_order(_series_is_negative(data)) +end + +# Old spelling of "this plot has no title". User code still passes it, so it is +# normalized to `nothing` here — the one place that knows about the sentinel. +const _NO_TITLE_SENTINEL = " " + +# Base name a plot is saved under when it carries no title. +const _UNTITLED_SAVE_NAME = "dataframe" + +""" +Everything a backend recipe needs to draw one call, resolved once by +`_plot_dataframe!` so that the recipes in `ext/` consume already-decided values +instead of each deriving its own defaults. Every field is canonical: `data` is +the plotted `time × series` matrix with `power_scale` already applied, +`column_labels` are the finished legend labels, `seriescolor` holds one color per +drawn series (continuing the cycle past any series already on the plot), +`series_negative` is the net-sign classification the stacking and draw-order +rules share, `nofill`/`linestyle`/`linewidth` are always filled in, `title` is +`nothing` when the plot has no title, and `save_file` is the complete path to +write or `nothing` when the plot is not being saved. +""" +struct _PlotOptions{C} + bar::Bool + stack::Bool + stair::Bool + nofill::Bool + linestyle::Symbol + linewidth::Float64 + power_scale::Float64 + y_label::String + x_label::String + title::Union{String, Nothing} + save_file::Union{String, Nothing} + set_display::Bool + legend_position::Symbol + legend_font_size::Union{Float64, Nothing} + data::Matrix{Float64} + column_labels::Vector{String} + seriescolor::Vector{C} + series_negative::Vector{Bool} + interval::Float64 +end + +# `linestyle::Symbol` is the canonical spelling. `line_dash::String` was the +# PlotlyLight-only name for the same thing and is still accepted from old user +# code; folding it in here means neither recipe has to know two names exist. +function _resolve_linestyle(kwargs) + haskey(kwargs, :linestyle) && return Symbol(kwargs[:linestyle]) + haskey(kwargs, :line_dash) && return Symbol(kwargs[:line_dash]) + return :solid +end + +function _resolve_title(kwargs) + title = get(kwargs, :title, nothing) + if isnothing(title) || title == _NO_TITLE_SENTINEL + return nothing + end + return String(title) +end + +# The single place a save path is decided. Spaces in the title become +# underscores, which is what the `plot_demand`/`plot_results`/`plot_fuel` +# wrappers have always done; routing those wrappers through this helper rather +# than letting each rebuild the path is what keeps one filename convention +# across every entry point. +function _resolve_save_file(backend::PlottingBackend, title, kwargs) + save_dir = get(kwargs, :save, nothing) + isnothing(save_dir) && return nothing + format = get(kwargs, :format, _default_save_format(backend)) + name = replace(something(title, _UNTITLED_SAVE_NAME), " " => "_") + return joinpath(save_dir, "$(name).$(format)") +end + +# Key word values arrive with whatever type the caller wrote (`linewidth = 3`, +# `power_scale = 1000`), so each one is converted to the field type here: the +# parametric struct's default constructor matches on the exact type and would +# otherwise reject them. +function _PlotOptions( + p, + variable::DataFrames.DataFrame, + time_range::Vector, + backend::PlottingBackend, + kwargs, +) + bar = get(kwargs, :bar, false) + stack = get(kwargs, :stack, false) + title = _resolve_title(kwargs) + font_size = get(kwargs, :legend_font_size, nothing) + power_scale = Float64(get(kwargs, :power_scale, 1.0)) + + # The `DateTime` column is stripped, the labels are applied and the scaling + # is done once here; a recipe that repeated any of the three would be free to + # repeat it differently. + ndf = PA.no_datetime(variable) + data = Matrix{Float64}(ndf) + power_scale == 1.0 || (data ./= power_scale) + label_fn = get(kwargs, :label_fn, label_short) + column_labels = [string(label_fn(name)) for name in DataFrames.names(ndf)] + + # The color cycle continues past whatever is already drawn on `p`, so a `!` + # call layering a second set of traces does not restart at palette entry one. + drawn = _drawn_series_count(p, backend) + colors = get( + kwargs, + :seriescolor, + get_palette_seriescolor(backend, get(kwargs, :palette, PALETTE)), + ) + seriescolor = set_seriescolor(colors, vcat(ones(drawn), column_labels))[(drawn + 1):end] + + step = time_range[2] - time_range[1] + return _PlotOptions( + bar, + stack, + get(kwargs, :stair, false), + # An area fill is only meaningful under a stacked or bar plot, so a plain + # line plot draws no fill; `_plot_fuel!` forces `true` for its net-load + # overlay. + get(kwargs, :nofill, !bar && !stack), + _resolve_linestyle(kwargs), + Float64(get(kwargs, :linewidth, 1)), + power_scale, + String(get(kwargs, :y_label, "")), + string(IS.convert_compound_period(length(time_range) * step)), + title, + _resolve_save_file(backend, title, kwargs), + get(kwargs, :set_display, true), + Symbol(get(kwargs, :legend_position, :right)), + isnothing(font_size) ? nothing : Float64(font_size), + data, + column_labels, + seriescolor, + _series_is_negative(data), + # One hour expressed in the data's own time step: a bar plot divides its + # summed totals by it to report energy per hour. + Dates.Millisecond(Dates.Hour(1)) / Dates.Millisecond(step), + ) +end + +""" +Row indices selecting the user-requested time window from a full results time +axis; the legacy `initial_time`/`horizon` kwarg spellings stay accepted +alongside `start_time`/`len`. Slicing locally instead of forwarding to +`PowerAnalytics.compute` is deliberate: `compute` rejects unknown kwargs and, +in PowerAnalytics 1.4, mishandles time windows on simulation results (`len` is +treated as an execution count), so local row slicing is the only way to +preserve the old windowing behavior. +""" +# TODO upstream: fix `compute` time-window key words in PowerAnalytics +# (https://github.com/PabloBotin/PowerAnalytics.jl/issues/1), then forward +# `start_time`/`len` directly. +function _time_window_indices(time::AbstractVector, kwargs) + start_time = _window_kwarg(kwargs, :initial_time) + len = _window_kwarg(kwargs, :horizon) + i0 = if isnothing(start_time) + 1 + else + found = findfirst(==(start_time), time) + isnothing(found) && throw( + ArgumentError("start_time $start_time is not one of the results timestamps"), + ) + found + end + i1 = isnothing(len) ? length(time) : i0 + len - 1 + i1 <= length(time) || throw( + ArgumentError( + "the requested time window ends after the results end ($(last(time)))", + ), + ) + return i0:i1 +end + ################################### DEMAND ################################# """ @@ -155,67 +385,264 @@ plot = plot_demand(res) # Accepted Key Words - `linestyle::Symbol = :dash` : set line style -- `title::String`: Set a title for the plots -- `horizon::Int64`: To plot a shorter window of time than the full results -- `initial_time::DateTime`: To start the plot at a different time other than the results initial time -- `aggregate::String = "System", "PowerLoad", or "Bus"`: aggregate the demand other than by generator -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +- `horizon::Int64`: number of time periods to plot, counted from `initial_time` (`len` is accepted as an alias) +- `initial_time::DateTime`: To start the plot at a different time other than the results initial time (`start_time` is accepted as an alias) +- `aggregate::String = "System", "PowerLoad", or "Bus"`: aggregate the demand other than by generator. Applies ONLY to the `PSY.System` input; the `IS.Results` path always aggregates to a single "Load" trace and ignores `aggregate` entirely. +$(_COMMON_PLOT_KWARGS) - `filter_func::Function = `[`PowerSystems.get_available`](@extref PowerSystems InfrastructureSystems.get_available-Tuple{RenewableDispatch}): filter components included in plot +$(_BACKEND_KWARG) """ # ^ temporary workaround for https://github.com/Sienna-Platform/PowerSystems.jl/issues/1598 -function plot_demand(result::Union{IS.Results, PSY.System}; kwargs...) - return plot_demand!(_empty_plot(), result; kwargs...) -end - -@doc (@doc plot_demand) function plot_demand_plotly( +function plot_demand( result::Union{IS.Results, PSY.System}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return plot_demand_plotly!(_empty_plot_plotly(), result; kwargs...) + return plot_demand!(_empty_plot(backend), result; backend = backend, kwargs...) +end + +# Assemble the aggregated demand DataFrame (columns = demand categories, no +# DateTime column) and its time axis. Dispatching on the input type keeps the +# metrics-API and System paths separate. + +# The single demand column the `IS.Results` path always produces; the fixed name +# keeps palette and label behavior identical to the old API. +const _DEMAND_COLUMN = "Load" + +# Demand is read variable-first, exactly as the old `PA.get_load_data` did (it +# scanned `SUPPORTED_LOAD_VARIABLES = [ActivePowerVariable]` and only called +# `add_fixed_parameters!` for load types with no variable stored). The order is +# not cosmetic: under a controllable formulation (`PowerLoadInterruption`, +# `PowerLoadDispatch`) the variable is the *served* load, while the forecast +# parameter is the demand that was requested, and PowerSimulations stores that +# parameter with the opposite sign there — `get_multiplier_value` is +# `+max_active_power` for `AbstractControllablePowerLoadFormulation` against +# `-max_active_power` for `StaticPowerLoad`. Reading only `calc_load_forecast` +# therefore plots the wrong quantity *and* the wrong sign for dispatchable load, +# and in a mixed system the two sign conventions cancel against each other. +const _DEMAND_METRICS = (PA.Metrics.calc_active_power, PA.Metrics.calc_load_forecast) + +# The pool of loads to plot: a user-supplied filter folds into the selector, and +# the default matches the built-in `all_loads` selector. +_demand_selector(::Nothing) = PSY.rebuild_selector(PA.Selectors.all_loads; groupby = :all) +_demand_selector(filter_func::Function) = + PSY.make_selector(filter_func, PSY.ElectricLoad; groupby = :all) + +# The same pool narrowed to one concrete load type, which is what the old +# pipeline keyed on too. Resolving the `_DEMAND_METRICS` fallback per type is +# required because `PowerAnalytics.compute` throws as soon as *any* component in +# a selector is missing the result, so a whole-pool call on a mixed system would +# fall every load back to the forecast and cancel the signs described above. +_demand_type_selector(::Nothing, load_type::Type{<:PSY.ElectricLoad}) = + PSY.make_selector(load_type; groupby = :all) +_demand_type_selector(filter_func::Function, load_type::Type{<:PSY.ElectricLoad}) = + PSY.make_selector(filter_func, load_type; groupby = :all) + +# Compute one metric over one selector, returning `(time, values)` as fresh +# vectors, or `nothing` when that result is not stored for the selected +# components (the old pipeline skipped those keys silently). +function _try_selector_metric(metric, result::IS.Results, selector) + df = try + PA.compute(metric, result, selector) + catch e + _is_missing_result_error(e) && return nothing + rethrow() + end + return ( + Vector{Dates.DateTime}(PA.get_time_vec(df)), + Vector{Float64}(PA.get_data_vec(df)), + ) +end + +# Results path: the PowerAnalytics metrics API. +function _demand_data(result::IS.Results; kwargs...) + filter_func = get(kwargs, :filter_func, nothing) + time = Dates.DateTime[] + total = Float64[] + # Concrete load types present in the pool, ordered deterministically so that + # the summation order (and the floating-point rounding it implies) is + # reproducible. + pool = PSY.get_components(_demand_selector(filter_func), result) + for load_type in sort!(unique(typeof(c) for c in pool); by = nameof) + selector = _demand_type_selector(filter_func, load_type) + for metric in _DEMAND_METRICS + r = _try_selector_metric(metric, result, selector) + isnothing(r) && continue + metric_time, vals = r + if isempty(time) + time = metric_time + total = zeros(Float64, length(metric_time)) + elseif time != metric_time + throw( + ArgumentError( + "Mismatched time axes across load results for \"$load_type\"", + ), + ) + end + total .+= vals + break + end + end + # A load type attached to the system but absent from the problem template + # must not crash the plot: skip missing results like everywhere else and + # fall through to the empty-data ("No load data found") path. + isempty(time) && return (DataFrames.DataFrame(), Dates.DateTime[]) + + window = _time_window_indices(time, kwargs) + # Range indexing allocates fresh vectors, so nothing the metrics returned can + # be mutated downstream (e.g. via `extra_load`). + return (DataFrames.DataFrame(_DEMAND_COLUMN => total[window]), time[window]) +end + +# System path: the new API cannot read demand straight from a `PSY.System`, so +# this stays on the old PowerAnalytics interface, including the +# `aggregate::String` → `aggregation::Type` translation and the +# `start_time`/`len` alias normalization. +function _demand_data(system::PSY.System; kwargs...) + kwargs = _translate_demand_aggregate(_translate_demand_window(kwargs)) + load = PA.get_load_data(system; kwargs...) + return (PA.combine_categories(load.data), load.time) +end + +# Unset key words are dropped rather than forwarded as `nothing`, because the +# window readers distinguish "absent" from "nothing": forwarding an explicit +# `initial_time = nothing` would satisfy the lookup and stop `start_time` from +# ever being consulted. +function _demand_frame(result; kwargs...) + passed = Dict{Symbol, Any}((k, v) for (k, v) in kwargs if !isnothing(v)) + data, time = _demand_data(result; passed...) + return DataFrames.insertcols(data, 1, PA.DATETIME_COL => time) +end + +""" + get_demand_data(results) + +The demand data [`plot_demand`](@ref) draws from simulation results, as a +`DataFrame` whose first column is the `DateTime` axis and whose second is the +aggregated `"Load"` column. Use it to tabulate or post-process the same numbers +the plot shows. + +Reading a single load metric instead does **not** give the same answer: under a +controllable load formulation (`PowerLoadInterruption`, `PowerLoadDispatch`) the +load forecast parameter is the *requested* demand and PowerSimulations stores it +with the opposite sign, so a forecast-only total is understated on a controllable +system and cancels itself on a mixed one. Resolving that per concrete load type +is what this function exists to encapsulate. + +Results are always aggregated into a single column; use +[`get_demand_data(::PowerSystems.System)`](@ref) for the per-component breakdown +that accepts `aggregate`. + +When the results hold no load data this returns a 0-row frame, where +[`plot_demand`](@ref) instead throws an `ArgumentError`. An accessor's caller can +test `nrow` and carry on; a plot with nothing to draw is a mistake worth +reporting, so the two deliberately differ. + +!!! note + + The time windowing below is applied locally because `PowerAnalytics.compute` + mishandles window key words on simulation results. That workaround is + temporary, but this function is exported and so outlives it: if + PowerAnalytics grows a correct load metric the internals change and the + signature stays. + +# Arguments + +- `results::`[`InfrastructureSystems.Results`](@extref): results to read the demand from + (e.g., [`PowerSimulations.SimulationProblemResults`](@extref)) + +# Accepted Key Words + +- `horizon::Int64`: number of time periods to return, counted from `initial_time` (`len` is accepted as an alias) +- `initial_time::DateTime`: start at a time other than the results initial time (`start_time` is accepted as an alias) +- `filter_func::Function`: filter components included in the total +""" +function get_demand_data( + results::IS.Results; + filter_func = nothing, + initial_time = nothing, + start_time = nothing, + horizon = nothing, + len = nothing, +) + return _demand_frame( + results; + filter_func = filter_func, + initial_time = initial_time, + start_time = start_time, + horizon = horizon, + len = len, + ) +end + +""" + get_demand_data(system) + +The demand data [`plot_demand`](@ref) draws from a `System`, as a `DataFrame` +whose first column is the `DateTime` axis. Unlike the +[`get_demand_data(::InfrastructureSystems.Results)`](@ref) method, this one reads +the load time series rather than solved variables, so `aggregate` selects how the +columns are grouped. + +# Arguments + +- `system::`[`PowerSystems.System`](@extref): system to read the demand from + +# Accepted Key Words + +- `horizon::Int64`: number of time periods to return, counted from `initial_time` (`len` is accepted as an alias) +- `initial_time::DateTime`: start at a time other than the system initial time (`start_time` is accepted as an alias) +- `aggregate::String = "System", "PowerLoad", or "Bus"`: group the demand columns by something other than generator +- `filter_func::Function`: filter components included in the total +""" +function get_demand_data( + system::PSY.System; + aggregate = nothing, + filter_func = nothing, + initial_time = nothing, + start_time = nothing, + horizon = nothing, + len = nothing, +) + return _demand_frame( + system; + aggregate = aggregate, + filter_func = filter_func, + initial_time = initial_time, + start_time = start_time, + horizon = horizon, + len = len, + ) end function _plot_demand!(p, result::Union{IS.Results, PSY.System}, backend; kwargs...) set_display = get(kwargs, :set_display, true) - save_fig = get(kwargs, :save, nothing) bar = get(kwargs, :bar, false) title = get(kwargs, :title, "Demand") y_label = get(kwargs, :y_label, bar ? "MWh" : "MW") palette = get(kwargs, :palette, PALETTE) + save_file = _resolve_save_file(backend, title, kwargs) - # Translate the user-facing `aggregate::String` kwarg into PA's typed - # `aggregation` kwarg before calling `get_load_data`. - kwargs = _translate_demand_aggregate(kwargs) - load = PA.get_load_data(result; kwargs...) - # Build a mutable copy with defaults so we splat exactly once below. - kwargs = popkwargs(kwargs, :filter_func) + load_agg, load_time = _demand_data(result; kwargs...) + if isempty(load_agg) + throw(ArgumentError("No load data found")) + end + # A splatted key word wins over an explicit one, so the key words this + # wrapper passes itself — and acts on itself — are dropped from the splat. + kwargs = Dict{Symbol, Any}( + (k, v) for (k, v) in kwargs if k ∉ [:filter_func, :save, :title, :set_display] + ) # Optional per-timestep load added to demand (e.g. storage charging or source # input, so the net-load line matches the top of the generation stack in `plot_fuel!`). extra_load = get(kwargs, :extra_load, nothing) kwargs = popkwargs(kwargs, :extra_load) - linestyle = get(kwargs, :linestyle, :solid) - kwargs[:linestyle] = Symbol(linestyle) - kwargs[:line_dash] = string(linestyle) + # `linestyle` is the canonical spelling for both backends (`_PlotOptions` + # also folds in a caller-supplied `line_dash`), so it is set once here. + kwargs[:linestyle] = _resolve_linestyle(kwargs) kwargs[:linewidth] = get(kwargs, :linewidth, 1) kwargs[:seriescolor] = get(kwargs, :seriescolor, get_palette_seriescolor(backend, palette)) - load_agg = PA.combine_categories(load.data) - - if isnothing(load_agg) - throw(ErrorException("No load data found")) - end - if !isnothing(extra_load) el = collect(extra_load) for c in DataFrames.names(load_agg) @@ -231,7 +658,7 @@ function _plot_demand!(p, result::Union{IS.Results, PSY.System}, backend; kwargs p = _plot_dataframe!( p, load_agg, - load.time, + load_time, backend; y_label = y_label, set_display = false, @@ -240,10 +667,8 @@ function _plot_demand!(p, result::Union{IS.Results, PSY.System}, backend; kwargs ) set_display && _display_plot(backend, p) - if !isnothing(save_fig) - title = replace(title, " " => "_") - format = get(kwargs, :format, "png") - save_plot(p, joinpath(save_fig, "$title.$format"), backend; kwargs...) + if !isnothing(save_file) + save_plot(p, save_file, backend; kwargs...) end return p end @@ -251,12 +676,9 @@ end """ plot_demand!(plot, result) plot_demand!(plot, system) - plot_demand_plotly!(plot, result) - plot_demand_plotly!(plot, system) Plots the demand in the system onto an existing plot handle. The `!`-form mutates -or extends `plot`; the `_plotly` variants render with the PlotlyLight backend -instead of CairoMakie. +or extends `plot`; pass the `backend` key word to pick the renderer. # Arguments @@ -268,37 +690,24 @@ instead of CairoMakie. # Accepted Key Words - `linestyle::Symbol = :dash` : set line style -- `title::String`: Set a title for the plots -- `horizon::Int64`: To plot a shorter window of time than the full results -- `initial_time::DateTime`: To start the plot at a different time other than the results initial time +- `horizon::Int64`: number of time periods to plot, counted from `initial_time` (`len` is accepted as an alias) +- `initial_time::DateTime`: To start the plot at a different time other than the results initial time (`start_time` is accepted as an alias) - `aggregate::String = "System", "PowerLoad", or "Bus"`: aggregate the demand by [`PowerSystems.System`](@extref), [`PowerSystems.PowerLoad`](@extref), or [`PowerSystems.Bus`](@extref), - rather than by generator -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size + rather than by generator. Applies ONLY to the `PSY.System` input; the `IS.Results` path + always aggregates to a single "Load" trace and ignores `aggregate` entirely. +$(_COMMON_PLOT_KWARGS) - `filter_func::Function = `[`PowerSystems.get_available`](@extref PowerSystems InfrastructureSystems.get_available-Tuple{RenewableDispatch}): filter components included in plot - `palette` : color palette from [`load_palette`](@ref) +$(_BACKEND_KWARG) """ -function plot_demand!(p, result::Union{IS.Results, PSY.System}; kwargs...) - return _plot_demand!(p, result, CairoMakieBackend(); kwargs...) -end - -@doc (@doc plot_demand!) function plot_demand_plotly!( +function plot_demand!( p, result::Union{IS.Results, PSY.System}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return _plot_demand!(p, result, PlotlyLightBackend(); kwargs...) + return _plot_demand!(p, result, backend; kwargs...) end ################################# Plotting a Single DataFrame ########################## @@ -327,70 +736,82 @@ plot = plot_dataframe(df, time_range) # Accepted Key Words - `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +$(_COMMON_PLOT_KWARGS) +$(_BACKEND_KWARG) """ -function plot_dataframe(df::DataFrames.DataFrame; kwargs...) - return plot_dataframe!(_empty_plot(), PA.no_datetime(df), df.DateTime; kwargs...) -end function plot_dataframe( - df::DataFrames.DataFrame, - time_range::Union{DataFrames.DataFrame, Array, StepRange}; - kwargs..., -) - return plot_dataframe!(_empty_plot(), df, time_range; kwargs...) -end - -@doc (@doc plot_dataframe) function plot_dataframe_plotly( df::DataFrames.DataFrame; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return plot_dataframe_plotly!( - _empty_plot_plotly(), + return plot_dataframe!( + _empty_plot(backend), PA.no_datetime(df), df.DateTime; + backend = backend, kwargs..., ) end -function plot_dataframe_plotly( + +function plot_dataframe( df::DataFrames.DataFrame, time_range::Union{DataFrames.DataFrame, Array, StepRange}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return plot_dataframe_plotly!(_empty_plot_plotly(), df, time_range; kwargs...) + return plot_dataframe!( + _empty_plot(backend), + df, + time_range; + backend = backend, + kwargs..., + ) end +# A `time_range` handed in as a `DataFrame` carries the axis in its first column. function _plot_dataframe!( p, variable::DataFrames.DataFrame, - time_range::Union{DataFrames.DataFrame, Array, StepRange}, + time_range::DataFrames.DataFrame, backend; kwargs..., ) - tr = - typeof(time_range) == DataFrames.DataFrame ? time_range[:, 1] : collect(time_range) - return _dataframe_plots_internal(p, variable, tr, backend; kwargs...) + return _plot_dataframe!(p, variable, time_range[:, 1], backend; kwargs...) +end + +function _plot_dataframe!( + p, + variable::DataFrames.DataFrame, + time_range::Union{Array, StepRange}, + backend; + kwargs..., +) + # A caller may hand in `nothing` to ask for a fresh plot; resolving it here + # means the recipes can take a concrete plot type. + isnothing(p) && (p = _empty_plot(backend)) + # Nothing downstream — labels, legend, saving — is meaningful without data, + # so the empty case ends here rather than in each recipe. + if isempty(variable) + @warn "Plot dataframe empty: skipping plot creation" + return p + end + tr = collect(time_range) + return _dataframe_plots_internal( + p, + tr, + backend, + _PlotOptions(p, variable, tr, backend, kwargs); + kwargs..., + ) end """ plot_dataframe!(plot, df) plot_dataframe!(plot, df, time_range) - plot_dataframe_plotly!(plot, df) - plot_dataframe_plotly!(plot, df, time_range) Plots data from a [`DataFrames.DataFrame`](@extref) where each row represents a time -period and each column represents a trace, onto an existing plot handle. The -`_plotly` variants render with the PlotlyLight backend instead of CairoMakie. +period and each column represents a trace, onto an existing plot handle. Pass the +`backend` key word to pick the renderer. # Arguments @@ -401,207 +822,145 @@ If only the `DataFrame` is provided, it must have a column of `DateTime` values. # Accepted Key Words - `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +$(_COMMON_PLOT_KWARGS) +$(_BACKEND_KWARG) """ -function plot_dataframe!(p, df::DataFrames.DataFrame; kwargs...) - return _plot_dataframe!( - p, - PA.no_datetime(df), - df.DateTime, - CairoMakieBackend(); - kwargs..., - ) -end - function plot_dataframe!( - p, - variable::DataFrames.DataFrame, - time_range::Union{DataFrames.DataFrame, Array, StepRange}; - kwargs..., -) - return _plot_dataframe!(p, variable, time_range, CairoMakieBackend(); kwargs...) -end - -@doc (@doc plot_dataframe!) function plot_dataframe_plotly!( p, df::DataFrames.DataFrame; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return _plot_dataframe!( - p, - PA.no_datetime(df), - df.DateTime, - PlotlyLightBackend(); - kwargs..., - ) + return _plot_dataframe!(p, PA.no_datetime(df), df.DateTime, backend; kwargs...) end -function plot_dataframe_plotly!( +function plot_dataframe!( p, variable::DataFrames.DataFrame, time_range::Union{DataFrames.DataFrame, Array, StepRange}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return _plot_dataframe!(p, variable, time_range, PlotlyLightBackend(); kwargs...) + return _plot_dataframe!(p, variable, time_range, backend; kwargs...) end -################################# Plotting PowerData ########################## - -""" - plot_powerdata(powerdata) - -Makes a plot from a `PowerAnalytics.PowerData` object, such as the result of -`PowerAnalytics.get_generation_data` - -# Arguments +################################# Plotting a Results Dictionary ########################## + +# Split a dict of result DataFrames from its shared time axis: `DateTime` +# columns are stripped (copying) from every value and the time axis is taken +# from the first value's `DateTime` column, replicating the shape the old +# `PowerAnalytics.PowerData` constructor produced. The strip is not redundant +# with the one inside `PowerAnalytics.combine_categories`, because +# `_flatten_result_categories` emits one trace per stored column and would +# otherwise plot the `DateTime` column as a series. +function _split_results_time(results::Dict{String, DataFrames.DataFrame}) + data = Dict{String, DataFrames.DataFrame}(k => PA.no_datetime(v) for (k, v) in results) + return (data, first(values(results)).DateTime) +end -- `powerdata::PowerAnalytics.PowerData`: The `PowerData` object to be plotted +# `PowerAnalytics.combine_categories` owns the aggregation itself: `names` +# restricts and orders the entries, `aggregate` maps each entry's `time × column` +# matrix to one column, empty entries are dropped silently, and an all-empty +# input yields an empty `DataFrame`. The only thing added here is the error for +# an unknown entry, which upstream reports as a bare `KeyError` that names +# neither the key word nor the entries that would have been valid. +function _combine_result_categories( + data::Dict{String, DataFrames.DataFrame}; + names::Union{Vector{String}, Vector{Symbol}, Nothing} = nothing, + aggregate::Union{Function, Nothing} = nothing, +) + # `Vector{Symbol}` is accepted for the deprecated `plot_powerdata` path, + # whose `PowerData` dicts were keyed by `Symbol` under the old API. + entries = String.(something(names, collect(keys(data)))) + for k in entries + haskey(data, k) || throw( + ArgumentError( + "`names` entry $(repr(k)) is not one of the results entries: " * + "$(sort!(collect(keys(data))))", + ), + ) + end + return PA.combine_categories(data; names = entries, aggregate = aggregate) +end -# Accepted Key Words -- `combine_categories::Bool = false` : plot category values or each value in a category -- `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size -""" -function plot_powerdata(powerdata::PA.PowerData; kwargs...) - return plot_powerdata!(_empty_plot(), powerdata; kwargs...) +# Flatten without aggregation: one trace per stored column, labeled +# "__" so the default `label_short` legend labels reduce to the +# column (usually component) names and collisions across entries are impossible. +function _flatten_result_categories(data::Dict{String, DataFrames.DataFrame}) + cols = Pair{String, Any}[] + for k in sort!(collect(keys(data))) + df = data[k] + for c in DataFrames.names(df) + push!(cols, "$(k)__$(c)" => df[!, c]) + end + end + return DataFrames.DataFrame(cols) end -@doc (@doc plot_powerdata) function plot_powerdata_plotly( - powerdata::PA.PowerData; +function _plot_results!( + p, + data::Dict{String, DataFrames.DataFrame}, + time, + backend; kwargs..., ) - return plot_powerdata_plotly!(_empty_plot_plotly(), powerdata; kwargs...) -end - -function _plot_powerdata!(p, powerdata::PA.PowerData, backend; kwargs...) title = get(kwargs, :title, "") set_display = get(kwargs, :set_display, true) - save_fig = get(kwargs, :save, nothing) + save_file = _resolve_save_file(backend, title, kwargs) - if get(kwargs, :combine_categories, true) - aggregate = get(kwargs, :aggregate, nothing) - names = get(kwargs, :names, nothing) - data = PA.combine_categories(powerdata.data; names = names, aggregate = aggregate) + df = if get(kwargs, :combine_categories, true) + _combine_result_categories( + data; + names = get(kwargs, :names, nothing), + aggregate = get(kwargs, :aggregate, nothing), + ) else - data = powerdata.data + _flatten_result_categories(data) end - kwargs = - Dict{Symbol, Any}((k, v) for (k, v) in kwargs if k ∉ [:title, :save, :set_display]) + kwargs = Dict{Symbol, Any}( + (k, v) for (k, v) in kwargs if + k ∉ [:title, :save, :set_display, :combine_categories, :names, :aggregate] + ) - p = _plot_dataframe!(p, data, powerdata.time, backend; set_display = false, kwargs...) + p = _plot_dataframe!(p, df, time, backend; set_display = false, kwargs...) set_display && _display_plot(backend, p) - if !isnothing(save_fig) - title = replace(title, " " => "_") - format = get(kwargs, :format, "png") - save_plot(p, joinpath(save_fig, "$title.$format"), backend; kwargs...) + if !isnothing(save_file) + save_plot(p, save_file, backend; kwargs...) end return p end -""" - plot_powerdata!(plot, powerdata) - plot_powerdata_plotly!(plot, powerdata) - -Makes a plot from a `PowerAnalytics.PowerData` object, such as the result of -`PowerAnalytics.get_generation_data`, onto an existing plot handle. The `_plotly` -variant renders with the PlotlyLight backend instead of CairoMakie. - -# Arguments - -- `plot`: existing plot handle returned by a previous PowerGraphics plot call (optional; e.g. [`plot_powerdata`](@ref)) -- `powerdata::PowerAnalytics.PowerData`: The `PowerData` object to be plotted - -# Accepted Key Words -- `combine_categories::Bool = false` : plot category values or each value in a category -- `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size -""" -function plot_powerdata!(p, powerdata::PA.PowerData; kwargs...) - return _plot_powerdata!(p, powerdata, CairoMakieBackend(); kwargs...) -end - -@doc (@doc plot_powerdata!) function plot_powerdata_plotly!( - p, - powerdata::PA.PowerData; - kwargs..., -) - return _plot_powerdata!(p, powerdata, PlotlyLightBackend(); kwargs...) -end - """ plot_results(results) -Makes a plot from a results dictionary object +Makes a plot from a results dictionary object. Each entry's `DateTime` column is +stripped and the time axis is taken from the first entry. # Arguments -- `results::Dict{String, DataFrame`: The results to be plotted +- `results::Dict{String, DataFrame}`: The results to be plotted # Accepted Key Words -- `combine_categories::Bool = false` : plot category values or each value in a category -- `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +- `combine_categories::Bool = true` : plot one aggregated trace per entry (the default), or one trace per column of each entry when `false` +- `names::Vector{String}`: subset and order of the entries to plot when `combine_categories = true` +- `aggregate::Function`: reduction applied to each entry's `time × column` matrix when `combine_categories = true` (default `x -> sum(x; dims = 2)`). The function must return an array with one value per time period (length `nrow`); scalar returns are unsupported. +$(_COMMON_PLOT_KWARGS) +$(_BACKEND_KWARG) """ -function plot_results(results::Dict{String, DataFrames.DataFrame}; kwargs...) - return plot_powerdata!(_empty_plot(), PA.PowerData(results); kwargs...) -end - -@doc (@doc plot_results) function plot_results_plotly( +function plot_results( results::Dict{String, DataFrames.DataFrame}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return plot_powerdata_plotly!(_empty_plot_plotly(), PA.PowerData(results); kwargs...) + return plot_results!(_empty_plot(backend), results; backend = backend, kwargs...) end """ plot_results!(plot, results) -Makes a plot from a results dictionary +Makes a plot from a results dictionary onto an existing plot handle. Each entry's +`DateTime` column is stripped and the time axis is taken from the first entry. # Arguments @@ -609,31 +968,20 @@ Makes a plot from a results dictionary - `results::Dict{String, DataFrame}`: The results to be plotted # Accepted Key Words -- `combine_categories::Bool = false` : plot category values or each value in a category -- `curtailment::Bool`: plot the curtailment with the variable -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +- `combine_categories::Bool = true` : plot one aggregated trace per entry (the default), or one trace per column of each entry when `false` +- `names::Vector{String}`: subset and order of the entries to plot when `combine_categories = true` +- `aggregate::Function`: reduction applied to each entry's `time × column` matrix when `combine_categories = true` (default `x -> sum(x; dims = 2)`). The function must return an array with one value per time period (length `nrow`); scalar returns are unsupported. +$(_COMMON_PLOT_KWARGS) +$(_BACKEND_KWARG) """ -function plot_results!(p, results::Dict{String, DataFrames.DataFrame}; kwargs...) - return plot_powerdata!(p, PA.PowerData(results); kwargs...) -end - -@doc (@doc plot_results!) function plot_results_plotly!( +function plot_results!( p, results::Dict{String, DataFrames.DataFrame}; + backend::PlottingBackend = CairoMakieBackend(), kwargs..., ) - return plot_powerdata_plotly!(p, PA.PowerData(results); kwargs...) + data, time = _split_results_time(results) + return _plot_results!(p, data, time, backend; kwargs...) end ################################# Plotting Fuel Plot of Results ########################## @@ -658,83 +1006,578 @@ plot = plot_fuel(res) # Accepted Key Words - `generator_mapping_file` = "file_path" : file path to yaml defining generator category by fuel and primemover -- `variables::Union{Nothing, Vector{Symbol}}` = nothing : specific variables to plot -- `slacks::Bool = true` : display slack variables +- `slacks::Bool = true` : display the system balance slack variables as "Unserved Energy" and "Over Generation". Nodal and area formulations attach one slack per bus/area; those are summed into a single series per direction. Reactive slacks (the `"Q"` meta of full AC formulations) are excluded, since this is an active-power plot. - `load::Bool = true` : display load line - `curtailment::Bool = true`: To plot the curtailment in the stack plot -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +- `storage::Bool = true`: include storage components (as " In"/" Out" traces) +- `sources::Bool = true`: include source components (as " In"/" Out" traces) +- `initial_time::DateTime`: To start the plot at a different time other than the results initial time (`start_time` is accepted as an alias) +- `horizon::Int64`: number of time periods to plot, counted from `initial_time` (`len` is accepted as an alias) +$(_COMMON_PLOT_KWARGS) - `filter_func::Function = `[`PowerSystems.get_available`](@extref PowerSystems InfrastructureSystems.get_available-Tuple{RenewableDispatch}): filter components included in plot +$(_BACKEND_KWARG) """ -function plot_fuel(result::IS.Results; kwargs...) - return plot_fuel!(_empty_plot(), result; kwargs...) +function plot_fuel( + result::IS.Results; + backend::PlottingBackend = CairoMakieBackend(), + kwargs..., +) + return plot_fuel!(_empty_plot(backend), result; backend = backend, kwargs...) end -@doc (@doc plot_fuel) function plot_fuel_plotly(result::IS.Results; kwargs...) - return plot_fuel_plotly!(_empty_plot_plotly(), result; kwargs...) +# The `(backend, result)` positional order is part of the report template's +# contract: `report` renders a user-supplied `.jmd`, and the shipped +# `generic_report_template.jmd` has called this since before `backend` became a +# key word. Templates copied from an earlier release still call it, so removing +# it would throw `UndefVarError` on their next `report` rather than degrade. +_report_plot_fuel(backend::PlottingBackend, result; kwargs...) = + plot_fuel(result; backend = backend, kwargs...) + +# The fuel stack is assembled on the PowerAnalytics metrics/selectors API, one +# metric evaluation per component, because the old pipeline's semantics cannot +# be reproduced with whole-selector `compute` calls: components whose results +# are absent must be skipped silently, each generator needs a +# variable → parameter → aux-variable fallback chain, and categories with no +# contributing component must vanish instead of producing all-zero columns. + +# TODO upstream: PowerAnalytics has no built-in metrics for these entry types +# (it should export forecast metrics for the storage/source time-series +# parameters); build them locally until then. See +# https://github.com/PabloBotin/PowerAnalytics.jl/issues/4. +const _CALC_POWER_OUTPUT = + PA.make_component_metric_from_entry("PowerOutput", PA.PSI.PowerOutput) +const _CALC_ACTIVE_POWER_IN_FORECAST = PA.make_component_metric_from_entry( + "ActivePowerInForecast", + PA.PSI.ActivePowerInTimeSeriesParameter, +) +const _CALC_ACTIVE_POWER_OUT_FORECAST = PA.make_component_metric_from_entry( + "ActivePowerOutForecast", + PA.PSI.ActivePowerOutTimeSeriesParameter, +) + +# Fallback chain for generators: dispatch power if the component was modeled +# with a variable, otherwise its forecast parameter (e.g. `FixedOutput` +# formulations), otherwise the `PowerOutput` aux variable. Only the first +# available metric contributes, mirroring the old `add_fixed_parameters!` / +# `add_aux_variables!` promotion rules. +const _GENERATION_METRICS = ( + (PA.Metrics.calc_active_power, 1.0), + (PA.Metrics.calc_active_power_forecast, 1.0), + (_CALC_POWER_OUTPUT, 1.0), +) +# Storage and source components split into " In"/" Out" +# columns instead of a plain one. Charging drawn through `ActivePowerInVariable` +# is flipped to negative so it stacks below zero; the source input time-series +# parameter is already negative (its multiplier is `active_power_limits.min`), +# so it keeps its sign. Every available metric contributes: if a component ever +# had both the In/Out variable AND the time-series parameter stored, the two +# entries would double-count, but a single PSI problem assigns each component +# type exactly one formulation, so only one of the pair can produce results. +const _STORAGE_IN_METRICS = ((PA.Metrics.calc_active_power_in, -1.0),) +const _STORAGE_OUT_METRICS = ((PA.Metrics.calc_active_power_out, 1.0),) +const _SOURCE_IN_METRICS = + ((PA.Metrics.calc_active_power_in, -1.0), (_CALC_ACTIVE_POWER_IN_FORECAST, 1.0)) +const _SOURCE_OUT_METRICS = + ((PA.Metrics.calc_active_power_out, 1.0), (_CALC_ACTIVE_POWER_OUT_FORECAST, 1.0)) +# System balance slack entry types, in the display order they stack. The names +# come from `PA.BALANCE_SLACKVARS` so the naming has a single source of truth. +const _SLACK_ENTRY_TYPES = + (PA.PSI.SystemBalanceSlackUp, PA.PSI.SystemBalanceSlackDown) + +# Catch-all category for components matched by no rule in the generator +# mapping; matches the `Other` key in the default mapping and the color +# palette, like the old `PA.UNMAPPED_GENERATOR_CATEGORY`. +const _UNMAPPED_CATEGORY = "Other" + +# Exceptions that mean "this result simply is not present": a component absent +# from a stored result table raises `NoResultError`, a result key that was +# never stored raises `InvalidValue`. Anything else is a real error. +_is_missing_result_error(::PA.NoResultError) = true +_is_missing_result_error(::IS.InvalidValue) = true +_is_missing_result_error(::Any) = false + +# Accumulates fuel-category columns on a single shared time axis. +mutable struct _FuelAccumulator + time::Vector{Dates.DateTime} + cols::Dict{String, Vector{Float64}} end -# Backend-dispatched entry point for the Weave report template so the template -# stays backend-agnostic instead of branching on the backend type. -_report_plot_fuel(::CairoMakieBackend, result; kwargs...) = - plot_fuel(result; kwargs...) -_report_plot_fuel(::PlotlyLightBackend, result; kwargs...) = - plot_fuel_plotly(result; kwargs...) +function _FuelAccumulator() + return _FuelAccumulator(Dates.DateTime[], Dict{String, Vector{Float64}}()) +end -function _plot_fuel!(p, result::IS.Results, backend; kwargs...) - set_display = get(kwargs, :set_display, true) - save_fig = get(kwargs, :save, nothing) +function _add_fuel_values!( + acc::_FuelAccumulator, + name::String, + time::Vector{Dates.DateTime}, + vals::Vector{Float64}, +) + if isempty(acc.time) + acc.time = time + elseif acc.time != time + throw(ArgumentError("Mismatched time axes across fuel results for \"$name\"")) + end + col = get!(acc.cols, name) do + zeros(Float64, length(time)) + end + col .+= vals + return acc +end + +# Compute one metric for one component, returning `(time, values)` as fresh +# vectors so the metric's DataFrame is never mutated downstream, or `nothing` +# when the component has no such result (the old pipeline skipped it silently). +function _try_component_metric(metric, result::IS.Results, comp::PSY.Component) + df = try + PA.compute(metric, result, comp) + catch e + _is_missing_result_error(e) && return nothing + rethrow() + end + return ( + Vector{Dates.DateTime}(PA.get_time_vec(df)), + Vector{Float64}(PA.get_data_vec(df)), + ) +end + +function _accumulate_metrics!( + acc::_FuelAccumulator, + name::String, + metrics_and_signs, + result::IS.Results, + comp::PSY.Component, +) + for (metric, sign) in metrics_and_signs + r = _try_component_metric(metric, result, comp) + isnothing(r) && continue + time, vals = r + isone(sign) || (vals .*= sign) + _add_fuel_values!(acc, name, time, vals) + end + return acc +end + +# One component's contribution to its category, dispatched on the component +# role: generators contribute a plain "" column through the fallback +# chain; storage and sources contribute " In"/" Out". +function _accumulate_component!( + acc::_FuelAccumulator, + category::String, + result::IS.Results, + comp::PSY.Generator, +) + for (metric, sign) in _GENERATION_METRICS + r = _try_component_metric(metric, result, comp) + isnothing(r) && continue + time, vals = r + isone(sign) || (vals .*= sign) + _add_fuel_values!(acc, category, time, vals) + return acc + end + return acc +end + +function _accumulate_component!( + acc::_FuelAccumulator, + category::String, + result::IS.Results, + comp::PSY.Storage, +) + _accumulate_metrics!(acc, category * " In", _STORAGE_IN_METRICS, result, comp) + _accumulate_metrics!(acc, category * " Out", _STORAGE_OUT_METRICS, result, comp) + return acc +end + +function _accumulate_component!( + acc::_FuelAccumulator, + category::String, + result::IS.Results, + comp::PSY.Source, +) + _accumulate_metrics!(acc, category * " In", _SOURCE_IN_METRICS, result, comp) + _accumulate_metrics!(acc, category * " Out", _SOURCE_OUT_METRICS, result, comp) + return acc +end + +# Curtailment (forecast minus dispatch) applies only to generators that have +# both results; everything else contributes nothing. +_accumulate_curtailment!(acc::_FuelAccumulator, ::IS.Results, ::PSY.Component) = acc + +function _accumulate_curtailment!( + acc::_FuelAccumulator, + result::IS.Results, + comp::PSY.Generator, +) + r = _try_component_metric(PA.Metrics.calc_curtailment, result, comp) + isnothing(r) && return acc + time, vals = r + _add_fuel_values!(acc, "Curtailment", time, vals) + return acc +end + +""" +Every variable key holding a slack of entry type `T`, whatever component type +owns it. PowerSimulations attaches the balance slacks to `PSY.System` under +`CopperPlatePowerModel`/`PTDFPowerModel`, to `PSY.Area` under the area models, +and to `PSY.ACBus` — one column per bus — under the power-flow models, so +looking only for the `PSY.System` variant (as PowerAnalytics' +`calc_system_slack_up` does) makes nodal and area slacks disappear. Full AC +models split the bus slacks into `"P"` and `"Q"` metas; only `"P"` is returned, +because the fuel stack is an active-power plot. +""" +function _slack_keys(result::IS.Results, ::Type{T}) where {T <: PA.PSI.VariableType} + return filter(PA.PSI.list_variable_keys(result)) do key + PA.PSI.get_entry_type(key) === T && key.meta != "Q" + end +end + +function _accumulate_slacks!(acc::_FuelAccumulator, result::IS.Results) + for entry in _SLACK_ENTRY_TYPES + slack_keys = _slack_keys(result, entry) + # Results without slack variables simply skip the category. + isempty(slack_keys) && continue + name = PA.BALANCE_SLACKVARS[entry] + dfs = PA.PSI.read_results_with_keys( + result, + slack_keys; + table_format = IS.TableFormat.WIDE, + ) + for df in values(dfs) + # One column per bus/area (exactly one for the `PSY.System` case); + # the whole direction stacks as a single series. + vals = zeros(Float64, DataFrames.nrow(df)) + for col in DataFrames.names(df, DataFrames.Not(PA.DATETIME_COL)) + vals .+= df[!, col] + end + _add_fuel_values!( + acc, + name, + Vector{Dates.DateTime}(df[!, PA.DATETIME_COL]), + vals, + ) + end + end + return acc +end + +# Category selectors: the precompiled defaults, or a custom mapping file parsed +# per call. Which categories act as generator vs. storage/source is decided by +# the component roles in the pool, not by the mapping's metadata, so +# `parse_injector_categories` (which works with or without a `__META` section) +# is the right parser here. +_fuel_categories(::Nothing) = PA.Selectors.injector_categories + +# `ext_category` discrimination existed only in the old mapping lookup; the +# PowerAnalytics 1.0 selector parser has no equivalent, so rules carrying it +# still match — just without the ext discrimination. Scan the raw YAML and warn +# so users of such mappings are not silently surprised. +_has_ext_category(::Any) = false +_has_ext_category(v::AbstractVector) = any(_has_ext_category, v) +function _has_ext_category(d::AbstractDict) + return haskey(d, "ext_category") || any(_has_ext_category, values(d)) +end + +function _fuel_categories(file::AbstractString) + if _has_ext_category(YAML.load_file(file)) + @warn "The generator mapping file $file contains `ext_category` keys, which " * + "the PowerAnalytics 1.0 selector parser does not support; those rules " * + "will match without the ext discrimination." + end + return PA.parse_injector_categories(file) +end + +# The generator mapping file behind the category selectors. When the caller +# supplies none, `_fuel_categories` hands back `PA.Selectors.injector_categories`, +# which PowerAnalytics builds as +# `parse_injector_categories(PA.FUEL_TYPES_DATA_FILE)` +# (PowerAnalytics/src/builtin_component_selectors.jl), so that same file +# reproduces the default categories rule for rule. +_fuel_mapping_file(::Nothing) = PA.FUEL_TYPES_DATA_FILE +_fuel_mapping_file(file::AbstractString) = file + +# `parse_injector_categories` and `PA.Selectors.injector_categories` both take +# PowerAnalytics' default root type, so re-deriving rule specificity has to use +# the same one or `parse_fuel_category`'s `typeintersect` lands elsewhere than it +# did when the sub-selectors were built. +const _MAPPING_ROOT_TYPE = PSY.StaticInjection + +_pool_components(::Type{T}, result::IS.Results, filter_func::Function) where {T} = + PSY.get_components(filter_func, T, result) +_pool_components(::Type{T}, result::IS.Results, ::Nothing) where {T} = + PSY.get_components(T, result) + +# The components eligible for fuel plotting: available generators, storage, and +# sources (never loads), optionally restricted by a user filter, matching the +# old `make_fuel_dictionary` iteration. The `storage`/`sources` kwargs of +# `plot_fuel` drop those roles entirely, like the old key filters did. The pool +# is deliberately heterogeneous, so its element type cannot be concrete; it is +# narrowed to the mapping's own root type rather than left at `PSY.Component`. +function _injector_pool(result::IS.Results, filter_func, storage::Bool, sources::Bool) + pool = Vector{_MAPPING_ROOT_TYPE}() + append!(pool, _pool_components(PSY.Generator, result, filter_func)) + storage && append!(pool, _pool_components(PSY.Storage, result, filter_func)) + sources && append!(pool, _pool_components(PSY.Source, result, filter_func)) + return pool +end + +# Number of `supertype` steps from `t` up to `target`, `typemax(Int)` when `t` +# is not a subtype of it at all. The old mapping lookup compared the mapping's +# `gentype` strings against type names; matching the resolved type objects +# instead keeps bare names working (PowerAnalytics' `lookup_gentype` resolves +# them against `PowerSystems`) while telling `Foo.Thermal` apart from +# `Bar.Thermal`, which name matching cannot. `@nospecialize` keeps this to a +# single compiled method instead of one per (component type, rule type) pair. +function _type_distance(@nospecialize(t::Type), @nospecialize(target::Type)) + t <: target || return typemax(Int) + dist = 0 + while true + t === target && return dist + # `target` is a subtype of `t` without appearing in its nominal chain + # (e.g. a `Union` produced by `typeintersect`): still a match, but rank + # it behind every rule whose type the chain does reach. + t === Any && return typemax(Int) - 1 + t = supertype(t) + dist += 1 + end +end + +# One rule of the generator mapping: a category, the rule's specificity taken +# from the `(gentype, primemover, fuel)` triple PowerAnalytics itself parsed out +# of the mapping YAML, and its member components. +struct _FuelRule + category::String + gen_type::Type + pm_wild::Bool + fuel_wild::Bool + members::Set{_MAPPING_ROOT_TYPE} +end + +# The component type a category sub-selector filters on. PowerAnalytics builds +# every one of them via `make_selector(filter_closure, gen_type)`, i.e. as a +# `FilterComponentSelector` (PowerAnalytics/src/builtin_component_selectors.jl, +# `make_fuel_component_selector`). Anything else means the parser changed shape, +# and `Union{}` — which no surviving mapping rule can yield — makes the caller's +# correspondence check fail loudly. +_selector_component_type(selector::IS.FilterComponentSelector) = selector.component_type +_selector_component_type(::IS.ComponentSelector) = Union{} + +# The `(gentype, primemover, fuel)` specificity of every rule listed under +# `category`, in the order PowerAnalytics turns them into sub-selectors. +# `parse_fuel_category` is PowerAnalytics' own parser, so the type and enum items +# here are exactly the ones baked into the corresponding sub-selector's filter, +# and rules that `make_fuel_component_selector` drops (their `gentype` +# intersected away to `Union{}` under the root type) are dropped here too. +function _mapping_rule_specs(raw_mapping::AbstractDict, category::AbstractString) + specs = Vector{Tuple{Type, Bool, Bool}}() + for rule in get(raw_mapping, category, ()) + gen_type, prime_mover, fuel = + PA.parse_fuel_category(rule; root_type = _MAPPING_ROOT_TYPE) + gen_type === Union{} && continue + push!(specs, (gen_type, isnothing(prime_mover), isnothing(fuel))) + end + return specs +end + +# TODO upstream: PowerAnalytics should either expose each mapping rule's +# specificity or document the one-sub-selector-per-rule ordering this replay +# depends on, so the ladder below can be deleted. Not yet filed. +# +# `parse_generator_mapping_file` broadcasts `make_fuel_component_selector` over a +# category's rule list, drops the `nothing`s, and wraps the survivors in a +# `ListComponentSelector`, whose `get_groups` returns its contents verbatim — so +# group `i` comes from surviving rule `i`. PowerAnalytics never promised that +# ordering, so cross-check it against the one thing each group independently +# carries: the component type its filter is built on. A mismatch means the +# specificity above cannot be trusted, and quietly ranking on it would sort +# components into the wrong fuel category with no other symptom. +function _validate_rule_correspondence( + category::AbstractString, + mapping_file::AbstractString, + groups, + specs::Vector{Tuple{Type, Bool, Bool}}, +) + if length(groups) == length(specs) && + all(_selector_component_type(g) === first(s) for (g, s) in zip(groups, specs)) + return nothing + end + throw( + ErrorException( + "Cannot recover generator-mapping rule specificity for category " * + "\"$category\" of $mapping_file: PowerAnalytics $(pkgversion(PA)) " * + "produced sub-selectors $([PA.get_name(g) for g in groups]) on types " * + "$([_selector_component_type(g) for g in groups]), which do not " * + "correspond one-to-one and in order with the parsed rule types " * + "$([first(s) for s in specs]). PowerGraphics relies on " * + "`parse_generator_mapping_file` emitting one sub-selector per " * + "mapping rule, in order, to rank rules by specificity; please " * + "report this as a PowerGraphics issue.", + ), + ) +end + +# Every mapping rule that has at least one member in `result`, paired with the +# specificity of the YAML rule that produced it. +function _fuel_rules( + result::IS.Results, + categories, + mapping_file::AbstractString, + filter_func, +) + raw_mapping = YAML.load_file(mapping_file) + rules = _FuelRule[] + for (category, selector) in categories + groups = collect(PSY.get_groups(selector, result)) + specs = _mapping_rule_specs(raw_mapping, category) + _validate_rule_correspondence(category, mapping_file, groups, specs) + for (group, (gen_type, pm_wild, fuel_wild)) in zip(groups, specs) + members = + Set{_MAPPING_ROOT_TYPE}(PSY.get_components(filter_func, group, result)) + isempty(members) && continue + push!(rules, _FuelRule(category, gen_type, pm_wild, fuel_wild, members)) + end + end + return rules +end + +# Rank a rule for `comp` the way the old first-match-wins ladder did: most +# specific component type first, then prime-mover-specific over wildcard, then +# fuel-specific over wildcard. Smaller ranks win. +function _rule_rank(comp::PSY.Component, rule::_FuelRule) + return (_type_distance(typeof(comp), rule.gen_type), rule.pm_wild, rule.fuel_wild) +end + +""" +Assign each pooled component to exactly one fuel category. The new +PowerAnalytics category selectors are independent, so a component can match +several (e.g. every gas generator matches both `NG-CC` and `NG-Steam` through +the fuel-only fallback rules); replaying the old priority ladder over the +per-rule subselectors keeps each component in a single category and prevents +its energy from being double-counted. Components matching no rule are returned +separately for the "$(_UNMAPPED_CATEGORY)" bucket. +""" +function _assign_fuel_categories( + result::IS.Results, + categories, + mapping_file::AbstractString, + pool, + filter_func, +) + rules = _fuel_rules(result, categories, mapping_file, filter_func) + assignments = Dict{String, Vector{_MAPPING_ROOT_TYPE}}() + unmatched = _MAPPING_ROOT_TYPE[] + for comp in pool + best_category = nothing + best_rank = (typemax(Int), true, true) + for rule in rules + comp in rule.members || continue + rank = _rule_rank(comp, rule) + if isnothing(best_category) || rank < best_rank + best_category = rule.category + best_rank = rank + end + end + if isnothing(best_category) + push!(unmatched, comp) + else + comps = get!(assignments, best_category) do + Vector{_MAPPING_ROOT_TYPE}() + end + push!(comps, comp) + end + end + return assignments, unmatched +end + +""" +Assemble the fuel-stack DataFrame (columns = category names in +palette-first-then-sorted order, no `DateTime` column) and its time axis from +the PowerAnalytics metrics/selectors API. Categories with no contributing +component are dropped rather than emitted as all-zero columns. +""" +function _fuel_data(result::IS.Results, palette_categories::Vector{String}; kwargs...) + # `get_system` is brought into PowerAnalytics from PowerSimulations, so it + # can be reached without going through the unexported `PA.PSI` alias. + if isnothing(PA.get_system(result)) + throw( + ArgumentError( + "No System data present: please run `set_system!(results, sys)` or " * + "load the results with `populate_system = true`", + ), + ) + end + haskey(kwargs, :variables) && + @warn "The `variables` kwarg is no longer supported and is ignored; " * + "use filter_func/generator_mapping_file instead." + filter_func = get(kwargs, :filter_func, nothing) curtailment = get(kwargs, :curtailment, true) slacks = get(kwargs, :slacks, true) + storage = get(kwargs, :storage, true) + sources = get(kwargs, :sources, true) + mapping_arg = get(kwargs, :generator_mapping_file, nothing) + categories = _fuel_categories(mapping_arg) + mapping_file = _fuel_mapping_file(mapping_arg) + + pool = _injector_pool(result, filter_func, storage, sources) + assignments, unmatched = + _assign_fuel_categories(result, categories, mapping_file, pool, filter_func) + + acc = _FuelAccumulator() + for (category, comps) in assignments, comp in comps + _accumulate_component!(acc, category, result, comp) + end + if !isempty(unmatched) + unmatched_names = sort([PSY.get_name(c) for c in unmatched]) + @error "No category in the generator mapping for components: " * + "$(join(unmatched_names, ", ")); plotting them as \"$(_UNMAPPED_CATEGORY)\"" + for comp in unmatched + _accumulate_component!(acc, _UNMAPPED_CATEGORY, result, comp) + end + end + if curtailment + for comp in pool + _accumulate_curtailment!(acc, result, comp) + end + end + slacks && _accumulate_slacks!(acc, result) + + isempty(acc.cols) && throw(ErrorException("No generation data found in the results")) + + # Palette categories first (in palette order), then the sorted remainder; + # this column order is the trace order backends draw, so it must not change. + matched = intersect(palette_categories, collect(keys(acc.cols))) + remainder = sort(setdiff(collect(keys(acc.cols)), palette_categories)) + window = _time_window_indices(acc.time, kwargs) + fuel_agg = DataFrames.DataFrame([ + name => acc.cols[name][window] for name in vcat(matched, remainder) + ],) + return (fuel_agg, acc.time[window]) +end + +function _plot_fuel!(p, result::IS.Results, backend; kwargs...) + set_display = get(kwargs, :set_display, true) load = get(kwargs, :load, true) title = get(kwargs, :title, "Fuel") stack = get(kwargs, :stack, true) - bar = get(kwargs, :bar, false) palette = get(kwargs, :palette, PALETTE) + save_file = _resolve_save_file(backend, title, kwargs) kwargs = Dict{Symbol, Any}((k, v) for (k, v) in kwargs if k ∉ [:title, :save, :set_display]) - # Generation stack - gen = PA.get_generation_data(result; kwargs...) - sys = PA.PSI.get_system(result) - if sys === nothing - throw( - ArgumentError("No System data present: please run `set_system!(results, sys)`"), - ) - end - cat = PA.make_fuel_dictionary(sys; kwargs...) - fuel = PA.categorize_data(gen.data, cat; curtailment = curtailment, slacks = slacks) + # Generation stack, assembled on the PowerAnalytics metrics/selectors API. + fuel_agg, fuel_time = _fuel_data(result, get_palette_category(palette); kwargs...) filter_func = get(kwargs, :filter_func, PSY.get_available) kwargs = popkwargs(kwargs, :filter_func) - # passing names here enforces order; append any fuel categories not in the palette - palette_categories = get_palette_category(palette) - matched = intersect(palette_categories, keys(fuel)) - unmatched = setdiff(keys(fuel), palette_categories) - fuel_agg = PA.combine_categories(fuel; names = vcat(matched, sort(collect(unmatched)))) y_label, power_scale = _resolve_power_units(fuel_agg, kwargs) kwargs = popkwargs(popkwargs(popkwargs(kwargs, :y_label), :power_scale), :auto_units) - seriescolor = get( - kwargs, - :seriescolor, - match_fuel_colors(fuel_agg, backend; palette = palette), - ) + seriescolor = + get(kwargs, :seriescolor, match_fuel_colors(fuel_agg, backend; palette = palette)) p = _plot_dataframe!( p, fuel_agg, - gen.time, + fuel_time, backend; stack = stack, seriescolor = seriescolor, @@ -754,16 +1597,12 @@ function _plot_fuel!(p, result::IS.Results, backend; kwargs...) if load # Net-load line = demand + storage charging + source input, so it coincides # with the top of the generation stack (both are drawn as negative bands by - # the sign-aware stacker; only curtailment sits above the line). - charge = nothing - charge_cols = [k for k in keys(fuel) if endswith(k, " In")] - if !isempty(charge_cols) - nrows = length(gen.time) - charge = zeros(nrows) - for k in charge_cols - m = Matrix(PA.no_datetime(fuel[k])) # negative (charging) - charge .+= -vec(sum(m; dims = 2)) # -> positive load - end + # the sign-aware stacker; only curtailment sits above the line). The + # " In" columns are negative, so their flipped sum is the extra + # load the overlay must include. + in_cols = [c for c in DataFrames.names(fuel_agg) if endswith(c, " In")] + if !isempty(in_cols) + kwargs[:extra_load] = -vec(sum(Matrix(fuel_agg[!, in_cols]); dims = 2)) end p = _plot_demand!( p, @@ -784,21 +1623,18 @@ function _plot_fuel!(p, result::IS.Results, backend; kwargs...) # TODO: how to display this? set_display && _display_plot(backend, p) - if !isnothing(save_fig) - title = replace(title, " " => "_") - format = get(kwargs, :format, "png") - save_plot(p, joinpath(save_fig, "$title.$format"), backend; kwargs...) + if !isnothing(save_file) + save_plot(p, save_file, backend; kwargs...) end return p end """ plot_fuel!(plot, results) - plot_fuel_plotly!(plot, results) Plots a stack plot of the results by fuel type onto an existing plot handle and -assigns each fuel type a specific color. The `_plotly` variant renders with the -PlotlyLight backend instead of CairoMakie. +assigns each fuel type a specific color. Pass the `backend` key word to pick the +renderer. # Arguments @@ -809,31 +1645,25 @@ PlotlyLight backend instead of CairoMakie. # Accepted Key Words - `generator_mapping_file` = "file_path" : file path to yaml defining generator category by fuel and primemover -- `variables::Union{Nothing, Vector{Symbol}}` = nothing : specific variables to plot -- `slacks::Bool = true` : display slack variables +- `slacks::Bool = true` : display the system balance slack variables as "Unserved Energy" and "Over Generation". Nodal and area formulations attach one slack per bus/area; those are summed into a single series per direction. Reactive slacks (the `"Q"` meta of full AC formulations) are excluded, since this is an active-power plot. - `load::Bool = true` : display load line - `curtailment::Bool = true`: To plot the curtailment in the stack plot -- `set_display::Bool = true`: set to false to prevent the plots from displaying -- `save::String = "file_path"`: set a file path to save the plots -- `format::String = "png"`: file extension for saved plots. CairoMakie supports `"png"`, `"pdf"`, `"svg"`. PlotlyLight only supports `"html"` (other values are written as `.html` with a warning). -- `seriescolor::Array`: Set different colors for the plots -- `title::String = "Title"`: Set a title for the plots -- `stack::Bool = true`: stack plot traces -- `bar::Bool` : create bar plot -- `nofill::Bool` : force empty area fill -- `stair::Bool`: Make a stair plot instead of a stack plot -- `label_fn::Function = label_short`: function applied to legend labels (typically the raw `Variable__Component` strings produced by PowerAnalytics). Built-in options: `label_short`, `label_component`, `label_variable`, `label_acronym`, `label_first_word`, `label_truncate(n)`. Note that when `combine_categories = true` (the default for `plot_powerdata`, `plot_results`, and `plot_fuel`), columns are aggregated to category names *before* `label_fn` runs — those names don't contain `__`, so the default `label_short` is a no-op. Pass `combine_categories = false` to see the effect of `label_fn` on the raw labels. -- `legend_position::Symbol = :right`: legend placement, `:right` or `:bottom` -- `legend_font_size::Number`: override the legend label font size +- `storage::Bool = true`: include storage components (as " In"/" Out" traces) +- `sources::Bool = true`: include source components (as " In"/" Out" traces) +- `initial_time::DateTime`: To start the plot at a different time other than the results initial time (`start_time` is accepted as an alias) +- `horizon::Int64`: number of time periods to plot, counted from `initial_time` (`len` is accepted as an alias) +$(_COMMON_PLOT_KWARGS) - `filter_func::Function = `[`PowerSystems.get_available`](@extref PowerSystems InfrastructureSystems.get_available-Tuple{RenewableDispatch}): filter components included in plot - `palette` : Color palette as from [`load_palette`](@ref). +$(_BACKEND_KWARG) """ -function plot_fuel!(p, result::IS.Results; kwargs...) - return _plot_fuel!(p, result, CairoMakieBackend(); kwargs...) -end - -@doc (@doc plot_fuel!) function plot_fuel_plotly!(p, result::IS.Results; kwargs...) - return _plot_fuel!(p, result, PlotlyLightBackend(); kwargs...) +function plot_fuel!( + p, + result::IS.Results; + backend::PlottingBackend = CairoMakieBackend(), + kwargs..., +) + return _plot_fuel!(p, result, backend; kwargs...) end """ @@ -854,7 +1684,7 @@ PlotlyLight plots dispatch to the PlotlyLight writer (html). res = solve_op_problem!(OpProblem) plot = plot_fuel(res) save_plot(plot, "my_plot.png") # CairoMakie -plot = plot_fuel_plotly(res) +plot = plot_fuel(res; backend = PlotlyLightBackend()) save_plot(plot, "my_plot.html") # PlotlyLight ``` diff --git a/src/definitions.jl b/src/definitions.jl index 5913fc6..bf6183e 100644 --- a/src/definitions.jl +++ b/src/definitions.jl @@ -53,6 +53,8 @@ end const PALETTE = load_palette() +# Unused inside PowerGraphics since both backends default to the full palette; +# kept because it is not underscore-prefixed and downstream code may call it. function get_default_palette(palette) default_palette = PaletteColor[] default_order = [6, 52, 14, 1, 32, 7, 18, 20, 27, 53, 17] # the default order from the color palette # @@ -81,10 +83,7 @@ function get_palette_cairomakie(palette) end function get_palette_plotly(palette) - getfield.(get_default_palette(palette), :RGB) -end - -function get_palette_plotly_fuel(palette) + # PlotlyLight expects colors as RGB strings. getfield.(palette, :RGB) end @@ -92,6 +91,11 @@ function get_palette_category(palette) getfield.(palette, :category) end +# Default series colors for a backend. Both backends select the *same* colors — +# the whole palette, so more series get a distinct color before the cycle +# repeats — and differ only in the representation each plotting library wants. +# Keeping the selection here (rather than in the two recipes) is what stops the +# same data from picking up different colors depending on the backend. function get_palette_seriescolor(backend::CairoMakieBackend, palette) return get_palette_cairomakie(palette) end @@ -122,24 +126,14 @@ function _match_fuel_colors(names, palette, color_range, fallback_colors) return default end +# One method for every backend: the palette selection is identical and the +# per-library color representation is already handled by the dispatched +# `get_palette_seriescolor`, so there is nothing left for a backend to override. function match_fuel_colors( data::DataFrames.DataFrame, - backend::CairoMakieBackend; + backend::PlottingBackend; palette = PALETTE, ) - colors = get_palette_cairomakie(palette) + colors = get_palette_seriescolor(backend, palette) return _match_fuel_colors(DataFrames.names(data), palette, colors, colors) end - -function match_fuel_colors( - data::DataFrames.DataFrame, - backend::PlotlyLightBackend; - palette = PALETTE, -) - return _match_fuel_colors( - DataFrames.names(data), - palette, - get_palette_plotly_fuel(palette), - get_palette_plotly(palette), - ) -end diff --git a/src/deprecated.jl b/src/deprecated.jl new file mode 100644 index 0000000..63f7244 --- /dev/null +++ b/src/deprecated.jl @@ -0,0 +1,236 @@ +# BEGIN 0.23.0 deprecations + +# Shared by every deprecated `_plotly`-suffixed shim. The backend is a value — +# `src/backends.jl` already models it as one — so encoding it in the function +# name doubled the public API without buying any dispatch; the `_plotly` names +# now forward to the un-suffixed function with `backend = PlotlyLightBackend()`. +# A caller-supplied `backend` is rejected instead of silently overridden, +# because the name and the key word would then disagree about which backend to +# use, and a shim that ignored the key word would be the worse surprise. +function _plotly_suffix_backend(old::String, new::String, kwargs) + haskey(kwargs, :backend) && throw( + ArgumentError( + "`$old` always renders with `PlotlyLightBackend()` and does not accept a " * + "`backend` key word; call `$new(...; backend = ...)` instead.", + ), + ) + @warn "`$old` is deprecated; call `$new(...; backend = PlotlyLightBackend())` " * + "instead. The `_plotly`-suffixed names will be removed in a future " * + "breaking release." + return PlotlyLightBackend() +end + +function _warn_plot_powerdata_deprecated(name::String, replacement::String) + @warn "$name(::PowerAnalytics.PowerData) is deprecated because PowerAnalytics' " * + "PowerData predates its 1.0 metrics API; use $replacement with a " * + "`Dict{String, DataFrame}` (or `plot_dataframe` for a single DataFrame) " * + "instead. This method will be removed in a future breaking release." + return +end + +# Forward a `PA.PowerData` to the dict-of-DataFrames shape the `plot_results` +# pipeline consumes: keys become strings and any `DateTime` columns are +# stripped, while the time axis comes from `powerdata.time`. +function _powerdata_to_results(powerdata::PA.PowerData) + return Dict{String, DataFrames.DataFrame}( + string(k) => PA.no_datetime(v) for (k, v) in powerdata.data + ) +end + +""" + plot_powerdata(powerdata) + +!!! warning "Deprecated" + This method is deprecated because `PowerAnalytics.PowerData` predates the + PowerAnalytics 1.0 metrics API. Use [`plot_results`](@ref) with a + `Dict{String, DataFrame}` (or [`plot_dataframe`](@ref) for a single + `DataFrame`) instead. It will be removed in a future breaking release. + +Makes a plot from a `PowerAnalytics.PowerData` object by forwarding its `data` +and `time` fields to the [`plot_results`](@ref) pipeline; accepts the same key +words as [`plot_results`](@ref). + +# Accepted Key Words +$(_BACKEND_KWARG) +""" +function plot_powerdata( + powerdata::PA.PowerData; + backend::PlottingBackend = CairoMakieBackend(), + kwargs..., +) + _warn_plot_powerdata_deprecated("plot_powerdata", "plot_results") + return _plot_results!( + _empty_plot(backend), + _powerdata_to_results(powerdata), + powerdata.time, + backend; + kwargs..., + ) +end + +""" + plot_powerdata!(plot, powerdata) + +!!! warning "Deprecated" + This method is deprecated because `PowerAnalytics.PowerData` predates the + PowerAnalytics 1.0 metrics API. Use [`plot_results!`](@ref) with a + `Dict{String, DataFrame}` (or [`plot_dataframe!`](@ref) for a single + `DataFrame`) instead. It will be removed in a future breaking release. + +Makes a plot from a `PowerAnalytics.PowerData` object onto an existing plot +handle by forwarding its `data` and `time` fields to the [`plot_results!`](@ref) +pipeline; accepts the same key words as [`plot_results!`](@ref). + +# Accepted Key Words +$(_BACKEND_KWARG) +""" +function plot_powerdata!( + p, + powerdata::PA.PowerData; + backend::PlottingBackend = CairoMakieBackend(), + kwargs..., +) + _warn_plot_powerdata_deprecated("plot_powerdata!", "plot_results!") + return _plot_results!( + p, + _powerdata_to_results(powerdata), + powerdata.time, + backend; + kwargs..., + ) +end + +""" + plot_demand_plotly(result) + plot_demand_plotly!(plot, result) + +!!! warning "Deprecated" + Use [`plot_demand`](@ref) / [`plot_demand!`](@ref) with + `backend = PlotlyLightBackend()` instead. The `_plotly`-suffixed names will + be removed in a future breaking release. +""" +function plot_demand_plotly(result::Union{IS.Results, PSY.System}; kwargs...) + backend = _plotly_suffix_backend("plot_demand_plotly", "plot_demand", kwargs) + return plot_demand(result; backend = backend, kwargs...) +end + +@doc (@doc plot_demand_plotly) function plot_demand_plotly!( + p, + result::Union{IS.Results, PSY.System}; + kwargs..., +) + backend = _plotly_suffix_backend("plot_demand_plotly!", "plot_demand!", kwargs) + return plot_demand!(p, result; backend = backend, kwargs...) +end + +""" + plot_dataframe_plotly(df) + plot_dataframe_plotly(df, time_range) + plot_dataframe_plotly!(plot, df) + plot_dataframe_plotly!(plot, df, time_range) + +!!! warning "Deprecated" + Use [`plot_dataframe`](@ref) / [`plot_dataframe!`](@ref) with + `backend = PlotlyLightBackend()` instead. The `_plotly`-suffixed names will + be removed in a future breaking release. +""" +function plot_dataframe_plotly(df::DataFrames.DataFrame; kwargs...) + backend = _plotly_suffix_backend("plot_dataframe_plotly", "plot_dataframe", kwargs) + return plot_dataframe(df; backend = backend, kwargs...) +end + +function plot_dataframe_plotly( + df::DataFrames.DataFrame, + time_range::Union{DataFrames.DataFrame, Array, StepRange}; + kwargs..., +) + backend = _plotly_suffix_backend("plot_dataframe_plotly", "plot_dataframe", kwargs) + return plot_dataframe(df, time_range; backend = backend, kwargs...) +end + +@doc (@doc plot_dataframe_plotly) function plot_dataframe_plotly!( + p, + df::DataFrames.DataFrame; + kwargs..., +) + backend = _plotly_suffix_backend("plot_dataframe_plotly!", "plot_dataframe!", kwargs) + return plot_dataframe!(p, df; backend = backend, kwargs...) +end + +function plot_dataframe_plotly!( + p, + variable::DataFrames.DataFrame, + time_range::Union{DataFrames.DataFrame, Array, StepRange}; + kwargs..., +) + backend = _plotly_suffix_backend("plot_dataframe_plotly!", "plot_dataframe!", kwargs) + return plot_dataframe!(p, variable, time_range; backend = backend, kwargs...) +end + +""" + plot_results_plotly(results) + plot_results_plotly!(plot, results) + +!!! warning "Deprecated" + Use [`plot_results`](@ref) / [`plot_results!`](@ref) with + `backend = PlotlyLightBackend()` instead. The `_plotly`-suffixed names will + be removed in a future breaking release. +""" +function plot_results_plotly(results::Dict{String, DataFrames.DataFrame}; kwargs...) + backend = _plotly_suffix_backend("plot_results_plotly", "plot_results", kwargs) + return plot_results(results; backend = backend, kwargs...) +end + +@doc (@doc plot_results_plotly) function plot_results_plotly!( + p, + results::Dict{String, DataFrames.DataFrame}; + kwargs..., +) + backend = _plotly_suffix_backend("plot_results_plotly!", "plot_results!", kwargs) + return plot_results!(p, results; backend = backend, kwargs...) +end + +""" + plot_fuel_plotly(result) + plot_fuel_plotly!(plot, result) + +!!! warning "Deprecated" + Use [`plot_fuel`](@ref) / [`plot_fuel!`](@ref) with + `backend = PlotlyLightBackend()` instead. The `_plotly`-suffixed names will + be removed in a future breaking release. +""" +function plot_fuel_plotly(result::IS.Results; kwargs...) + backend = _plotly_suffix_backend("plot_fuel_plotly", "plot_fuel", kwargs) + return plot_fuel(result; backend = backend, kwargs...) +end + +@doc (@doc plot_fuel_plotly) function plot_fuel_plotly!(p, result::IS.Results; kwargs...) + backend = _plotly_suffix_backend("plot_fuel_plotly!", "plot_fuel!", kwargs) + return plot_fuel!(p, result; backend = backend, kwargs...) +end + +""" + plot_powerdata_plotly(powerdata) + plot_powerdata_plotly!(plot, powerdata) + +!!! warning "Deprecated" + Deprecated twice over: `PowerAnalytics.PowerData` predates the + PowerAnalytics 1.0 metrics API, and the `_plotly` suffix has been replaced + by the `backend` key word. Use [`plot_results`](@ref) / + [`plot_results!`](@ref) with a `Dict{String, DataFrame}` and + `backend = PlotlyLightBackend()` instead. These names will be removed in a + future breaking release. +""" +function plot_powerdata_plotly(powerdata::PA.PowerData; kwargs...) + backend = _plotly_suffix_backend("plot_powerdata_plotly", "plot_powerdata", kwargs) + return plot_powerdata(powerdata; backend = backend, kwargs...) +end + +@doc (@doc plot_powerdata_plotly) function plot_powerdata_plotly!( + p, + powerdata::PA.PowerData; + kwargs..., +) + backend = _plotly_suffix_backend("plot_powerdata_plotly!", "plot_powerdata!", kwargs) + return plot_powerdata!(p, powerdata; backend = backend, kwargs...) +end diff --git a/src/label_utils.jl b/src/label_utils.jl index b76b74b..b177340 100644 --- a/src/label_utils.jl +++ b/src/label_utils.jl @@ -10,21 +10,21 @@ prefix to its acronym while keeping the full component name. # When does this fire? `label_fn` runs on the column names of the dataframe that is actually plotted. -For `plot_powerdata` / `plot_results` / `plot_fuel`, the default -`combine_categories = true` aggregates first — the resulting columns are bare -category names (e.g. `"HydroDispatch"`, `"Natural Gas"`) without the `__` -separator, so `label_short` is a no-op on them. To see shortening in action, -pass `combine_categories = false` so the raw `Variable__Component` labels reach +For `plot_results` / `plot_fuel`, the default `combine_categories = true` +aggregates first — the resulting columns are bare category names (e.g. +`"HydroDispatch"`, `"Natural Gas"`) without the `__` separator, so +`label_short` is a no-op on them. To see shortening in action, pass +`combine_categories = false` so the raw `Variable__Component` labels reach `label_fn`. # Usage ```julia -plot_powerdata(gen; combine_categories = false) # default: "APV: HydroDispatch" -plot_powerdata(gen; combine_categories = false, label_fn = label_component) # "HydroDispatch" -plot_powerdata(gen; combine_categories = false, label_fn = label_acronym) # "APV__HD" -plot_powerdata(gen; combine_categories = false, label_fn = label_truncate(20)) # truncate to 20 chars -plot_powerdata(gen; combine_categories = false, label_fn = s -> s) # original full labels +plot_results(res; combine_categories = false) # default: "APV: HydroDispatch" +plot_results(res; combine_categories = false, label_fn = label_component) # "HydroDispatch" +plot_results(res; combine_categories = false, label_fn = label_acronym) # "APV__HD" +plot_results(res; combine_categories = false, label_fn = label_truncate(20)) # truncate to 20 chars +plot_results(res; combine_categories = false, label_fn = s -> s) # original full labels ``` """ @@ -133,11 +133,11 @@ Can be composed with other label functions. # Example ```julia -plot_powerdata(gen; label_fn = label_truncate(20)) +plot_results(res; label_fn = label_truncate(20)) # "ActivePowerVariable…" # Compose with label_short: -plot_powerdata(gen; label_fn = s -> label_truncate(15)(label_short(s))) +plot_results(res; label_fn = s -> label_truncate(15)(label_short(s))) ``` """ function label_truncate(n::Int) diff --git a/test/plot_introspection.jl b/test/plot_introspection.jl new file mode 100644 index 0000000..40968a0 --- /dev/null +++ b/test/plot_introspection.jl @@ -0,0 +1,289 @@ +# Backend-agnostic introspection of a PowerGraphics plot object. +# +# Value assertions used to be written against `PlotlyLight.Plot.data` only, so +# CairoMakie was covered by shallow counts and a numeric regression could be +# fixed in one backend while staying broken in the other (this is exactly how +# PR #140's bar-plot bug survived). These helpers read the drawn series back out +# of either backend's plot object so the same assertion can run against both. +# +# The two object models are genuinely different, so the extraction is documented +# per case below rather than pretended to be identical. + +const CairoMakiePlot = Base.get_extension(PowerGraphics, :CairoMakieExt).CairoMakiePlot + +""" +One drawn series read back out of a plot object. + +- `label`: the legend label the series was drawn with. +- `values`: the y-values, see [`series_ydata`](@ref) for what "y-values" means + per backend and per `kind`. +- `color`: the series color canonicalized to `(r, g, b)` bytes in `0:255`, so a + CairoMakie `Colors.RGBA` and a PlotlyLight `"rgba(r, g, b, a)"` string compare + equal when they select the same palette entry. +- `kind`: `:line`, `:stairs`, `:band`, `:bar` (CairoMakie) or `:scatter`, + `:bar` (PlotlyLight). +- `linewidth`: the drawn line width, or `nothing` for marks that carry none + (bands and bars on either backend). +""" +struct PlotSeries + label::String + values::Vector{Float64} + color::Union{NTuple{3, Int}, Nothing} + kind::Symbol + linewidth::Union{Float64, Nothing} +end + +########################### color canonicalization ########################### + +# CairoMakie hands back a `Colorant`; `band!` wraps it as `(color, alpha)`; +# PlotlyLight keeps the palette's `"rgba(r, g, b, a)"` string, and callers may +# pass a named color such as `"black"` (the net-load overlay does). All four +# reduce to the same `(r, g, b)` byte triple. +_canonical_color(c::PowerGraphics.Colors.Colorant) = ( + round(Int, 255 * PowerGraphics.Colors.red(c)), + round(Int, 255 * PowerGraphics.Colors.green(c)), + round(Int, 255 * PowerGraphics.Colors.blue(c)), +) + +_canonical_color(c::Tuple) = _canonical_color(first(c)) + +function _canonical_color(s::AbstractString) + m = match(r"^rgba?\(\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)", s) + if isnothing(m) + return _canonical_color(parse(PowerGraphics.Colors.RGB, s)) + end + return (parse(Int, m[1]), parse(Int, m[2]), parse(Int, m[3])) +end + +# Anything else (e.g. a colormap symbol) is reported as "no comparable color" +# rather than guessed at. +_canonical_color(::Any) = nothing + +######################### PlotlyLight plot objects ########################### + +""" + plot_series(plot) + +The drawn series of `plot`, in draw order, as [`PlotSeries`](@ref). + +**PlotlyLight**: one entry per trace in `plot.data`; `values` is the trace's +stored `y`, always the series' own (raw) data, because Plotly stacks at render +time through `stackgroup`. + +**CairoMakie**: one entry per *labeled* plot object on `plot.axis`; unlabeled +objects are skipped, which drops the companion `band!` of a stacked stair plot. +`values` depends on the mark: + +- `:line` / `:stairs` — the y-values as drawn, which under a stacked `nofill` or + `stair` call is the *cumulative outer envelope*, not the series' own data. +- `:band` — the series' own contribution, recovered from the stored + `(lower, upper)` envelopes. The stack baseline is whichever envelope sits + nearer zero, so `sum(abs, upper) < sum(abs, lower)` identifies a + downward-stacked band; the comparison is scale-free, which matters because a + category generating nothing sums to float noise and is stacked downward. +- `:bar` — the bar heights. A stacked bar plot is a *single* `barplot!` with + vector attributes, flattened here into one `PlotSeries` per bar to line up + with PlotlyLight's one-trace-per-series output. +""" +function plot_series(plot::PlotlyLight.Plot) + return [_plotly_series(trace) for trace in plot.data] +end + +# `type = "bar"` traces keep their color under `marker`, scatter traces under +# `line`; only scatter traces carry a width. +function _plotly_series(trace) + is_bar = get(trace, :type, "scatter") == "bar" + color = if is_bar + haskey(trace, :marker) ? _canonical_color(trace.marker.color) : nothing + else + haskey(trace, :line) ? _canonical_color(trace.line.color) : nothing + end + linewidth = if !is_bar && haskey(trace, :line) && haskey(trace.line, :width) + Float64(trace.line.width) + else + nothing + end + return PlotSeries( + String(trace.name), + collect(Float64, trace.y), + color, + is_bar ? :bar : :scatter, + linewidth, + ) +end + +########################## CairoMakie plot objects ########################### + +function plot_series(plot::CairoMakiePlot) + series = PlotSeries[] + for mark in plot.axis.scene.plots + append!(series, _makie_series(mark)) + end + return series +end + +# A mark drawn without a `label` is decoration, not a series. +_makie_labels(mark) = + haskey(mark.attributes, :label) ? _as_label_vector(mark.label[]) : String[] + +_as_label_vector(label::AbstractString) = [String(label)] +_as_label_vector(labels::AbstractVector) = String.(labels) + +# `barplot!` takes vector attributes for a stacked bar; every other mark takes +# scalars. Dispatch keeps the two shapes apart instead of testing at run time. +_color_vector(color::AbstractVector, n::Int) = [_canonical_color(c) for c in color] +_color_vector(color, n::Int) = fill(_canonical_color(color), n) + +# Makie stores positional data as `Point{2}`; the y-component is element 2. +_ycoords(points) = [Float64(p[2]) for p in points] + +_makie_linewidth(mark) = + haskey(mark.attributes, :linewidth) ? Float64(mark.attributes[:linewidth][]) : nothing + +# Fallback: any mark PowerGraphics does not draw contributes no series. +_makie_series(::Any) = PlotSeries[] + +function _makie_series(mark::Makie.Lines) + return _makie_point_series(mark, :line) +end + +function _makie_series(mark::Makie.Stairs) + return _makie_point_series(mark, :stairs) +end + +function _makie_point_series(mark, kind::Symbol) + labels = _makie_labels(mark) + isempty(labels) && return PlotSeries[] + return [ + PlotSeries( + only(labels), + _ycoords(mark[1][]), + _canonical_color(mark.attributes[:color][]), + kind, + _makie_linewidth(mark), + ), + ] +end + +function _makie_series(mark::Makie.Band) + labels = _makie_labels(mark) + isempty(labels) && return PlotSeries[] + lower = _ycoords(mark[1][]) + upper = _ycoords(mark[2][]) + # See `plot_series`: the stack baseline is whichever envelope sits nearer + # zero, and a downward-stacked band is baselined on its upper envelope. + values = if sum(abs, upper) < sum(abs, lower) + lower .- upper + else + upper .- lower + end + return [ + PlotSeries( + only(labels), + values, + _canonical_color(mark.attributes[:color][]), + :band, + nothing, + ), + ] +end + +function _makie_series(mark::Makie.BarPlot) + labels = _makie_labels(mark) + isempty(labels) && return PlotSeries[] + heights = _ycoords(mark[1][]) + colors = _color_vector(mark.attributes[:color][], length(labels)) + return [ + PlotSeries(labels[ix], [heights[ix]], colors[ix], :bar, nothing) for + ix in eachindex(labels) + ] +end + +############################## shared accessors ############################## + +""" + series_labels(plot) + +Ordered legend labels of the drawn series — the backend's draw order, which is +`_series_draw_order` (net-negative series first) for every non-bar plot. +""" +series_labels(plot) = [s.label for s in plot_series(plot)] + +""" + series_count(plot) + +Number of drawn series. Counts what is actually on the plot, so it is an +independent check of CairoMakie's own `series_count` bookkeeping field. +""" +series_count(plot) = length(plot_series(plot)) + +""" + series_ydata(plot) + +Y-values of the drawn series, in draw order. See [`plot_series`](@ref) for what +these mean per backend and mark. +""" +series_ydata(plot) = [s.values for s in plot_series(plot)] + +""" + series_colors(plot) + +Series colors as `(r, g, b)` byte triples in draw order, comparable across +backends even though CairoMakie stores `Colors.RGBA` and PlotlyLight stores +`"rgba(…)"` strings. +""" +series_colors(plot) = [s.color for s in plot_series(plot)] + +""" + series_linewidths(plot) + +Drawn line widths in draw order; `nothing` for bands and bars, which carry none. +""" +series_linewidths(plot) = [s.linewidth for s in plot_series(plot)] + +""" + series_map(plot) + +`label => values` for every drawn series. Throws if two series share a label, +because a duplicate label would silently drop coverage. +""" +function series_map(plot) + out = Dict{String, Vector{Float64}}() + for s in plot_series(plot) + haskey(out, s.label) && + error("duplicate series label $(repr(s.label)) in plot introspection") + out[s.label] = s.values + end + return out +end + +""" + series_values(plot, label) + +Y-values of the single series drawn with `label`. +""" +function series_values(plot, label::AbstractString) + matches = [s for s in plot_series(plot) if s.label == label] + length(matches) == 1 || error( + "expected exactly one series labeled $(repr(label)), found $(length(matches))", + ) + return only(matches).values +end + +""" + plot_title(plot) + +The visible plot title, or `nothing` when the plot carries none. CairoMakie +leaves `Axis.title` as `""` when unset and PlotlyLight omits `layout.title` +entirely; both are reported as `nothing`. +""" +function plot_title(plot::CairoMakiePlot) + title = plot.axis.title[] + return isempty(title) ? nothing : String(title) +end + +function plot_title(plot::PlotlyLight.Plot) + haskey(plot.layout, :title) || return nothing + haskey(plot.layout.title, :text) || return nothing + return String(plot.layout.title.text) +end diff --git a/test/runtests.jl b/test/runtests.jl index 2257df7..8c188b4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -42,6 +42,11 @@ const generic_template = joinpath(template_dir, "generic_report_template.jmd") PA_DIR = string(dirname(dirname(pathof(PowerAnalytics)))) include(joinpath(PA_DIR, "test", "test_data", "results_data.jl")) +# Shared helpers, not a test file: `@includetests` only globs `test_*.jl`, so +# this has to be included explicitly, and it must be included here rather than +# from a test file so that running a single file still gets it. +include(joinpath(TEST_DIR, "plot_introspection.jl")) + LOG_LEVELS = Dict( "Debug" => Logging.Debug, "Info" => Logging.Info, diff --git a/test/test_backend_parity.jl b/test/test_backend_parity.jl new file mode 100644 index 0000000..6546296 --- /dev/null +++ b/test/test_backend_parity.jl @@ -0,0 +1,425 @@ +# Regression tests for the two refactors that unified the backends: +# +# 1. the `_plotly`-suffixed API collapsed into a `backend` key word, with the +# old names kept as deprecated shims, and +# 2. eight per-plot behaviors (fill default, line width, line style, draw +# order, title sentinel, empty input, default save format, palette +# selection) resolved once in `src/call_plots.jl` instead of twice in the +# recipes. +# +# Both refactors are only worth anything if the two backends now agree, so the +# assertions here are written through the backend-agnostic helpers in +# `plot_introspection.jl` and compare CairoMakie against PlotlyLight directly. + +const PARITY_BACKENDS = + (("cairomakie", CairoMakieBackend()), ("plotlylight", PlotlyLightBackend())) +const PARITY_EXTENSION = Dict("cairomakie" => ".png", "plotlylight" => ".html") + +parity_time() = collect(range(DateTime("2024-01-01T00:00:00"); step = Hour(1), length = 6)) + +# All-positive columns, so the draw order is the column order and the palette +# selection can be checked position by position. +function parity_dataframe() + return DataFrame( + "alpha" => [1.0, 2.0, 3.0, 4.0, 5.0, 6.0], + "beta" => [6.0, 5.0, 4.0, 3.0, 2.0, 1.0], + "gamma" => [0.5, 0.5, 0.5, 0.5, 0.5, 0.5], + ) +end + +# Mixed signs: "charge" and "spill" are net-negative, so both backends must draw +# them first (they stack below the zero axis and would otherwise be hidden +# behind the positive bands). +function parity_signed_dataframe() + return DataFrame( + "thermal" => [5.0, 6.0, 7.0, 8.0, 9.0, 10.0], + "charge" => [-1.0, -2.0, 0.0, -1.0, -0.5, -0.5], + "wind" => [2.0, 2.0, 2.0, 2.0, 2.0, 2.0], + "spill" => [0.0, -1.0, -1.0, 0.0, -2.0, -1.0], + ) +end + +function parity_dataframe_with_time() + df = parity_dataframe() + DataFrames.insertcols!(df, 1, "DateTime" => parity_time()) + return df +end + +# `plot_results` consumes a dict of DataFrames that each carry their own +# DateTime column. +function parity_results_dict() + df = parity_dataframe_with_time() + return Dict{String, DataFrames.DataFrame}( + "Thermal" => df[!, ["DateTime", "alpha", "beta"]], + "Wind" => df[!, ["DateTime", "gamma"]], + ) +end + +# Two plots are "the same plot" when they draw the same series, in the same +# order, with the same colors and the same values. That is exactly the contract +# a deprecated shim owes its replacement. +function assert_same_plot(a, b) + @test series_labels(a) == series_labels(b) + @test series_colors(a) == series_colors(b) + ya, yb = series_ydata(a), series_ydata(b) + @test length(ya) == length(yb) + for (va, vb) in zip(ya, yb) + @test va ≈ vb + end +end + +# Call `f` asserting that it logs a deprecation warning, and return its value. +# `@test_logs` swallows the record, which also keeps the deprecation noise out +# of the suite's log-event tracker. +test_deprecated(f::Function) = @test_logs (:warn, r"deprecated") match_mode = :any f() + +@testset "deprecated _plotly shims forward to the PlotlyLight backend" begin + (results_uc, _) = run_test_sim(TEST_RESULT_DIR, TEST_SIM_NAME) + gen_uc = get_generation_data(results_uc) + df = parity_dataframe() + df_dt = parity_dataframe_with_time() + time = parity_time() + results_dict = parity_results_dict() + plotly = PlotlyLightBackend() + fresh() = PG._empty_plot(plotly) + + # Every shim must produce exactly what the un-suffixed function with + # `backend = PlotlyLightBackend()` produces, and must say it is deprecated. + assert_same_plot( + test_deprecated(() -> plot_dataframe_plotly(df_dt; set_display = false)), + plot_dataframe(df_dt; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_dataframe_plotly(df, time; set_display = false)), + plot_dataframe(df, time; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_dataframe_plotly!(fresh(), df_dt; set_display = false)), + plot_dataframe!(fresh(), df_dt; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated( + () -> plot_dataframe_plotly!(fresh(), df, time; set_display = false), + ), + plot_dataframe!(fresh(), df, time; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_results_plotly(results_dict; set_display = false)), + plot_results(results_dict; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated( + () -> plot_results_plotly!(fresh(), results_dict; set_display = false), + ), + plot_results!(fresh(), results_dict; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_demand_plotly(results_uc; set_display = false)), + plot_demand(results_uc; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated( + () -> plot_demand_plotly!(fresh(), results_uc; set_display = false), + ), + plot_demand!(fresh(), results_uc; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_fuel_plotly(results_uc; set_display = false)), + plot_fuel(results_uc; backend = plotly, set_display = false), + ) + assert_same_plot( + test_deprecated(() -> plot_fuel_plotly!(fresh(), results_uc; set_display = false)), + plot_fuel!(fresh(), results_uc; backend = plotly, set_display = false), + ) + # `_report_plot_fuel` is private but reachable from report templates copied + # out of an earlier release, which call it positionally. It has no other + # caller in the repository, so without this it reads as dead code and gets + # deleted — which is exactly what happened once already. + assert_same_plot( + PG._report_plot_fuel(plotly, results_uc; set_display = false), + plot_fuel(results_uc; backend = plotly, set_display = false), + ) + # `plot_powerdata` is deprecated twice over, so both layers warn. + assert_same_plot( + test_deprecated(() -> PG.plot_powerdata_plotly(gen_uc; set_display = false)), + test_deprecated( + () -> PG.plot_powerdata(gen_uc; backend = plotly, set_display = false), + ), + ) + assert_same_plot( + test_deprecated( + () -> PG.plot_powerdata_plotly!(fresh(), gen_uc; set_display = false), + ), + test_deprecated( + () -> + PG.plot_powerdata!(fresh(), gen_uc; backend = plotly, set_display = false), + ), + ) + + # A shim carries its backend in its name, so accepting a `backend` key word + # too would leave the name and the key word free to disagree. Rejecting it + # is the contract; silently overriding the caller would be worse. + @test_throws ArgumentError plot_dataframe_plotly(df, time; backend = plotly) + @test_throws ArgumentError plot_dataframe_plotly(df_dt; backend = plotly) + @test_throws ArgumentError plot_dataframe_plotly!(fresh(), df_dt; backend = plotly) + @test_throws ArgumentError plot_dataframe_plotly!(fresh(), df, time; backend = plotly) + @test_throws ArgumentError plot_results_plotly(results_dict; backend = plotly) + @test_throws ArgumentError plot_results_plotly!(fresh(), results_dict; backend = plotly) + @test_throws ArgumentError plot_demand_plotly(results_uc; backend = plotly) + @test_throws ArgumentError plot_demand_plotly!(fresh(), results_uc; backend = plotly) + @test_throws ArgumentError plot_fuel_plotly(results_uc; backend = plotly) + @test_throws ArgumentError plot_fuel_plotly!(fresh(), results_uc; backend = plotly) + @test_throws ArgumentError PG.plot_powerdata_plotly(gen_uc; backend = plotly) + @test_throws ArgumentError PG.plot_powerdata_plotly!(fresh(), gen_uc; backend = plotly) + # Passing a CairoMakie backend to a `_plotly` name must be rejected on the + # same grounds, not quietly honored. + @test_throws ArgumentError plot_dataframe_plotly( + df, + time; + backend = CairoMakieBackend(), + ) +end + +@testset "default save format follows the backend" begin + df = parity_dataframe() + time = parity_time() + out_path = joinpath(TEST_OUTPUTS, "parity_save") + isdir(out_path) && rm(out_path; recursive = true) + mkpath(out_path) + + for (backend_pkg, backend) in PARITY_BACKENDS + dir = joinpath(out_path, backend_pkg) + mkpath(dir) + # A shared "png" default would make every default-path PlotlyLight save + # trip the "only supports HTML" warning and silently rewrite the path, + # so the absence of any warning here is the point of the assertion. + @test_logs min_level = Logging.Warn plot_dataframe( + df, + time; + backend = backend, + set_display = false, + title = "defaulted", + save = dir, + ) + @test readdir(dir) == ["defaulted" * PARITY_EXTENSION[backend_pkg]] + end + + # An explicit `format` still wins over the backend default. + svg_dir = joinpath(out_path, "explicit_svg") + mkpath(svg_dir) + plot_dataframe( + df, + time; + set_display = false, + title = "explicit", + save = svg_dir, + format = "svg", + ) + @test readdir(svg_dir) == ["explicit.svg"] + + # Extension matching must be case-insensitive on both backends. CairoMakie + # lowercases before checking, so an uppercase `.HTML` has to be recognized as + # HTML by PlotlyLight too rather than treated as an unsupported extension and + # silently rewritten to a different path than the caller asked for. + upper_path = joinpath(out_path, "upper.HTML") + pl_plot = plot_dataframe(df, time; backend = PlotlyLightBackend(), set_display = false) + @test @test_logs min_level = Logging.Warn save_plot(pl_plot, upper_path) == upper_path + @test isfile(upper_path) + + # CairoMakie cannot write HTML at all, so it must say so rather than write a + # PNG under an .html name. + cm_plot = plot_dataframe(df, time; set_display = false) + @test_throws ArgumentError save_plot(cm_plot, joinpath(out_path, "nope.html")) + @test_throws ArgumentError plot_dataframe( + df, + time; + set_display = false, + title = "nope", + save = out_path, + format = "html", + ) + + # PlotlyLight can only write HTML, so a non-html extension is warned about + # and rewritten rather than dropped. + pl_plot = plot_dataframe(df, time; backend = PlotlyLightBackend(), set_display = false) + rewritten = @test_logs (:warn, r"only supports HTML") match_mode = :any save_plot( + pl_plot, + joinpath(out_path, "rewritten.pdf"), + ) + @test rewritten == joinpath(out_path, "rewritten.html") + @test isfile(rewritten) + @test !isfile(joinpath(out_path, "rewritten.pdf")) + + # `_resolve_save_file` is the only place that builds a save path, so every + # entry point saves once and under one filename convention. + spaced_dir = joinpath(out_path, "spaced") + mkpath(spaced_dir) + plot_dataframe(df, time; set_display = false, title = "My Plot", save = spaced_dir) + @test readdir(spaced_dir) == ["My_Plot.png"] + + results_dir = joinpath(out_path, "results_save") + mkpath(results_dir) + plot_results( + parity_results_dict(); + set_display = false, + title = "My Results", + save = results_dir, + ) + @test readdir(results_dir) == ["My_Results.png"] + + @info("removing test files") + rm(out_path; recursive = true) +end + +@testset "linewidth is honored by both backends" begin + df = parity_dataframe() + time = parity_time() + # PlotlyLight used to drop `linewidth` on the floor (it only read the + # PlotlyLight-specific spelling), so a caller got a hairline plot on one + # backend and a thick one on the other from identical code. + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe(df, time; backend = backend, set_display = false, linewidth = 7) + @test series_linewidths(p) == [7.0, 7.0, 7.0] + end + + # The default is 1 on both, so a caller who passes nothing gets matching + # plots too. + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe(df, time; backend = backend, set_display = false) + @test series_linewidths(p) == [1.0, 1.0, 1.0] + end +end + +@testset "series draw order is identical across backends" begin + df = parity_signed_dataframe() + time = parity_time() + # Net-negative series first, each group keeping its column order. + expected = ["charge", "spill", "thermal", "wind"] + @test PG._series_draw_order(Matrix(df)) == [2, 4, 1, 3] + + # The plain (non-stacked, non-filled) branch is included on purpose: + # CairoMakie used to reorder only in its stacked branches, so a plain line + # plot came out in a different order than the same call on PlotlyLight. + for mode in ( + (), + (:stack => true,), + (:stack => true, :nofill => true), + (:stair => true,), + (:stack => true, :stair => true), + ) + for (backend_pkg, backend) in PARITY_BACKENDS + p = plot_dataframe(df, time; backend = backend, set_display = false, mode...) + @test series_labels(p) == expected + end + end + + # Bar plots aggregate over time into one value per category and are drawn in + # column order on both backends, so they must NOT be reordered. + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe( + df, + time; + backend = backend, + set_display = false, + bar = true, + stack = true, + ) + @test series_labels(p) == DataFrames.names(df) + end +end + +@testset "blank and omitted titles are treated as no title" begin + df = parity_dataframe() + time = parity_time() + out_path = joinpath(TEST_OUTPUTS, "parity_title") + isdir(out_path) && rm(out_path; recursive = true) + mkpath(out_path) + + for (backend_pkg, backend) in PARITY_BACKENDS + ext = PARITY_EXTENSION[backend_pkg] + for (tag, title_kwargs) in (("omitted", ()), ("sentinel", (:title => " ",))) + dir = joinpath(out_path, backend_pkg * "_" * tag) + mkpath(dir) + p = plot_dataframe( + df, + time; + backend = backend, + set_display = false, + save = dir, + title_kwargs..., + ) + # `" "` is the old spelling of "this plot has no title"; neither it + # nor an omitted title may reach the rendered figure. + @test isnothing(plot_title(p)) + # An untitled plot still needs a deterministic file name. + @test readdir(dir) == ["dataframe" * ext] + end + + # A real title is kept, both on the figure and in the file name. + dir = joinpath(out_path, backend_pkg * "_titled") + mkpath(dir) + p = plot_dataframe( + df, + time; + backend = backend, + set_display = false, + save = dir, + title = "Real Title", + ) + @test plot_title(p) == "Real Title" + # The title reaches the figure verbatim but the file name replaces + # spaces with underscores, which is what every entry point has always + # done. + @test readdir(dir) == ["Real_Title" * ext] + end + + @info("removing test files") + rm(out_path; recursive = true) +end + +@testset "an empty dataframe warns and leaves the plot untouched" begin + df = parity_dataframe() + time = parity_time() + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe(df, time; backend = backend, set_display = false) + before_labels = series_labels(p) + before_values = series_ydata(p) + + returned = + @test_logs (:warn, r"Plot dataframe empty") match_mode = :any plot_dataframe!( + p, + DataFrames.DataFrame(), + time; + backend = backend, + set_display = false, + ) + # The same handle comes back, with nothing added and nothing redrawn. + @test returned === p + @test series_labels(p) == before_labels + @test series_ydata(p) == before_values + end +end + +@testset "both backends select the same default palette colors" begin + df = parity_dataframe() + time = parity_time() + # The representations differ by design (`Colors.RGBA` for CairoMakie, + # `"rgba(…)"` strings for PlotlyLight), so the assertion is on the palette + # *selection*: series `i` takes palette entry `i`, on both backends. + expected = [_canonical_color(c.color) for c in PG.PALETTE[1:DataFrames.ncol(df)]] + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe(df, time; backend = backend, set_display = false) + @test series_colors(p) == expected + end + + # More series than palette entries cycles back to the start rather than + # falling off the end, identically on both backends. + wide = DataFrames.DataFrame([ + "c$ix" => fill(Float64(ix), length(time)) for ix in 1:(length(PG.PALETTE) + 2) + ],) + wide_expected = [_canonical_color(c.color) for c in vcat(PG.PALETTE, PG.PALETTE[1:2])] + for (_, backend) in PARITY_BACKENDS + p = plot_dataframe(wide, time; backend = backend, set_display = false) + @test series_colors(p) == wide_expected + end +end diff --git a/test/test_demand_semantics.jl b/test/test_demand_semantics.jl new file mode 100644 index 0000000..df22e16 --- /dev/null +++ b/test/test_demand_semantics.jl @@ -0,0 +1,246 @@ +# Regression tests for the demand data contract on `IS.Results`. +# +# PowerGraphics reads load variable-first (`calc_active_power`, falling back to +# `calc_load_forecast`), which is what the old `PA.get_load_data` did. The +# fixtures below exist because that order is only observable when a load is +# modeled with a controllable formulation: under `PowerLoadInterruption` the +# `ActivePowerVariable` is the *served* load, while PowerSimulations stores the +# `ActivePowerTimeSeriesParameter` with the opposite sign than it does under +# `StaticPowerLoad`. Reading the forecast alone therefore plots the wrong +# quantity and the wrong sign, and on a mixed static/controllable system the two +# sign conventions cancel — which is what these tests pin down. The rest of the +# suite runs on an all-static fixture where both readings coincide. + +# `n_interruptible` of the three 5-bus loads are rebuilt as +# `InterruptiblePowerLoad`; `capacity_scale` throttles thermal capacity so that +# the solver actually sheds load and served ≠ forecast. +function build_interruptible_load_system(; n_interruptible::Int, capacity_scale::Float64) + sys = deepcopy(PSB.build_system(PSB.PSITestSystems, "c_sys5_uc")) + for old in collect(get_components(PowerLoad, sys))[1:n_interruptible] + new = InterruptiblePowerLoad(; + name = get_name(old), + available = true, + bus = get_bus(old), + active_power = get_active_power(old), + reactive_power = get_reactive_power(old), + max_active_power = get_max_active_power(old), + max_reactive_power = get_max_reactive_power(old), + base_power = get_base_power(old), + operation_cost = LoadCost(; + variable = CostCurve(LinearCurve(1000.0)), + fixed = 0.0, + ), + ) + add_component!(sys, new) + copy_time_series!(new, old) + remove_component!(sys, old) + end + for g in get_components(ThermalStandard, sys) + lims = get_active_power_limits(g) + set_active_power_limits!(g, (min = 0.0, max = lims.max * capacity_scale)) + set_rating!(g, get_rating(g) * capacity_scale) + end + return sys +end + +function solve_interruptible_load_problem(sys) + template = ProblemTemplate(NetworkModel(CopperPlatePowerModel; use_slacks = false)) + set_device_model!(template, ThermalStandard, ThermalBasicUnitCommitment) + set_device_model!(template, RenewableDispatch, RenewableFullDispatch) + set_device_model!(template, RenewableNonDispatch, FixedOutput) + set_device_model!(template, PowerLoad, StaticPowerLoad) + set_device_model!(template, InterruptiblePowerLoad, PowerLoadInterruption) + prob = DecisionModel( + template, + sys; + optimizer = optimizer_with_attributes(HiGHS.Optimizer, "mip_rel_gap" => 0.01), + horizon = Hour(12), + ) + build!(prob; output_dir = mktempdir()) + solve!(prob) + return OptimizationProblemResults(prob) +end + +# Total demand per timestep the way the old PowerAnalytics pipeline reported it. +# Categories emptied out by a `filter_func` are skipped: upstream +# `PA.combine_categories` throws a `MethodError` on those. +function old_api_demand(res; filter_func = nothing) + data = if isnothing(filter_func) + get_load_data(res) + else + get_load_data(res; filter_func = filter_func) + end + total = Float64[] + for (_, df) in data.data + cols = no_datetime(df) + ncol(cols) == 0 && continue + vals = vec(sum(Matrix(cols); dims = 2)) + if isempty(total) + total = vals + else + total .+= vals + end + end + return total +end + +# Read through the backend-agnostic harness in `plot_introspection.jl` rather +# than off `PlotlyLight.Plot.data`: a value assertion written against one +# backend's object model is exactly what lets a regression survive in the other. +demand_trace(p) = series_values(p, "Load") + +# The `PSY.System` path aggregates per load rather than into a single "Load" +# column, so its window has to be read off the summed traces. +total_trace(p) = sum(series_ydata(p)) + +# Float-noise slack on comparisons of MW totals: 1e-6 MW is 1 W, i.e. 1e-8 per +# unit on the 100 MVA system base, far below anything the solver resolves. +const DEMAND_MW_TOL = 1.0e-6 + +@testset "demand on a mixed static + controllable load system" begin + res = solve_interruptible_load_problem( + build_interruptible_load_system(; n_interruptible = 2, capacity_scale = 0.55), + ) + p = plot_demand(res; backend = PG.PlotlyLightBackend(), set_display = false) + plotted = demand_trace(p) + + # Sign contract. Reading `calc_load_forecast` alone returns the static loads + # positive and the controllable loads negative, so the aggregate came out + # negative before the variable-first fallback existed. + @test all(>=(0.0), plotted) + + # Magnitude contract against the old pipeline, per timestep. + @test plotted ≈ old_api_demand(res) + + # The fixture has teeth only if the solver actually shed load, i.e. served + # demand is strictly below the forecast for at least one period. + forecast = + -get_data_vec( + PA.compute( + PA.Metrics.calc_load_forecast, + res, + make_selector(InterruptiblePowerLoad; groupby = :all), + ), + ) + served = get_data_vec( + PA.compute( + PA.Metrics.calc_active_power, + res, + make_selector(InterruptiblePowerLoad; groupby = :all), + ), + ) + @test all(served .<= forecast .+ DEMAND_MW_TOL) + @test any(served .< forecast .- DEMAND_MW_TOL) + + # A whole-pool `calc_active_power` read cannot express this: the static + # loads have no `ActivePowerVariable`, so the call throws and everything + # falls back to the forecast. This is why resolution is per load type. + @test_throws Exception PA.compute( + PA.Metrics.calc_active_power, + res, + rebuild_selector(PA.Selectors.all_loads; groupby = :all), + ) + + # `filter_func` still restricts the pool, and still matches the old reader. + only_bus2 = x -> get_name(x) == "Bus2" + p_f = plot_demand( + res; + backend = PG.PlotlyLightBackend(), + set_display = false, + filter_func = only_bus2, + ) + @test demand_trace(p_f) ≈ old_api_demand(res; filter_func = only_bus2) + @test sum(demand_trace(p_f)) < sum(plotted) +end + +@testset "net-load overlay tracks served load when load is shed" begin + res = solve_interruptible_load_problem( + build_interruptible_load_system(; n_interruptible = 3, capacity_scale = 0.55), + ) + p = plot_fuel( + res; + backend = PG.PlotlyLightBackend(), + set_display = false, + auto_units = false, + ) + netload = demand_trace(p) + @test all(>=(0.0), netload) + + # With a copper-plate network, no slacks and no storage, generation equals + # served load every period, so the net-load line must sit exactly on top of + # the generation stack. Curtailment is drawn above the line, not in it. + generation = zeros(Float64, length(netload)) + for s in plot_series(p) + s.label in ("Load", "Curtailment") && continue + generation .+= s.values + end + @test generation ≈ netload +end + +@testset "plot_demand window aliases apply on the PSY.System path" begin + sys = deepcopy(PSB.build_system(PSB.PSITestSystems, "c_sys5_uc")) + initial_times = collect(get_forecast_initial_times(sys)) + t0 = initial_times[2] + + full = total_trace( + plot_demand(sys; backend = PG.PlotlyLightBackend(), set_display = false), + ) + windowed = total_trace( + plot_demand( + sys; + backend = PG.PlotlyLightBackend(), + set_display = false, + start_time = t0, + len = 3, + ), + ) + # `start_time`/`len` are documented aliases, so they must slice rather than + # be silently dropped, and must agree with the canonical spellings. + @test length(windowed) == 3 + @test length(full) > 3 + @test windowed ≈ total_trace( + plot_demand( + sys; + backend = PG.PlotlyLightBackend(), + set_display = false, + initial_time = t0, + horizon = 3, + ), + ) +end + +@testset "get_demand_data returns the numbers plot_demand draws" begin + res = solve_interruptible_load_problem( + build_interruptible_load_system(; n_interruptible = 2, capacity_scale = 0.55), + ) + + df = get_demand_data(res) + @test names(df)[1] == PA.DATETIME_COL + @test eltype(df[!, PA.DATETIME_COL]) <: Dates.DateTime + + # The reason this accessor is exported at all: it must agree with the plot, + # not merely be plausible. Anything less and callers would be better off + # reading a metric directly, which is the trap the sign fix exists to close. + p = plot_demand(res; backend = PG.PlotlyLightBackend(), set_display = false) + @test df[!, "Load"] ≈ series_values(p, "Load") + + # `aggregate` is meaningful only on the `PSY.System` path. Accepting and + # ignoring it here would hand back a single aggregated column while implying + # a per-bus breakdown, so the `IS.Results` method must not take it at all. + @test_throws MethodError get_demand_data(res; aggregate = "Bus") +end + +@testset "get_demand_data window key words and their aliases slice" begin + sys = deepcopy(PSB.build_system(PSB.PSITestSystems, "c_sys5_uc")) + t0 = collect(get_forecast_initial_times(sys))[2] + + full = get_demand_data(sys) + windowed = get_demand_data(sys; start_time = t0, len = 3) + @test DataFrames.nrow(windowed) == 3 + @test DataFrames.nrow(full) > 3 + @test windowed == get_demand_data(sys; initial_time = t0, horizon = 3) + + # The `System` path groups columns, so `aggregate` has to reach the reader. + @test names(get_demand_data(sys; aggregate = "System")) != + names(get_demand_data(sys; aggregate = "Bus")) +end diff --git a/test/test_fuel_categories.jl b/test/test_fuel_categories.jl new file mode 100644 index 0000000..9d4f783 --- /dev/null +++ b/test/test_fuel_categories.jl @@ -0,0 +1,201 @@ +# Regression tests for how `plot_fuel` recovers a generator-mapping rule's +# specificity and uses it to put each component in exactly one fuel category. +# +# The thing under test is fragile by nature: PowerAnalytics hands back one +# `ComponentSelector` per category, and PowerGraphics has to know which YAML rule +# produced each of its sub-selectors in order to replay the old first-match-wins +# ladder. Getting that wrong does not throw -- it silently files components under +# the wrong fuel. So the assertions below are written to fail if the ranking +# degrades, not merely if it errors. + +const SPECIFICITY_MAPPING = + joinpath(TEST_DIR, "test_yamls", "generator_mapping_specificity.yaml") + +# Reuses the serialized store written by the other fuel tests. +(fuelcat_results_uc, _) = run_test_sim(TEST_RESULT_DIR, TEST_SIM_NAME) +const FUELCAT_SYS = PSI.get_system(fuelcat_results_uc) + +@testset "rule specificity is recovered from the mapping, not from selector names" begin + categories = PA.parse_injector_categories(SPECIFICITY_MAPPING) + + # Order the categories with the broad rule FIRST and hand that order to + # `_assign_fuel_categories` directly. Ranking is strict (`rank < best_rank`), + # so if specificity ever collapses -- every rule looking equally broad -- the + # first-seen category wins and these assertions fail deterministically rather + # than depending on `Dict` iteration order. + ordered = [ + name => categories[name] for name in + ["BroadThermal", "NGCombustionTurbine", "CoalOnly", "Hydropower", "PV", "Wind"] + ] + thermal = collect(get_components(ThermalStandard, FUELCAT_SYS)) + @test !isempty(thermal) + + assignments, unmatched = PG._assign_fuel_categories( + fuelcat_results_uc, + ordered, + SPECIFICITY_MAPPING, + thermal, + nothing, + ) + @test isempty(unmatched) + assigned = + Dict(get_name(c) => category for (category, comps) in assignments for c in comps) + + # Prime-mover + fuel specific beats the type-only rule over the same gentype. + @test assigned["Solitude"] == "NGCombustionTurbine" + @test assigned["Alta"] == "NGCombustionTurbine" + # Fuel specificity ALONE beats the type-only rule: both rules are prime-mover + # wildcards, so this fails the moment the fuel axis stops being recovered. + @test assigned["Brighton"] == "CoalOnly" + # Nothing narrower matches these, so the broad rule is genuinely correct. + @test assigned["Park City"] == "BroadThermal" + @test assigned["Sundance"] == "BroadThermal" + + # Every component ends up in exactly one category -- the whole point of the + # ladder, since overlapping rules would otherwise double-count energy. + @test sum(length, values(assignments)) == length(thermal) +end + +@testset "mapping rules dropped by PowerAnalytics are dropped at the same position" begin + # "Hydropower" lists three rules, the first of which (`gentype: ACBus`) + # cannot intersect the `StaticInjection` root type and is discarded by + # `make_fuel_component_selector`. If PowerGraphics did not replay that drop, + # its rules would be paired with the wrong sub-selectors and the + # correspondence check would throw. + categories = PA.parse_injector_categories(SPECIFICITY_MAPPING) + groups = collect(PSY.get_groups(categories["Hydropower"], fuelcat_results_uc)) + specs = PG._mapping_rule_specs(PG.YAML.load_file(SPECIFICITY_MAPPING), "Hydropower") + @test length(specs) == length(groups) == 2 + @test first.(specs) == [PSY.HydroGen, PSY.StaticInjection] + + # And the hydro components really do land in "Hydropower" end to end. + hydro = collect(get_components(HydroGen, FUELCAT_SYS)) + @test !isempty(hydro) + assignments, unmatched = PG._assign_fuel_categories( + fuelcat_results_uc, + categories, + SPECIFICITY_MAPPING, + hydro, + nothing, + ) + @test isempty(unmatched) + @test sort(get_name.(assignments["Hydropower"])) == sort(get_name.(hydro)) +end + +@testset "broken group/rule correspondence fails loudly" begin + # A silently wrong fuel plot is the failure mode this check exists to + # prevent, so a mismatch must throw and must name the category and file. + err = try + PG._validate_rule_correspondence( + "MyCategory", + SPECIFICITY_MAPPING, + (), + Tuple{Type, Bool, Bool}[(ThermalStandard, true, true)], + ) + nothing + catch e + e + end + @test err isa ErrorException + @test occursin("MyCategory", err.msg) + @test occursin(SPECIFICITY_MAPPING, err.msg) + + # A sub-selector that is not the `FilterComponentSelector` PowerAnalytics + # builds carries no rule type, so it can never be matched to a rule. + @test PG._selector_component_type(make_selector(ThermalStandard)) === Union{} + @test_throws ErrorException PG._validate_rule_correspondence( + "MyCategory", + SPECIFICITY_MAPPING, + [make_selector(ThermalStandard)], + Tuple{Type, Bool, Bool}[(ThermalStandard, true, true)], + ) +end + +module FuelCatModA +abstract type Thermal end +struct Gen <: Thermal end +end + +module FuelCatModB +abstract type Thermal end +end + +@testset "type distance separates same-named types in different modules" begin + # PowerAnalytics' `lookup_gentype` accepts `Module.TypeName`, so two rules + # can legitimately name different types that share a `nameof`. Matching on + # the name alone ranked them identically. + @test PG._type_distance(FuelCatModA.Gen, FuelCatModA.Thermal) == 1 + @test PG._type_distance(FuelCatModA.Gen, FuelCatModB.Thermal) == typemax(Int) + + # Bare `gentype` names still work, because PowerAnalytics resolves them + # against PowerSystems before PowerGraphics ever sees a type. + @test PG._type_distance(ThermalStandard, ThermalStandard) == 0 + @test PG._type_distance(ThermalStandard, StaticInjection) < + PG._type_distance(ThermalStandard, Any) + @test PG._type_distance(ThermalStandard, HydroGen) == typemax(Int) + + # More specific rule types must outrank less specific ones for the ladder in + # `_rule_rank` to mean anything. + @test PG._type_distance(ThermalStandard, ThermalGen) < + PG._type_distance(ThermalStandard, StaticInjection) +end + +@testset "specificity mapping drives plot_fuel end to end" begin + for (backend_pkg, backend) in + (("cairomakie", CairoMakieBackend()), ("plotlylight", PlotlyLightBackend())) + p = plot_fuel( + fuelcat_results_uc; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + storage = false, + sources = false, + slacks = false, + generator_mapping_file = SPECIFICITY_MAPPING, + ) + labels = series_labels(p) + @test "NGCombustionTurbine" in labels + @test "CoalOnly" in labels + # No component may be filed under "Other": the mapping covers the whole + # generator pool, so anything landing there means a rule stopped matching. + @test !("Other" in labels) + + # Each narrow category's column is exactly the sum of its member + # components' generation. That is the property the specificity ranking + # decides: if the ranking degraded and Alta/Solitude were refiled under + # "BroadThermal", this column would stop matching its oracle. + # + # The member lists are spelled out rather than read back from + # `_assign_fuel_categories` on purpose. Deriving them from the code + # under test would move the oracle in lockstep with the defect and pin + # nothing; hardcoding them states the mapping the fixture YAML documents + # as intended, which is what a regression has to violate. + # + # Do NOT reintroduce a `sum(...) > 0` check here. Of the fixture's + # thermal units only "Brighton" is ever committed -- "Alta", "Solitude", + # "Park City" and "Sundance" all sit at zero -- so those columns are + # floating-point noise (order 1e-15 MW) whose sign flips between equally + # optimal solutions of the UC. A sign assertion on them passes or fails + # on the solver's rounding, not on this package's behavior. + for (category, members) in + (("NGCombustionTurbine", ["Alta", "Solitude"]), ("CoalOnly", ["Brighton"])) + expected = sum( + PA.get_data_vec( + PA.compute( + PA.Metrics.calc_active_power, + fuelcat_results_uc, + get_component(ThermalStandard, FUELCAT_SYS, name), + ), + ) for name in members + ) + # Absolute tolerance in MW: CairoMakie's stacked bands are read back + # by differencing cumulative envelopes, so a category's values carry + # rounding proportional to the whole stack, not to their own + # magnitude. A relative tolerance would therefore be unsatisfiable + # for the near-zero columns while 1e-8 MW stays far below any + # refiling, which moves whole units of generation. + @test isapprox(series_values(p, category), expected; atol = 1e-8) + end + end +end diff --git a/test/test_fuel_stack_behavior.jl b/test/test_fuel_stack_behavior.jl new file mode 100644 index 0000000..410b6bc --- /dev/null +++ b/test/test_fuel_stack_behavior.jl @@ -0,0 +1,465 @@ +# Behavioral tests for the fuel-stack and demand data contracts. `plot_fuel` is +# built on the PowerAnalytics metrics/selectors API, but reimplements the storage +# "In"/"Out" split, curtailment and the system-balance slacks by hand, so those +# categories need a numeric pin rather than a sign-only one. The old +# PowerAnalytics aggregation (`get_generation_data`/`categorize_data`, still +# exported and maintained) serves as the independent oracle: PowerGraphics no +# longer calls it, which is exactly what makes it a valid cross-check. +# +# Every value assertion runs against BOTH backends through the helpers in +# `plot_introspection.jl`. Writing them against `PlotlyLight.Plot.data` alone is +# what let PR #140's bar-plot defect be fixed in one backend and stay broken in +# the other; a backend-specific assertion below is marked with the reason it +# cannot be stated for both. + +const FUEL_BACKENDS = + (("cairomakie", CairoMakieBackend()), ("plotlylight", PlotlyLightBackend())) + +# `run_test_sim` deserializes the shared simulation store, so it is read once for +# the whole file rather than per testset. +(fuel_results_uc, fuel_results_ed) = run_test_sim(TEST_RESULT_DIR, TEST_SIM_NAME) + +# The old-API aggregation appears here only to derive the expected column order; +# its values are pinned by the equivalence testset below. +fuel_uc_old = categorize_data( + get_generation_data(fuel_results_uc).data, + make_fuel_dictionary(PSI.get_system(fuel_results_uc)), +) + +# Column-order contract: palette categories first (in palette order), then the +# sorted remainder. Plots must present traces in exactly this order. +fuel_matched = intersect(PG.get_palette_category(PG.PALETTE), keys(fuel_uc_old)) +fuel_expected_order = + vcat(fuel_matched, sort(collect(setdiff(keys(fuel_uc_old), fuel_matched)))) + +@testset "fuel column-order contract" begin + # The fixture must exercise the hand-written storage and curtailment + # categories, or nothing below has teeth. + @test issubset(["Storage In", "Storage Out", "Curtailment"], fuel_matched) + @test names(PA.combine_categories(fuel_uc_old; names = fuel_expected_order)) == + fuel_expected_order +end + +function test_fuel_stack(backend_pkg::String, backend::PG.PlottingBackend) + @testset "pin $backend_pkg fuel stack behavior on simulation results" begin + # Bar mode preserves trace order on both backends: PlotlyLight emits one + # trace per category and CairoMakie one vector-labeled `barplot!` that + # the introspection helper flattens back into per-category series. + p_bar = plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + bar = true, + stack = true, + ) + @test series_labels(p_bar) == vcat(fuel_expected_order, ["Load"]) + + # Stacked-area fuel plot: same trace set (order-insensitive because both + # backends draw net-negative series first). + p_area = + plot_fuel(fuel_results_uc; backend = backend, set_display = false, + stack = true) + @test sort(series_labels(p_area)) == sort(vcat(fuel_expected_order, ["Load"])) + + # Sign contract on PowerGraphics' own traces: storage charging renders + # below the axis, discharging above it, and curtailment (forecast minus + # dispatch) is non-negative up to solver tolerance. + @test all(<=(1e-6), series_values(p_area, "Storage In")) + @test all(>=(-1e-6), series_values(p_area, "Storage Out")) + @test all(>=(-1e-4), series_values(p_area, "Curtailment")) + + # CairoMakie tracks its own series counter to rebuild the legend across + # layered calls; cross-check it against the marks actually on the axis. + @test series_count(p_area) == length(fuel_expected_order) + 1 + end + + @testset "$backend_pkg fuel net-load overlay includes storage charging" begin + # With unit auto-scaling disabled all traces are in raw MW, so the "Load" + # overlay must equal demand plus the magnitude of the (negative) storage + # charging trace — the net-load line coincides with the top of the + # generation stack. + # + # The overlay is drawn by a separate single-column `_plot_dataframe!` + # call, so CairoMakie's stacked-line envelope for it is the raw demand + # series and compares directly with the PlotlyLight trace. + p = plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) + load_y = series_values(p, "Load") + in_y = series_values(p, "Storage In") + demand = PA.combine_categories(get_load_data(fuel_results_uc).data)[!, "Load"] + # The battery actually charges in the test solution, so this has teeth. + @test sum(in_y) < 0 + @test load_y ≈ demand .- in_y + end + + @testset "$backend_pkg fuel trace values match the old-API aggregation" begin + # Numeric equivalence contract between the migrated metrics-API pipeline + # and the old PowerAnalytics aggregation, over EVERY category the old API + # emits. The categories PowerGraphics reimplements by hand — the + # " In"/"Out" storage split, "Curtailment" and the "Unserved + # Energy"/"Over Generation" slacks — are the ones most likely to carry a + # wrong sign, a doubled contribution or a dropped component, so they are + # pinned by value and not merely by sign. UC solves with + # `use_slacks = false` and ED with `use_slacks = true`, so the pair also + # covers the slack categories. + for result in (fuel_results_uc, fuel_results_ed) + fuel_old = categorize_data( + get_generation_data(result).data, + make_fuel_dictionary(PSI.get_system(result)), + ) + @test !isempty(fuel_old) + + # `auto_units = false` keeps every trace in raw MW, so no unit + # scaling sits between the two pipelines. + p = plot_fuel( + result; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) + # "Load" is the net-load overlay, not a fuel category, so it is the + # one trace legitimately absent from `fuel_old`. Every other trace + # must have a counterpart, and no old-API category may be missing + # from the plot: a one-sided category is a migration defect, not a + # representational difference. + traces = filter(kv -> first(kv) != "Load", series_map(p)) + @test Set(keys(traces)) == Set(keys(fuel_old)) + + for k in sort(collect(intersect(keys(traces), keys(fuel_old)))) + expected = vec(sum(Matrix(no_datetime(fuel_old[k])); dims = 2)) + @test traces[k] ≈ expected + end + end + end + + @testset "$backend_pkg fuel category toggles drop exactly their categories" begin + # ED holds storage and solves with `use_slacks = true`, so every optional + # category family is present by default and each kwarg has something to + # drop. + labels = + kwargs -> sort( + series_labels( + plot_fuel( + fuel_results_ed; + backend = backend, + set_display = false, + stack = true, + kwargs..., + ), + ), + ) + names_default = labels(()) + names_nocurtailment = labels((:curtailment => false,)) + names_noslacks = labels((:slacks => false,)) + names_nostorage = labels((:storage => false,)) + + @test issubset( + [ + "Storage In", + "Storage Out", + "Curtailment", + "Unserved Energy", + "Over Generation", + ], + names_default, + ) + # `setdiff` preserves the (sorted) order of its first argument. + @test setdiff(names_default, names_nocurtailment) == ["Curtailment"] + @test setdiff(names_default, names_noslacks) == + ["Over Generation", "Unserved Energy"] + @test setdiff(names_default, names_nostorage) == ["Storage In", "Storage Out"] + end + + @testset "$backend_pkg unmatched components route to Other with an error log" begin + incomplete_mapping = + joinpath(TEST_DIR, "test_yamls", "generator_mapping_incomplete.yaml") + + p_inc = + @test_logs (:error, r"No category in the generator mapping") match_mode = :any plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + generator_mapping_file = incomplete_mapping, + ) + @test "Other" in series_labels(p_inc) + + # The unmatched hydro generation lands intact in "Other": same total as + # the "Hydropower" category under the default mapping. + p_def = plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) + @test sum(series_values(p_inc, "Other")) ≈ + sum(series_values(p_def, "Hydropower")) + end + + @testset "pin $backend_pkg demand plot behavior on simulation results" begin + load_uc = get_load_data(fuel_results_uc) + expected = PA.combine_categories(load_uc.data) + + # The results-path demand frame is a single non-negative "Load" column. + @test names(expected) == ["Load"] + @test all(>=(-1e-6), expected[!, "Load"]) + @test length(load_uc.time) == nrow(expected) + + p = plot_demand(fuel_results_uc; backend = backend, set_display = false) + @test series_labels(p) == ["Load"] + @test series_values(p, "Load") ≈ expected[!, "Load"] + + # Legacy time-window kwargs must keep working through the migration. + p_h = plot_demand( + fuel_results_uc; + backend = backend, + set_display = false, + horizon = 3, + ) + @test series_values(p_h, "Load") ≈ expected[1:3, "Load"] + + # Index 25 is the start of the second simulation step, a timestamp that + # is valid under both the old and the new results readers. + t0 = load_uc.time[25] + p_it = plot_demand( + fuel_results_uc; + backend = backend, + set_display = false, + initial_time = t0, + horizon = 2, + ) + @test series_values(p_it, "Load") ≈ expected[25:26, "Load"] + + # The start_time/len spellings behave identically to initial_time/horizon. + p_sl = plot_demand( + fuel_results_uc; + backend = backend, + set_display = false, + start_time = t0, + len = 2, + ) + @test series_values(p_sl, "Load") ≈ expected[25:26, "Load"] + + # filter_func restricts which loads are included. + only_bus2 = x -> get_name(get_bus(x)) == "bus2" + expected_f = PA.combine_categories( + get_load_data(fuel_results_uc; filter_func = only_bus2).data, + ) + p_f = plot_demand( + fuel_results_uc; + backend = backend, + set_display = false, + filter_func = only_bus2, + ) + @test series_values(p_f, "Load") ≈ expected_f[!, "Load"] + @test sum(expected_f[!, "Load"]) < sum(expected[!, "Load"]) + end +end + +for (backend_pkg, backend) in FUEL_BACKENDS + test_fuel_stack(backend_pkg, backend) +end + +@testset "fuel stack is identical across backends" begin + # The per-backend testsets above pin each backend against the same oracle; + # this compares the two backends directly, so a defect that shifts BOTH in + # the same direction is still caught by the oracle while a one-sided + # regression is caught here with a much smaller diff to read. + plots = Dict( + pkg => plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) for (pkg, backend) in FUEL_BACKENDS + ) + cm = plots["cairomakie"] + pl = plots["plotlylight"] + + @test series_labels(cm) == series_labels(pl) + @test series_colors(cm) == series_colors(pl) + for (a, b) in zip(series_ydata(cm), series_ydata(pl)) + @test a ≈ b + end +end + +@testset "plot_demand and plot_fuel save exactly one file" begin + # `_plot_demand!` used to read `:save` without removing it from the key words + # it forwarded, so the delegated `_plot_dataframe!` saved the figure and the + # wrapper then saved it again under a space-sanitized name: one call, two + # files. `_plot_results!` and `_plot_fuel!` stripped `:save` and did not. + # Every wrapper now resolves its path once through `_resolve_save_file`. + save_root = joinpath(TEST_OUTPUTS, "fuel_save") + isdir(save_root) && rm(save_root; recursive = true) + mkpath(save_root) + + demand_dir = joinpath(save_root, "demand") + mkpath(demand_dir) + plot_demand( + fuel_results_uc; + set_display = false, + title = "My Demand", + save = demand_dir, + ) + @test readdir(demand_dir) == ["My_Demand.png"] + + fuel_dir = joinpath(save_root, "fuel") + mkpath(fuel_dir) + plot_fuel(fuel_results_uc; set_display = false, title = "My Fuel", save = fuel_dir) + @test readdir(fuel_dir) == ["My_Fuel.png"] + + @info("removing test files") + rm(save_root; recursive = true) +end + +# --- Balance slacks owned by something other than `PSY.System` (issue #94) --- + +# PowerSimulations attaches the balance slacks to the component type implied by +# the network formulation, so a nodal formulation stores one slack column per +# `ACBus`. The load is scaled up so the slacks are actually nonzero and the +# aggregation assertions have teeth. +function run_nodal_slack_model() + sys = deepcopy(PSB.build_system(PSB.PSITestSystems, "c_sys5_uc")) + for load in get_components(PowerLoad, sys) + set_max_active_power!(load, 3 * get_max_active_power(load)) + end + template = ProblemTemplate(NetworkModel(DCPPowerModel; use_slacks = true)) + set_device_model!(template, ThermalStandard, ThermalBasicUnitCommitment) + set_device_model!(template, PowerLoad, StaticPowerLoad) + set_device_model!(template, Line, StaticBranch) + model = DecisionModel( + template, + sys; + optimizer = optimizer_with_attributes(HiGHS.Optimizer), + horizon = Hour(6), + ) + build!(model; output_dir = mktempdir()) + solve!(model) + return OptimizationProblemResults(model) +end + +@testset "bus-level balance slacks appear in the fuel stack" begin + # The solve is backend-independent, so it runs once for both backends. + res = run_nodal_slack_model() + + # The regression of #94: the slacks are keyed on `ACBus`, not `System`, so + # looking only for the `System` variant made them vanish from the plot. + @test Set( + PSI.encode_key_as_string(k) for k in PSI.list_variable_keys(res) if + PSI.get_entry_type(k) in keys(PA.BALANCE_SLACKVARS) + ) == Set(["SystemBalanceSlackUp__ACBus", "SystemBalanceSlackDown__ACBus"]) + + # Each direction is the row-wise sum over the per-bus columns; the oracle is + # read straight from the stored results and is the same for both backends. + expected_slacks = Dict{String, Vector{Float64}}() + for (name, entry) in ( + ("Unserved Energy", PSI.SystemBalanceSlackUp), + ("Over Generation", PSI.SystemBalanceSlackDown), + ) + entry_keys = + [k for k in PSI.list_variable_keys(res) if PSI.get_entry_type(k) == entry] + df = only( + values( + PSI.read_results_with_keys( + res, + entry_keys; + table_format = IS.TableFormat.WIDE, + ), + ), + ) + # More than DateTime plus one column, i.e. genuinely nodal. + @test ncol(df) > 2 + expected_slacks[name] = vec(sum(Matrix(no_datetime(df)); dims = 2)) + end + + slack_counts = Int[] + for (backend_pkg, backend) in FUEL_BACKENDS + @testset "$backend_pkg bus-level slack aggregation" begin + p = plot_fuel( + res; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) + labels = series_labels(p) + @test "Unserved Energy" in labels + @test "Over Generation" in labels + + for (name, expected) in expected_slacks + @test series_values(p, name) ≈ expected + end + # The scaled-up load leaves energy unserved, so the sum above is not + # trivially zero. + @test sum(series_values(p, "Unserved Energy")) > 0 + + push!(slack_counts, series_count(p)) + end + end + # Both backends must draw the same number of series, or the slack categories + # reached only one of them. + @test allequal(slack_counts) +end + +@testset "system-level balance slacks are unchanged" begin + # The ED template is CopperPlate with `use_slacks = true`, so the slacks are + # keyed on `PSY.System` — the only case PowerAnalytics' own system metrics + # handle. Those values are the pre-fix reference and must be reproduced. + calc_slack_down = + PA.make_system_metric_from_entry("SystemSlackDown", PSI.SystemBalanceSlackDown) + expected_system_slacks = Dict( + name => Vector{Float64}(PA.get_data_vec(PA.compute(metric, fuel_results_ed))) + for + (name, metric) in ( + ("Unserved Energy", PA.Metrics.calc_system_slack_up), + ("Over Generation", calc_slack_down), + ) + ) + + for (backend_pkg, backend) in FUEL_BACKENDS + @testset "$backend_pkg system-level slack values" begin + p = plot_fuel( + fuel_results_ed; + backend = backend, + set_display = false, + stack = true, + auto_units = false, + ) + for (name, expected) in expected_system_slacks + @test series_values(p, name) ≈ expected + end + end + end +end + +@testset "results without balance slacks skip the slack categories" begin + # The UC template runs with `use_slacks = false`: no slack variable is + # stored, so the categories must be absent rather than raising. + @test !any( + PSI.get_entry_type(k) in keys(PA.BALANCE_SLACKVARS) for + k in PSI.list_variable_keys(fuel_results_uc) + ) + for (backend_pkg, backend) in FUEL_BACKENDS + @testset "$backend_pkg skips absent slack categories" begin + p = plot_fuel( + fuel_results_uc; + backend = backend, + set_display = false, + stack = true, + ) + @test isdisjoint( + series_labels(p), + ["Unserved Energy", "Over Generation"], + ) + end + end +end diff --git a/test/test_plot_creation.jl b/test/test_plot_creation.jl index 69cf878..0174002 100644 --- a/test/test_plot_creation.jl +++ b/test/test_plot_creation.jl @@ -6,12 +6,14 @@ function test_plots(file_path::String; backend_pkg::String = "cairomakie") plot_dataframe_fn = plot_dataframe plot_dataframe_fn! = plot_dataframe! plot_demand_fn = plot_demand + plot_results_fn = plot_results plot_powerdata_fn = PG.plot_powerdata plot_fuel_fn = plot_fuel elseif backend_pkg == "plotlylight" plot_dataframe_fn = plot_dataframe_plotly plot_dataframe_fn! = plot_dataframe_plotly! plot_demand_fn = plot_demand_plotly + plot_results_fn = plot_results_plotly plot_powerdata_fn = PG.plot_powerdata_plotly plot_fuel_fn = plot_fuel_plotly else @@ -23,16 +25,12 @@ function test_plots(file_path::String; backend_pkg::String = "cairomakie") @info("running tests with $backend_pkg with display $set_display and cleanup $cleanup") (results_uc, results_ed) = run_test_sim(TEST_RESULT_DIR, TEST_SIM_NAME) - problem_results = run_test_prob() gen_uc = get_generation_data(results_uc) - gen_ed = get_generation_data(results_ed) - gen_pb = get_generation_data(problem_results) load_uc = get_load_data(results_uc) - load_ed = get_load_data(results_ed) - load_pb = get_load_data(problem_results) - svc_uc = get_service_data(results_uc) - svc_ed = get_service_data(results_ed) - svc_pb = get_service_data(problem_results) + # The dict-of-DataFrames shape `plot_results` consumes; each entry keeps its + # own DateTime column. + results_dict = + Dict{String, DataFrames.DataFrame}(string(k) => v for (k, v) in gen_uc.data) @testset "test $backend_pkg plot production" begin out_path = joinpath(file_path, backend_pkg * "_plots") @@ -111,42 +109,53 @@ function test_plots(file_path::String; backend_pkg::String = "cairomakie") cleanup && rm(out_path; recursive = true) end - @testset "test $backend_pkg powerdata plot production" begin - out_path = joinpath(file_path, backend_pkg * "_powerdata_plots") + @testset "test $backend_pkg results plot production" begin + out_path = joinpath(file_path, backend_pkg * "_results_plots") !isdir(out_path) && mkdir(out_path) - plot_powerdata_fn( - gen_uc; + plot_results_fn( + results_dict; set_display = set_display, title = "pg_data", save = out_path, bar = false, stack = false, ) - plot_powerdata_fn( - gen_uc; + plot_results_fn( + results_dict; set_display = set_display, title = "pg_data_stack", save = out_path, bar = false, stack = true, ) - plot_powerdata_fn( - gen_uc; + plot_results_fn( + results_dict; set_display = set_display, title = "pg_data_bar", save = out_path, bar = true, stack = false, ) - plot_powerdata_fn( - gen_uc; + plot_results_fn( + results_dict; set_display = set_display, title = "pg_data_bar_stack", save = out_path, bar = true, stack = true, ) + # One trace per stored column instead of one aggregated trace per entry. + p = plot_results_fn( + results_dict; + set_display = set_display, + title = "pg_data_split", + save = out_path, + combine_categories = false, + ) + plot_length = backend_pkg == "cairomakie" ? p.series_count : length(p.data) + @test plot_length == + sum(DataFrames.ncol(no_datetime(v)) for v in values(gen_uc.data)) list = readdir(out_path) # PlotlyLight only supports HTML export, CairoMakie supports PNG @@ -156,6 +165,7 @@ function test_plots(file_path::String; backend_pkg::String = "cairomakie") "pg_data_stack$file_ext", "pg_data_bar$file_ext", "pg_data_bar_stack$file_ext", + "pg_data_split$file_ext", ] # expected results not created @test isempty(setdiff(expected_files, list)) @@ -166,6 +176,25 @@ function test_plots(file_path::String; backend_pkg::String = "cairomakie") cleanup && rm(out_path; recursive = true) end + @testset "test $backend_pkg deprecated powerdata forwarding" begin + out_path = joinpath(file_path, backend_pkg * "_powerdata_plots") + !isdir(out_path) && mkdir(out_path) + + # The `PA.PowerData` methods are deprecated shims: they must warn and + # forward to the `plot_results` pipeline. + @test_logs (:warn, r"deprecated") match_mode = :any plot_powerdata_fn( + gen_uc; + set_display = set_display, + title = "pg_powerdata", + save = out_path, + ) + file_ext = backend_pkg == "plotlylight" ? ".html" : ".png" + @test isfile(joinpath(out_path, "pg_powerdata$file_ext")) + + @info("removing test files") + cleanup && rm(out_path; recursive = true) + end + @testset "test $backend_pkg demand plot production" begin out_path = joinpath(file_path, backend_pkg * "_demand_plots") !isdir(out_path) && mkdir(out_path) diff --git a/test/test_yamls/generator_mapping.yaml b/test/test_yamls/generator_mapping.yaml index 153e25d..eb9059d 100644 --- a/test/test_yamls/generator_mapping.yaml +++ b/test/test_yamls/generator_mapping.yaml @@ -31,9 +31,12 @@ Nuclear: - {gentype: Any, primemover: null, fuel: NUCLEAR} Geothermal: - {gentype: Any, primemover: null, fuel: GEOTHERMAL} +# Fuel names must be valid PowerSystems.ThermalFuels entries: the new +# PowerAnalytics mapping parser resolves them to real enum values and throws on +# typos (the old parser stored raw strings that silently never matched). Biopower: - - {gentype: Any, primemover: null, fuel: AG_BIPRODUCT} - - {gentype: Any, primemover: null, fuel: WOOD_WASTE} + - {gentype: Any, primemover: null, fuel: AG_BYPRODUCT} + - {gentype: Any, primemover: null, fuel: WOOD_WASTE_SOLIDS} CSP: - {gentype: Any, primemover: CP, fuel: null} Other: diff --git a/test/test_yamls/generator_mapping_incomplete.yaml b/test/test_yamls/generator_mapping_incomplete.yaml new file mode 100644 index 0000000..54fc29b --- /dev/null +++ b/test/test_yamls/generator_mapping_incomplete.yaml @@ -0,0 +1,11 @@ +# A deliberately incomplete generator mapping: the hydro components of the +# test system match no category, so `plot_fuel` must route them to "Other" and +# log an error. +PV: + - {gentype: Any, primemover: PVe, fuel: null} +Wind: + - {gentype: Any, primemover: WT, fuel: null} +Storage: + - {gentype: Any, primemover: BA, fuel: null} +Thermal: + - {gentype: ThermalStandard, primemover: null, fuel: null} diff --git a/test/test_yamls/generator_mapping_specificity.yaml b/test/test_yamls/generator_mapping_specificity.yaml new file mode 100644 index 0000000..84e6420 --- /dev/null +++ b/test/test_yamls/generator_mapping_specificity.yaml @@ -0,0 +1,34 @@ +# A generator mapping built so that the specificity ranking is the ONLY thing +# that can decide where a component lands. Every thermal rule below names the +# same `gentype`, so the component-type distance ties across all of them and the +# prime mover / fuel wildcards have to break the tie. +# +# Against the `5_bus_hydro_uc_sys` fixture system: +# Solitude, Alta (CT, NATURAL_GAS) -> NGCombustionTurbine (beats BroadThermal +# on prime mover AND fuel) +# Brighton (ST, COAL) -> CoalOnly (beats BroadThermal +# on fuel ALONE) +# Park City, Sundance (CC, NATURAL_GAS) -> BroadThermal (nothing narrower +# matches them) +BroadThermal: + - {gentype: ThermalStandard, primemover: null, fuel: null} +NGCombustionTurbine: + - {gentype: ThermalStandard, primemover: CT, fuel: NATURAL_GAS} +CoalOnly: + - {gentype: ThermalStandard, primemover: null, fuel: COAL} +# `ACBus` cannot intersect the `StaticInjection` root type PowerAnalytics parses +# with, so `make_fuel_component_selector` returns `nothing` for the first rule +# and this category ends up with two sub-selectors for three listed rules. It is +# here so the group/rule correspondence check has to replay that drop at the +# right position instead of pairing groups with rules off by one. +Hydropower: + - {gentype: ACBus, primemover: null, fuel: null} + - {gentype: HydroGen, primemover: null, fuel: null} + - {gentype: Any, primemover: HY, fuel: null} +# The remaining generators of the fixture system, so the mapping covers the pool +# and nothing is routed to "Other" -- that bucket logs at Error level, which the +# suite's logger treats as a failure. +PV: + - {gentype: Any, primemover: PVe, fuel: null} +Wind: + - {gentype: Any, primemover: WT, fuel: null}