From 175bc88331045c1ddcf4747d26f6fdea83a142b6 Mon Sep 17 00:00:00 2001 From: Luke Kiernan Date: Tue, 12 May 2026 20:26:46 -0600 Subject: [PATCH 1/3] Absorb operation models from InfrastructureOptimizationModels Concrete DecisionModel/EmulationModel + stores, OptimizationProblemOutputs, and the operation-model glue methods (read_*, list_*_keys, _check_numerical_bounds, _pre_solve_model_checks) that compose POM-only types now live here. IOM keeps just the abstract interface. Deduplicate get_problem_type, validate_template, handle_initial_conditions! onto OperationModel{T}. Pairs with IOM PR #100. Co-Authored-By: Claude Opus 4.7 (1M context) --- Project.toml | 26 +- src/PowerOperationsModels.jl | 176 ++- src/operation/decision_model.jl | 340 ++++- src/operation/decision_model_store.jl | 168 +++ src/operation/emulation_model.jl | 423 ++++++- src/operation/emulation_model_store.jl | 200 +++ ...itial_conditions_update_in_memory_store.jl | 28 + .../model_numerical_analysis_utils.jl | 152 +++ src/operation/operation_model_glue.jl | 99 ++ src/operation/optimization_debugging.jl | 110 ++ src/operation/optimization_problem_outputs.jl | 1116 +++++++++++++++++ .../optimization_problem_outputs_export.jl | 130 ++ src/operation/problem_outputs.jl | 88 ++ src/operation/store_common.jl | 236 ++++ src/operation/template_validation.jl | 13 + src/operation/time_series_interface.jl | 93 ++ src/utils/print.jl | 78 ++ test/Project.toml | 2 +- test/test_model_decision.jl | 28 +- test/test_optimization_outputs.jl | 278 ++++ test/test_utils/mock_operation_models.jl | 22 +- test/test_utils/model_checks.jl | 4 +- 22 files changed, 3677 insertions(+), 133 deletions(-) create mode 100644 src/operation/decision_model_store.jl create mode 100644 src/operation/emulation_model_store.jl create mode 100644 src/operation/initial_conditions_update_in_memory_store.jl create mode 100644 src/operation/model_numerical_analysis_utils.jl create mode 100644 src/operation/operation_model_glue.jl create mode 100644 src/operation/optimization_debugging.jl create mode 100644 src/operation/optimization_problem_outputs.jl create mode 100644 src/operation/optimization_problem_outputs_export.jl create mode 100644 src/operation/problem_outputs.jl create mode 100644 src/operation/store_common.jl create mode 100644 src/operation/time_series_interface.jl create mode 100644 test/test_optimization_outputs.jl diff --git a/Project.toml b/Project.toml index e749a78..672e64c 100644 --- a/Project.toml +++ b/Project.toml @@ -4,8 +4,13 @@ version = "0.1.0" authors = ["Sienna-Platform"] [deps] +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DataFramesMeta = "1313f7d8-7da2-5740-9ea0-a2ca25f37964" +DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +HDF5 = "f67ccb44-e63f-5c2f-98bd-6dc0ccc4ba2f" InfrastructureOptimizationModels = "bed98974-b02a-5e2f-9ee0-a103f5c45069" InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240" @@ -13,33 +18,44 @@ JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" +MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" PowerNetworkMatrices = "bed98974-b02a-5e2f-9fe0-a103f5c450dd" PowerSystems = "bcd98974-b02a-5e2f-9ee0-a103f5c450dd" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +TimeSeries = "9e3dc215-6440-5c97-bce1-76c03772f85e" TimerOutputs = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" [weakdeps] PowerFlows = "94fada2c-0ca5-4b90-a1fb-4bc5b59ccfc7" +[sources] +InfrastructureOptimizationModels = {url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl", rev = "lk/move-operation-to-pom"} +InfrastructureSystems = {rev = "IS4", url = "https://github.com/NREL-Sienna/InfrastructureSystems.jl"} +PowerSystems = {rev = "psy6", url = "https://github.com/NREL-Sienna/PowerSystems.jl"} + [extensions] PowerFlowsExt = "PowerFlows" -[sources] -InfrastructureSystems = {url = "https://github.com/NREL-Sienna/InfrastructureSystems.jl", rev = "IS4"} -PowerSystems = {url = "https://github.com/NREL-Sienna/PowerSystems.jl", rev = "psy6"} -InfrastructureOptimizationModels = {url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl", rev = "lk/pom-test-fixes"} - [compat] +CSV = "0.10.16" +DataFrames = "1.8.2" +DataFramesMeta = "0.15.6" +DataStructures = "0.19.4" Dates = "1" DocStringExtensions = "~0.8, ~0.9" +HDF5 = "0.17.3" InfrastructureOptimizationModels = "0.1" InfrastructureSystems = "3" InteractiveUtils = "1.11.0" JuMP = "^1.28" +MathOptInterface = "1.51.0" PowerNetworkMatrices = "^0.19" PowerSystems = "5.3" +PrettyTables = "3.3.2" ProgressMeter = "1.11.0" +TimeSeries = "0.25.2" TimerOutputs = "~0.5" julia = "^1.11" diff --git a/src/PowerOperationsModels.jl b/src/PowerOperationsModels.jl index 27411c1..34457ba 100644 --- a/src/PowerOperationsModels.jl +++ b/src/PowerOperationsModels.jl @@ -4,8 +4,38 @@ module PowerOperationsModels # Package imports ################################################################################# import Dates +import DataStructures: OrderedDict +# I/O and tabular deps used by files moved from IOM (OPO, model stores). +import CSV +import DataFrames +import DataFrames: DataFrame, DataFrameRow, Not, innerjoin, select +import DataFramesMeta: @chain, @orderby, @rename, @select, @subset, @transform +import HDF5 +import MathOptInterface +const MOI = MathOptInterface +const MOIU = MathOptInterface.Utilities +import PrettyTables +import TimeSeries +const TS = TimeSeries import InfrastructureSystems import InfrastructureSystems: @assert_op, TableFormat +# Imports for symbols used unqualified by files moved from IOM (operation/ and OPO). +import InfrastructureSystems: + Outputs, + Forecast, + StaticTimeSeries, + InfrastructureSystemsType, + InfrastructureSystemsComponent, + InfrastructureSystemsContainer, + TimeSeriesCacheKey, + InvalidValue, + ConflictingInputsError, + compute_file_hash, + convert_for_path, + strip_module_name, + to_namedtuple, + get_uuid, + configure_logging import JuMP import JuMP.Containers: DenseAxisArray, SparseAxisArray import Logging @@ -218,6 +248,123 @@ include("common_models/reserve_range_constraints.jl") # before device-specific files that reference MBC_TYPES / IEC_TYPES. include("common_models/market_bid_plumbing.jl") +# Internal IOM symbols used by files moved from IOM (operation/, OPO). +import InfrastructureOptimizationModels: + OptimizationContainerMetadata, + AbstractDataset, + AbstractModelStore, + DatasetContainer, + DecisionModelIndexType, + EmulationModelIndexType, + HDF5Dataset, + InitialConditionKey, + InMemoryDataset, + LOG_GROUP_MODEL_STORE, + ModelInternal, + ModelStoreParams, + Simulation, + SimulationProblemOutputs, + SimulationOutputs, + SimulationSequence, + SimulationModels, + STORE_CONTAINERS, + auto_transform_time_series!, + calculate_parameter_values, + cost_function_unsynch, + deserialize_key, + encode_key, + encode_key_as_string, + encode_keys_as_strings, + get_param_eltype, + get_base_power, + get_expression_values, + get_horizon, + get_model_base_power, + get_parameter_values, + get_resolution, + get_timestamps, + get_column_names, + get_column_names_from_axis_array, + get_current_timestamp, + get_data_field, + get_execution_count, + get_executions, + get_expressions, + get_first_dimension_output_column_name, + get_forecast_horizon, + get_forecast_interval, + get_forecast_intervals, + get_initial_conditions_file, + get_last_recorded_row, + get_last_updated_timestamp, + get_metadata, + get_num_executions, + get_output_dir, + get_parameter_attributes, + get_second_dimension_output_column_name, + get_status, + get_store, + get_store_container_type, + get_store_params, + get_system_uuid, + get_time_series_cache, + get_time_series_counts, + get_time_series_counts_by_type, + get_time_series_resolutions, + get_update_timestamp, + is_built, + is_synchronized, + list_fields, + list_keys, + make_key, + to_outputs_dataframe, + read_duals, + read_expressions, + read_parameters, + serialize_metadata, + to_dataframe, + to_dict, + write_data, + CONTAINER_KEY_EMPTY_META, + _get_ramp_constraint_devices, + LOG_GROUP_OPTIMIZATION_CONTAINER, + set_status!, + set_store_params!, + get_problem_size +import InfrastructureSystems.Optimization: + AbstractOptimizationContainer, OptimizationKeyType +import InfrastructureSystems.Simulation: SimulationInfo, get_run_status, set_run_status! +import InfrastructureSystems: + Deterministic, + DeterministicSingleTimeSeries, + FlattenIteratorWrapper, + SingleTimeSeries, + TIME_SERIES_CACHE_SIZE_BYTES, + check_component, + check_components, + deserialize, + get_available, + get_components, + get_optimizer_stats, + get_parameters, + get_source_data, + get_time_series_array, + get_time_series_values, + get_total_cost, + get_variables, + has_components, + make_time_series_cache, + serialize, + write_outputs + +# Operation output containers + model stores (moved from IOM). Must come before +# update_initial_conditions.jl, decision_model.jl, and emulation_model.jl. +include("operation/optimization_problem_outputs_export.jl") +include("operation/optimization_problem_outputs.jl") +include("operation/decision_model_store.jl") +include("operation/emulation_model_store.jl") +include("operation/store_common.jl") + # Initial Conditions include("initial_conditions/add_initial_condition.jl") include("initial_conditions/device_initial_conditions.jl") @@ -285,28 +432,25 @@ include("area_interchange.jl") # Operation lifecycle: build/solve/run include("operation/build_problem.jl") include("initial_conditions/initialization.jl") -include("operation/template_validation.jl") include("operation/decision_model.jl") include("operation/emulation_model.jl") +# template_validation must come after decision_model.jl / emulation_model.jl +# because validate_template now dispatches on Default*Problem types defined there. +include("operation/template_validation.jl") +include("operation/initial_conditions_update_in_memory_store.jl") +include("operation/problem_outputs.jl") +include("operation/time_series_interface.jl") +include("operation/optimization_debugging.jl") +include("operation/model_numerical_analysis_utils.jl") +# Methods extracted from IOM operation_model_interface.jl that compose POM-only +# store/numerical-bounds functions. Must come after the files that define those +# concretes (stores, model_numerical_analysis_utils, optimization_debugging, +# instantiate_network_model) and after decision/emulation_model.jl. +include("operation/operation_model_glue.jl") include("utils/generate_valid_formulations.jl") include("utils/print.jl") -# Import private/internal helpers (use import to avoid undeclared warning) -import InfrastructureOptimizationModels: _get_ramp_constraint_devices -import InfrastructureOptimizationModels: - get_param_eltype, - CONTAINER_KEY_EMPTY_META - -# Import high-frequency IOM internals used throughout operation lifecycle code. -# Note: BUILD_PROBLEMS_TIMER and RUN_OPERATION_MODEL_TIMER are defined in POM's -# definitions.jl, so they are NOT imported from IOM. -import InfrastructureOptimizationModels: - LOG_GROUP_OPTIMIZATION_CONTAINER, - get_store, - set_status!, - get_problem_size - # Functions defined in POM (core/interfaces.jl) export construct_device! export construct_service! diff --git a/src/operation/decision_model.jl b/src/operation/decision_model.jl index b6ce1b9..edc3d72 100644 --- a/src/operation/decision_model.jl +++ b/src/operation/decision_model.jl @@ -1,3 +1,317 @@ +function get_deterministic_time_series_type(sys::IS.InfrastructureSystemsContainer) + time_series_types = get_time_series_counts_by_type(sys) + existing_types = Set(d["type"] for d in time_series_types) + if ("Deterministic" in existing_types) && + ("DeterministicSingleTimeSeries" in existing_types) + error( + "The System contains a combination of forecast data and transformed time series data. Currently this is not supported.", + ) + end + if "Deterministic" ∈ existing_types + return IS.Deterministic + elseif "DeterministicSingleTimeSeries" ∈ existing_types + return IS.DeterministicSingleTimeSeries + else + error( + "The System does not contain any forecast data or transformed time series data.", + ) + end +end + +""" +Abstract type for models that use default InfrastructureOptimizationModels formulations. For custom decision problems + use DecisionProblem as the super type. +""" +abstract type DefaultDecisionProblem <: DecisionProblem end + +""" +Generic InfrastructureOptimizationModels Operation Problem Type for unspecified models +""" +struct GenericOpProblem <: DefaultDecisionProblem end + +mutable struct DecisionModel{M <: DecisionProblem} <: OperationModel{M} + name::Symbol + template::AbstractProblemTemplate + sys::IS.InfrastructureSystemsContainer + internal::Union{Nothing, ModelInternal} + simulation_info::Union{Nothing, SimulationInfo} + store::DecisionModelStore + ext::Dict{String, Any} +end + +""" + DecisionModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model}=nothing; + kwargs...) where {M<:DecisionProblem} + +Build the optimization problem of type M with the specific system and template. + +# Arguments + + - `::Type{M} where M<:DecisionProblem`: The abstract operation model type + - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. + - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems + - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care + - `name = nothing`: name of model, string or symbol; defaults to the type of template converted to a symbol. + - `optimizer::Union{Nothing,MOI.OptimizerWithAttributes} = nothing` : The optimizer does + not get serialized. Callers should pass whatever they passed to the original problem. + - `horizon::Dates.Period = UNSET_HORIZON`: Manually specify the length of the forecast Horizon + - `resolution::Dates.Period = UNSET_RESOLUTION`: Manually specify the model's resolution + - `warm_start::Bool = true`: True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true + - `check_components::Bool = true`: True to check the components valid fields when building + - `initialize_model::Bool = true`: Option to decide to initialize the model or not. + - `initialization_file::String = ""`: This allows to pass pre-existing initialization values to avoid the solution of an optimization problem to find feasible initial conditions. + - `deserialize_initial_conditions::Bool = false`: Option to deserialize conditions + - `export_pwl_vars::Bool = false`: True to export all the pwl intermediate variables. It can slow down significantly the build and solve time. + - `allow_fails::Bool = false`: True to allow the simulation to continue even if the optimization step fails. Use with care. + - `optimizer_solve_log_print::Bool = false`: Uses JuMP.unset_silent() to print the optimizer's log. By default all solvers are set to MOI.Silent() + - `detailed_optimizer_stats::Bool = false`: True to save detailed optimizer stats log. + - `calculate_conflict::Bool = false`: True to use solver to calculate conflicts for infeasible problems. Only specific solvers are able to calculate conflicts. + - `direct_mode_optimizer::Bool = false`: True to use the solver in direct mode. Creates a [JuMP.direct_model](https://jump.dev/JuMP.jl/dev/reference/models/#JuMP.direct_model). + - `store_variable_names::Bool = false`: to store variable names in optimization model. Decreases the build times. + - `rebuild_model::Bool = false`: It will force the rebuild of the underlying JuMP model with each call to update the model. It increases solution times, use only if the model can't be updated in memory. + - `initial_time::Dates.DateTime = UNSET_INI_TIME`: Initial Time for the model solve. + - `time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES`: Size in bytes to cache for each time array. Default is 1 MiB. Set to 0 to disable. + +# Example + +```julia +template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) +OpModel = DecisionModel(MockOperationProblem, template, system) +``` +""" +function DecisionModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + settings::Settings, + jump_model::Union{Nothing, JuMP.Model} = nothing; + name = nothing, +) where {M <: DecisionProblem} + if name === nothing + name = nameof(M) + elseif name isa String + name = Symbol(name) + end + auto_transform_time_series!(sys, settings) + ts_type = get_deterministic_time_series_type(sys) + internal = ModelInternal( + OptimizationContainer(sys, settings, jump_model, ts_type), + ) + + template_ = deepcopy(template) + finalize_template!(template_, sys) + model = DecisionModel{M}( + name, + template_, + sys, + internal, + SimulationInfo(), + DecisionModelStore(), + Dict{String, Any}(), + ) + validate_time_series!(model) + return model +end + +function DecisionModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + name = nothing, + optimizer = nothing, + horizon = UNSET_HORIZON, + resolution = UNSET_RESOLUTION, + interval = UNSET_INTERVAL, + warm_start = true, + check_components = true, + initialize_model = true, + initialization_file = "", + deserialize_initial_conditions = false, + export_pwl_vars = false, + allow_fails = false, + optimizer_solve_log_print = false, + detailed_optimizer_stats = false, + calculate_conflict = false, + direct_mode_optimizer = false, + store_variable_names = false, + rebuild_model = false, + export_optimization_model = false, + check_numerical_bounds = true, + initial_time = UNSET_INI_TIME, + time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES, +) where {M <: DecisionProblem} + settings = Settings( + sys; + horizon = horizon, + resolution = resolution, + interval = interval, + initial_time = initial_time, + optimizer = optimizer, + time_series_cache_size = time_series_cache_size, + warm_start = warm_start, + check_components = check_components, + initialize_model = initialize_model, + initialization_file = initialization_file, + deserialize_initial_conditions = deserialize_initial_conditions, + export_pwl_vars = export_pwl_vars, + allow_fails = allow_fails, + calculate_conflict = calculate_conflict, + optimizer_solve_log_print = optimizer_solve_log_print, + detailed_optimizer_stats = detailed_optimizer_stats, + direct_mode_optimizer = direct_mode_optimizer, + check_numerical_bounds = check_numerical_bounds, + store_variable_names = store_variable_names, + rebuild_model = rebuild_model, + export_optimization_model = export_optimization_model, + ) + return DecisionModel{M}(template, sys, settings, jump_model; name = name) +end + +""" +Build the optimization problem of type M with the specific system and template + +# Arguments + + - `::Type{M} where M<:DecisionProblem`: The abstract operation model type + - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. + - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems + - `jump_model::Union{Nothing, JuMP.Model}` = nothing: Enables passing a custom JuMP model. Use with care. + +# Example + +```julia +template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) +problem = DecisionModel(MyOpProblemType, template, system, optimizer) +``` +""" +function DecisionModel( + ::Type{M}, + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) where {M <: DecisionProblem} + return DecisionModel{M}(template, sys, jump_model; kwargs...) +end + +function DecisionModel( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) + return DecisionModel{GenericOpProblem}(template, sys, jump_model; kwargs...) +end + +function DecisionModel{M}( + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) where {M <: DefaultDecisionProblem} + IS.ArgumentError( + "DefaultDecisionProblem subtypes require a template. Use DecisionModel subtyping instead.", + ) +end + +# get_problem_type lifted to OperationModel{T} in IOM/operation_model_abstract_types.jl + +# Probably could be more efficient by storing the info in the internal +function get_current_time(model::DecisionModel) + execution_count = get_execution_count(model) + initial_time = get_initial_time(model) + interval = get_interval(model) + return initial_time + interval * execution_count +end + +function init_model_store_params!(model::DecisionModel) + num_executions = get_executions(model) + horizon = get_horizon(model) + system = get_system(model) + settings = get_settings(model) + model_interval = get_interval(settings) + if model_interval != UNSET_INTERVAL + interval = model_interval + else + interval = get_forecast_interval(system) + end + resolution = get_resolution(model) + base_power = get_base_power(system) + sys_uuid = get_system_uuid(system) + store_params = ModelStoreParams( + num_executions, + horizon, + iszero(interval) ? resolution : interval, + resolution, + base_power, + sys_uuid, + get_metadata(get_optimization_container(model)), + ) + set_store_params!(get_internal(model), store_params) + return +end + +function validate_time_series!(model::DecisionModel{<:DefaultDecisionProblem}) + sys = get_system(model) + settings = get_settings(model) + available_resolutions = get_time_series_resolutions(sys) + + if get_resolution(settings) == UNSET_RESOLUTION && length(available_resolutions) != 1 + throw( + IS.ConflictingInputsError( + "Data contains multiple resolutions, the resolution keyword argument must be added to the Model. Time Series Resolutions: $(available_resolutions)", + ), + ) + elseif get_resolution(settings) != UNSET_RESOLUTION && length(available_resolutions) > 1 + if get_resolution(settings) ∉ available_resolutions + throw( + IS.ConflictingInputsError( + "Resolution $(get_resolution(settings)) is not available in the system data. Time Series Resolutions: $(available_resolutions)", + ), + ) + end + else + set_resolution!(settings, first(available_resolutions)) + end + + model_interval = get_interval(settings) + available_intervals = get_forecast_intervals(sys) + if model_interval == UNSET_INTERVAL && length(available_intervals) > 1 + throw( + IS.ConflictingInputsError( + "The system contains multiple forecast intervals $(available_intervals). " * + "The `interval` keyword argument must be provided to the DecisionModel constructor " * + "to select which interval to use.", + ), + ) + elseif model_interval != UNSET_INTERVAL && !isempty(available_intervals) + if model_interval ∉ available_intervals + throw( + IS.ConflictingInputsError( + "Interval $(Dates.canonicalize(model_interval)) is not available in the system data. " * + "Available forecast intervals: $(available_intervals)", + ), + ) + end + end + if get_horizon(settings) == UNSET_HORIZON + set_horizon!( + settings, + get_forecast_horizon(sys; interval = _to_is_interval(model_interval)), + ) + end + + counts = get_time_series_counts(sys) + if counts.forecast_count < 1 + error( + "The system does not contain forecast data. A DecisionModel can't be built.", + ) + end + return +end + +get_horizon(model::DecisionModel) = get_horizon(get_settings(model)) function build_pre_step!(model::DecisionModel{<:DecisionProblem}) TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Build pre-step" begin validate_template(model) @@ -12,7 +326,7 @@ function build_pre_step!(model::DecisionModel{<:DecisionProblem}) get_system(model), ) @info "Initializing ModelStoreParams" - IOM.init_model_store_params!(model) + init_model_store_params!(model) set_status!(model, ModelBuildStatus.IN_PROGRESS) end return @@ -96,7 +410,7 @@ function reset!(model::DecisionModel{<:DefaultDecisionProblem}) IOM.set_execution_count!(model, 0) end sys = get_system(model) - ts_type = IOM.get_deterministic_time_series_type(sys) + ts_type = get_deterministic_time_series_type(sys) IOM.set_container!( get_internal(model), OptimizationContainer( @@ -175,17 +489,17 @@ function solve!( try Logging.with_logger(logger) do try - IOM.initialize_storage!( + initialize_storage!( get_store(model), get_optimization_container(model), IOM.get_store_params(model), ) TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Solve" begin - IOM._pre_solve_model_checks(model, optimizer) + _pre_solve_model_checks(model, optimizer) IOM.solve_model!(model) current_time = get_initial_time(model) write_outputs!(get_store(model), model, current_time, current_time) - IOM.write_optimizer_stats!( + write_optimizer_stats!( get_store(model), get_optimizer_stats(model), current_time, @@ -204,7 +518,7 @@ function solve!( @info "\n$(RUN_OPERATION_MODEL_TIMER)\n" catch e @error "Decision Problem solve failed" exception = (e, catch_backtrace()) - IOM.set_run_status!(model, RunStatus.FAILED) + set_run_status!(model, RunStatus.FAILED) end end finally @@ -212,10 +526,10 @@ function solve!( close(logger) end - return IOM.get_run_status(model) + return get_run_status(model) end -function handle_initial_conditions!(model::DecisionModel{<:DecisionProblem}) +function handle_initial_conditions!(model::OperationModel) TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Model Initialization" begin if isempty(get_template(model)) return @@ -287,14 +601,8 @@ function build_if_not_already_built!(model::OperationModel; kwargs...) return end -function validate_template(::DecisionModel{M}) where {M <: DecisionProblem} - error("validate_template is not implemented for DecisionModel{$M}") -end - -function validate_template(model::DecisionModel{<:DefaultDecisionProblem}) - validate_template_impl!(model) - return -end +# Default + custom-problem validate_template dispatches are defined once on +# OperationModel{T} in operation/template_validation.jl. function _make_device_cache( filter_function::Function, diff --git a/src/operation/decision_model_store.jl b/src/operation/decision_model_store.jl new file mode 100644 index 0000000..e9f65c6 --- /dev/null +++ b/src/operation/decision_model_store.jl @@ -0,0 +1,168 @@ +""" +Stores outputs data for one DecisionModel +""" +mutable struct DecisionModelStore <: AbstractModelStore + # All DenseAxisArrays have axes (column names, row indexes) + duals::Dict{ConstraintKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} + parameters::Dict{ParameterKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} + variables::Dict{VariableKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} + aux_variables::Dict{AuxVarKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} + expressions::Dict{ExpressionKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} + optimizer_stats::OrderedDict{Dates.DateTime, OptimizerStats} +end + +function DecisionModelStore() + return DecisionModelStore( + Dict{ConstraintKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), + Dict{ParameterKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), + Dict{VariableKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), + Dict{AuxVarKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), + Dict{ExpressionKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), + OrderedDict{Dates.DateTime, OptimizerStats}(), + ) +end + +function initialize_storage!( + store::DecisionModelStore, + container::AbstractOptimizationContainer, + params::ModelStoreParams, +) + num_of_executions = get_num_executions(params) + if length(get_time_steps(container)) < 1 + error("The time step count in the optimization container is not defined") + end + time_steps_count = get_time_steps(container)[end] + initial_time = get_initial_time(container) + model_interval = get_interval(params) + for type in STORE_CONTAINERS + field_containers = getfield(container, type) + outputs_container = getfield(store, type) + for (key, field_container) in field_containers + !should_write_resulting_value(key) && continue + @debug "Adding $(encode_key_as_string(key)) to DecisionModelStore" _group = + LOG_GROUP_MODEL_STORE + column_names = get_column_names(container, type, field_container, key) + data = OrderedDict{ + Dates.DateTime, + DenseAxisArray{Float64, length(column_names) + 1}, + }() + for timestamp in + range(initial_time; step = model_interval, length = num_of_executions) + data[timestamp] = fill!( + DenseAxisArray{Float64}(undef, column_names..., 1:time_steps_count), + NaN, + ) + end + outputs_container[key] = data + end + end + return +end + +function write_output!( + store::DecisionModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{T, 2, <:Tuple{Vector{String}, UnitRange}}, +) where {T} + container = getfield(store, get_store_container_type(key)) + container[key][index] = array + return +end + +function write_output!( + store::DecisionModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{T, 2, <:Tuple{Vector{Int}, UnitRange}}, +) where {T} + columns = get_column_names_from_axis_array(array) + container = getfield(store, get_store_container_type(key)) + container[key][index] = DenseAxisArray(array.data, columns..., 1:size(array, 2)) + return +end + +function write_output!( + store::DecisionModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{T, 1, <:Tuple{Vector{String}}}, +) where {T} + container = getfield(store, get_store_container_type(key)) + container[key][index] = array + return +end + +function write_output!( + store::DecisionModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{T, 3, <:Tuple{Vector{String}, Vector{String}, UnitRange{Int}}}, +) where {T} + container = getfield(store, get_store_container_type(key)) + container[key][index] = array + return +end + +function write_output!( + store::DecisionModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::DecisionModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{T, 3, <:Tuple{Vector{String}, UnitRange, UnitRange}}, +) where {T} + container = getfield(store, get_store_container_type(key)) + container[key][index] = array + return +end + +function read_outputs( + store::DecisionModelStore, + key::OptimizationContainerKey; + index::Union{DecisionModelIndexType, Nothing} = nothing, +) + container = getfield(store, get_store_container_type(key)) + data = container[key] + if isnothing(index) + @assert_op length(data) == 1 + index = first(keys(data)) + end + + # Return a copy because callers may mutate it. + return deepcopy(data[index]) +end + +function write_optimizer_stats!( + store::DecisionModelStore, + stats::OptimizerStats, + index::DecisionModelIndexType, +) + # TODO: This check is incompatible with test calls to psi_checksolve_test + # Overwriting should not be allowed in normal operation. + # if index in keys(store.optimizer_stats) + # error("Bug: Overwriting optimizer stats for index = $index") + # end + store.optimizer_stats[index] = stats + return +end + +function read_optimizer_stats(store::DecisionModelStore) + stats = [IS.to_namedtuple(x) for x in values(store.optimizer_stats)] + df = DataFrames.DataFrame(stats) + DataFrames.insertcols!(df, 1, :DateTime => keys(store.optimizer_stats)) + return df +end + +function get_column_names(store::DecisionModelStore, key::OptimizationContainerKey) + container = getfield(store, get_store_container_type(key)) + return get_column_names_from_axis_array(key, first(values(container[key]))) +end diff --git a/src/operation/emulation_model.jl b/src/operation/emulation_model.jl index 9241729..1f85184 100644 --- a/src/operation/emulation_model.jl +++ b/src/operation/emulation_model.jl @@ -1,3 +1,347 @@ +""" +Abstract type for models that use default InfrastructureOptimizationModels formulations. For custom emulation problems + use EmulationProblem as the super type. +""" +abstract type DefaultEmulationProblem <: EmulationProblem end + +""" +Default InfrastructureOptimizationModels Emulation Problem Type for unspecified problems +""" +struct GenericEmulationProblem <: DefaultEmulationProblem end + +""" + EmulationModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model}=nothing; + kwargs...) where {M<:EmulationProblem} + +Build the optimization problem of type M with the specific system and template. + +# Arguments + + - `::Type{M} where M<:EmulationProblem`: The abstract Emulation model type + - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. + - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems + - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care + - `name = nothing`: name of model, string or symbol; defaults to the type of template converted to a symbol. + - `optimizer::Union{Nothing,MOI.OptimizerWithAttributes} = nothing` : The optimizer does + not get serialized. Callers should pass whatever they passed to the original problem. + - `warm_start::Bool = true`: True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true + - `initialize_model::Bool = true`: Option to decide to initialize the model or not. + - `initialization_file::String = ""`: This allows to pass pre-existing initialization values to avoid the solution of an optimization problem to find feasible initial conditions. + - `deserialize_initial_conditions::Bool = false`: Option to deserialize conditions + - `export_pwl_vars::Bool = false`: True to export all the pwl intermediate variables. It can slow down significantly the build and solve time. + - `allow_fails::Bool = false`: True to allow the simulation to continue even if the optimization step fails. Use with care. + - `calculate_conflict::Bool = false`: True to use solver to calculate conflicts for infeasible problems. Only specific solvers are able to calculate conflicts. + - `optimizer_solve_log_print::Bool = false`: Uses JuMP.unset_silent() to print the optimizer's log. By default all solvers are set to MOI.Silent() + - `detailed_optimizer_stats::Bool = false`: True to save detailed optimizer stats log. + - `direct_mode_optimizer::Bool = false`: True to use the solver in direct mode. Creates a [JuMP.direct_model](https://jump.dev/JuMP.jl/dev/reference/models/#JuMP.direct_model). + - `store_variable_names::Bool = false`: True to store variable names in optimization model. + - `rebuild_model::Bool = false`: It will force the rebuild of the underlying JuMP model with each call to update the model. It increases solution times, use only if the model can't be updated in memory. + - `initial_time::Dates.DateTime = UNSET_INI_TIME`: Initial Time for the model solve. + - `time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES`: Size in bytes to cache for each time array. Default is 1 MiB. Set to 0 to disable. + +# Example + +```julia +template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) +OpModel = EmulationModel(MockEmulationProblem, template, system) +``` +""" +mutable struct EmulationModel{M <: EmulationProblem} <: OperationModel{M} + name::Symbol + template::AbstractProblemTemplate + sys::IS.InfrastructureSystemsContainer + internal::ModelInternal + simulation_info::SimulationInfo + store::EmulationModelStore # might be extended to other stores for simulation + ext::Dict{String, Any} + + function EmulationModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + settings::Settings, + jump_model::Union{Nothing, JuMP.Model} = nothing; + name = nothing, + ) where {M <: EmulationProblem} + if name === nothing + name = nameof(M) + elseif name isa String + name = Symbol(name) + end + finalize_template!(template, sys) + internal = ModelInternal( + OptimizationContainer(sys, settings, jump_model, IS.SingleTimeSeries), + ) + new{M}( + name, + template, + sys, + internal, + SimulationInfo(), + EmulationModelStore(), + Dict{String, Any}(), + ) + end +end + +function EmulationModel{M}( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + resolution = UNSET_RESOLUTION, + name = nothing, + optimizer = nothing, + warm_start = true, + initialize_model = true, + initialization_file = "", + deserialize_initial_conditions = false, + export_pwl_vars = false, + allow_fails = false, + calculate_conflict = false, + optimizer_solve_log_print = false, + detailed_optimizer_stats = false, + direct_mode_optimizer = false, + check_numerical_bounds = true, + store_variable_names = false, + rebuild_model = false, + initial_time = UNSET_INI_TIME, + time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES, +) where {M <: EmulationProblem} + settings = Settings( + sys; + initial_time = initial_time, + optimizer = optimizer, + time_series_cache_size = time_series_cache_size, + warm_start = warm_start, + initialize_model = initialize_model, + initialization_file = initialization_file, + deserialize_initial_conditions = deserialize_initial_conditions, + export_pwl_vars = export_pwl_vars, + allow_fails = allow_fails, + calculate_conflict = calculate_conflict, + optimizer_solve_log_print = optimizer_solve_log_print, + detailed_optimizer_stats = detailed_optimizer_stats, + direct_mode_optimizer = direct_mode_optimizer, + check_numerical_bounds = check_numerical_bounds, + store_variable_names = store_variable_names, + rebuild_model = rebuild_model, + horizon = resolution, + resolution = resolution, + ) + model = EmulationModel{M}(template, sys, settings, jump_model; name = name) + validate_time_series!(model) + return model +end + +""" +Build the optimization problem of type M with the specific system and template + +# Arguments + + - `::Type{M} where M<:EmulationProblem`: The abstract Emulation model type + - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, + branches, and services. + - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems + - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care + +# Example + +```julia +template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) +problem = EmulationModel(MyEmProblemType, template, system, optimizer) +``` +""" +function EmulationModel( + ::Type{M}, + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) where {M <: EmulationProblem} + return EmulationModel{M}(template, sys, jump_model; kwargs...) +end + +function EmulationModel( + template::AbstractProblemTemplate, + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) + return EmulationModel{GenericEmulationProblem}(template, sys, jump_model; kwargs...) +end + +""" +Builds an empty emulation model. This constructor is used for the implementation of custom +emulation models that do not require a template. + +# Arguments + + - `::Type{M} where M<:EmulationProblem`: The abstract operation model type + - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems + - `jump_model::Union{Nothing, JuMP.Model}` = nothing: Enables passing a custom JuMP model. Use with care. + +# Example + +```julia +problem = EmulationModel(system, optimizer) +``` +""" +function EmulationModel{M}( + sys::IS.InfrastructureSystemsContainer, + jump_model::Union{Nothing, JuMP.Model} = nothing; + kwargs..., +) where {M <: EmulationProblem} + return EmulationModel{M}(template, sys, jump_model; kwargs...) +end + +# get_problem_type lifted to OperationModel{T} in IOM/operation_model_abstract_types.jl + +function validate_time_series!(model::EmulationModel{<:DefaultEmulationProblem}) + sys = get_system(model) + settings = get_settings(model) + available_resolutions = get_time_series_resolutions(sys) + + if get_resolution(settings) == UNSET_RESOLUTION && length(available_resolutions) != 1 + throw( + IS.ConflictingInputsError( + "Data contains multiple resolutions, the resolution keyword argument must be added to the Model. Time Series Resolutions: $(available_resolutions)", + ), + ) + elseif get_resolution(settings) != UNSET_RESOLUTION && length(available_resolutions) > 1 + if get_resolution(settings) ∉ available_resolutions + throw( + IS.ConflictingInputsError( + "Resolution $(get_resolution(settings)) is not available in the system data. Time Series Resolutions: $(available_resolutions)", + ), + ) + end + else + set_resolution!(settings, first(available_resolutions)) + end + + if get_horizon(settings) == UNSET_HORIZON + # Emulation Models Only solve one "step" so Horizon and Resolution must match + set_horizon!(settings, get_resolution(settings)) + end + + counts = get_time_series_counts(sys) + if counts.static_time_series_count < 1 + error( + "The system does not contain Static Time Series data. A EmulationModel can't be built.", + ) + end + return +end + +function get_current_time(model::EmulationModel) + execution_count = get_execution_count(model) + initial_time = get_initial_time(model) + resolution = get_resolution(model) + return initial_time + resolution * execution_count +end + +function init_model_store_params!(model::EmulationModel) + num_executions = get_executions(model) + system = get_system(model) + settings = get_settings(model) + horizon = interval = resolution = get_resolution(settings) + base_power = get_base_power(system) + sys_uuid = get_system_uuid(system) + set_store_params!( + get_internal(model), + ModelStoreParams( + num_executions, + horizon, + interval, + resolution, + base_power, + sys_uuid, + get_metadata(get_optimization_container(model)), + ), + ) + return +end + +function update_parameters!( + model::EmulationModel, + store::EmulationModelStore{InMemoryDataset}, +) + update_parameters!(model, store.data_container) + return +end + +function update_parameters!(model::EmulationModel, data::DatasetContainer{InMemoryDataset}) + cost_function_unsynch(get_optimization_container(model)) + for key in keys(get_parameters(model)) + update_parameter_values!(model, key, data) + end + if !is_synchronized(model) + update_objective_function!(get_optimization_container(model)) + obj_func = get_objective_expression(get_optimization_container(model)) + set_synchronized_status!(obj_func, true) + end + return +end + +function update_model!( + model::EmulationModel, + source::EmulationModelStore{InMemoryDataset}, + ini_cond_chronology, +) + TimerOutputs.@timeit RUN_SIMULATION_TIMER "Parameter Updates" begin + update_parameters!(model, source) + end + TimerOutputs.@timeit RUN_SIMULATION_TIMER "Ini Cond Updates" begin + update_initial_conditions!(model, source, ini_cond_chronology) + end + return +end + +""" +Standalone update for EmulationModel (non-simulation context). +Updates parameters and initial conditions from the model's own store. +""" +function update_model!(model::EmulationModel) + source = get_store(model) + TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Parameter Updates" begin + update_parameters!(model, source) + end + TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Ini Cond Updates" begin + for key in keys(get_initial_conditions(model)) + update_initial_conditions!(model, key, source) + end + end + return +end + +""" +Update parameter function an OperationModel +""" +function update_parameter_values!( + model::EmulationModel, + key::ParameterKey{T, U}, + input::DatasetContainer{InMemoryDataset}, +) where {T <: ParameterType, U <: IS.InfrastructureSystemsComponent} + # Enable again for detailed debugging + # TimerOutputs.@timeit RUN_SIMULATION_TIMER "$T $U Parameter Update" begin + optimization_container = get_optimization_container(model) + # FIXME: This parameter update logic belongs in POM or PSI, not IOM. + # Move this function (and the surrounding update chain) once EmulationModel + # lifecycle code is fully migrated. + update_container_parameter_values!(optimization_container, model, key, input) + parameter_attributes = get_parameter_attributes(optimization_container, key) + IS.@record :execution ParameterUpdateEvent( + T, + U, + "event", # parameter_attributes, + get_current_timestamp(model), + get_name(model), + ) + #end + return +end # FIXME untested. Moved to accommodate a few methods dispatching on EmulationModelStore, # but not run in the tests and not yet refactored for IOM-POM split. function build_pre_step!(model::EmulationModel) @@ -18,7 +362,7 @@ function build_pre_step!(model::EmulationModel) ) @info "Initializing ModelStoreParams" - IOM.init_model_store_params!(model) + init_model_store_params!(model) set_status!(model, ModelBuildStatus.IN_PROGRESS) end return @@ -126,7 +470,7 @@ function execute_emulation!( enable_progress_bar = _progress_meter_enabled(), kwargs..., ) - IOM._pre_solve_model_checks(model, optimizer) + _pre_solve_model_checks(model, optimizer) internal = get_internal(model) executions = IOM.get_executions(internal) # Temporary check. Needs better way to manage re-runs of the same model @@ -144,7 +488,7 @@ function execute_emulation!( IOM.solve_model!(model) current_time = initial_time + (execution - 1) * get_resolution(model) write_outputs!(get_store(model), model, execution, current_time) - IOM.write_optimizer_stats!( + write_optimizer_stats!( get_store(model), get_optimizer_stats(model), execution, @@ -220,7 +564,7 @@ function run!( try Logging.with_logger(logger) do try - IOM.initialize_storage!( + initialize_storage!( get_store(model), get_optimization_container(model), IOM.get_store_params(model), @@ -231,7 +575,7 @@ function run!( enable_progress_bar = enable_progress_bar, kwargs..., ) - IOM.set_run_status!(model, RunStatus.SUCCESSFULLY_FINALIZED) + set_run_status!(model, RunStatus.SUCCESSFULLY_FINALIZED) end if export_optimization_model TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Serialize" begin @@ -246,75 +590,18 @@ function run!( @info "\n$(RUN_OPERATION_MODEL_TIMER)\n" catch e @error "Emulation Problem Run failed" exception = (e, catch_backtrace()) - IOM.set_run_status!(model, RunStatus.FAILED) + set_run_status!(model, RunStatus.FAILED) end end finally IOM.unregister_recorders!(model) close(logger) end - return IOM.get_run_status(model) + return get_run_status(model) end -function handle_initial_conditions!(model::EmulationModel{<:EmulationProblem}) - TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Model Initialization" begin - if isempty(get_template(model)) - return - end - settings = get_settings(model) - initialize_model = get_initialize_model(settings) - deserialize_initial_conditions = get_deserialize_initial_conditions(settings) - serialized_initial_conditions_file = IOM.get_initial_conditions_file(model) - custom_init_file = get_initialization_file(settings) - - if !initialize_model && deserialize_initial_conditions - throw( - IS.ConflictingInputsError( - "!initialize_model && deserialize_initial_conditions", - ), - ) - elseif !initialize_model && !isempty(custom_init_file) - throw(IS.ConflictingInputsError("!initialize_model && initialization_file")) - end - - if !initialize_model - @info "Skip build of initial conditions" - return - end +# handle_initial_conditions! lifted to OperationModel in decision_model.jl +# (identical body for both DecisionModel and EmulationModel). - if !isempty(custom_init_file) - if !isfile(custom_init_file) - error("initialization_file = $custom_init_file does not exist") - end - if abspath(custom_init_file) != abspath(serialized_initial_conditions_file) - cp(custom_init_file, serialized_initial_conditions_file; force = true) - end - end - - if deserialize_initial_conditions && isfile(serialized_initial_conditions_file) - IOM.set_initial_conditions_data!( - get_optimization_container(model), - Serialization.deserialize(serialized_initial_conditions_file), - ) - @info "Deserialized initial_conditions_data" - else - @info "Make Initial Conditions Model" - build_initial_conditions!(model) - solve_and_write_initial_conditions!(model) - end - IOM.set_initial_conditions_model_container!( - get_internal(model), - nothing, - ) - end - return -end - -function validate_template(::EmulationModel{M}) where {M <: EmulationProblem} - error("validate_template is not implemented for EmulationModel{$M}") -end - -function validate_template(model::EmulationModel{<:DefaultEmulationProblem}) - validate_template_impl!(model) - return -end +# Default + custom-problem validate_template dispatches are defined once on +# OperationModel{T} in operation/template_validation.jl. diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl new file mode 100644 index 0000000..8dd0bd4 --- /dev/null +++ b/src/operation/emulation_model_store.jl @@ -0,0 +1,200 @@ +""" +Stores outputs data for one EmulationModel. +Parameterized by `T <: AbstractDataset` to support different storage backends +(e.g., `InMemoryDataset`, `HDF5Dataset`). +""" +mutable struct EmulationModelStore{T <: AbstractDataset} <: AbstractModelStore + data_container::DatasetContainer{T} + optimizer_stats::OrderedDict{Int, OptimizerStats} +end + +get_data_field(store::EmulationModelStore, ::Val{S}) where {S} = + getfield(store.data_container, S) +@inline Base.@constprop :aggressive get_data_field( + store::EmulationModelStore, + type::Symbol, +) = + get_data_field(store, Val(type)) + +function EmulationModelStore() + return EmulationModelStore( + DatasetContainer{InMemoryDataset}(), + OrderedDict{Int, OptimizerStats}(), + ) +end + +""" + Base.empty!(store::EmulationModelStore) + +Empty the [`EmulationModelStore`](@ref) +""" +function Base.empty!(store::EmulationModelStore) + stype = DatasetContainer + for (name, _) in zip(fieldnames(stype), fieldtypes(stype)) + if name ∉ [:values, :timestamps] + val = get_data_field(store, name) + try + empty!(val) + catch + @error "Base.empty! must be customized for type $stype or skipped" + rethrow() + end + elseif name == :update_timestamp + store.update_timestamp = UNSET_INI_TIME + else + setfield!( + store.data_container, + name, + zero(fieldtype(store.data_container, name)), + ) + end + end + empty!(store.optimizer_stats) + return +end + +function Base.isempty(store::EmulationModelStore) + stype = DatasetContainer + for (name, type) in zip(fieldnames(stype), fieldtypes(stype)) + if name ∉ [:values, :timestamps] + val = get_data_field(store, name) + try + !isempty(val) && return false + catch + @error "Base.isempty must be customized for type $stype or skipped" + rethrow() + end + elseif name == :update_timestamp + store.update_timestamp != UNSET_INI_TIME && return false + else + val = get_data_field(store, name) + iszero(val) && return false + end + end + return isempty(store.optimizer_stats) +end + +function initialize_storage!( + store::EmulationModelStore{InMemoryDataset}, + container::OptimizationContainer, + params::ModelStoreParams, +) + num_of_executions = get_num_executions(params) + for type in STORE_CONTAINERS + field_containers = getfield(container, type) + outputs_container = get_data_field(store, type) + for (key, field_container) in field_containers + @debug "Adding $(encode_key_as_string(key)) to EmulationModelStore" _group = + LOG_GROUP_MODEL_STORE + column_names = get_column_names(container, type, field_container, key) + outputs_container[key] = InMemoryDataset( + fill!( + DenseAxisArray{Float64}(undef, column_names..., 1:num_of_executions), + NaN, + ), + ) + end + end + return +end + +function write_output!( + store::EmulationModelStore, + name::Symbol, + key::OptimizationContainerKey, + index::EmulationModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{Float64, 2}, +) + if size(array, 2) == 1 + write_output!(store, name, key, index, update_timestamp, array[:, 1]) + else + container = get_data_field(store, get_store_container_type(key)) + set_value!( + container[key], + array, + index, + ) + set_last_recorded_row!(container[key], index) + set_update_timestamp!(container[key], update_timestamp) + end + return +end + +function write_output!( + store::EmulationModelStore, + ::Symbol, + key::OptimizationContainerKey, + index::EmulationModelIndexType, + update_timestamp::Dates.DateTime, + array::DenseAxisArray{Float64, 1}, +) + container = get_data_field(store, get_store_container_type(key)) + set_value!( + container[key], + array, + index, + ) + set_last_recorded_row!(container[key], index) + set_update_timestamp!(container[key], update_timestamp) + return +end + +function read_outputs( + store::EmulationModelStore{InMemoryDataset}, + key::OptimizationContainerKey; + index::Union{Int, Nothing} = nothing, + len::Union{Int, Nothing} = nothing, +) + container = get_data_field(store, get_store_container_type(key)) + data = container[key].values + # Return a copy because callers may mutate it. + if isnothing(index) + @assert_op len === nothing + return data[:, :] + elseif isnothing(len) + return data[:, index:end] + else + return data[:, index:(index + len - 1)] + end +end + +function get_column_names( + store::EmulationModelStore{InMemoryDataset}, + key::OptimizationContainerKey, +) + container = get_data_field(store, get_store_container_type(key)) + return get_column_names_from_axis_array(key, container[key].values) +end + +function get_dataset_size(store::EmulationModelStore, key::OptimizationContainerKey) + container = get_data_field(store, get_store_container_type(key)) + return size(container[key].values) +end + +function get_last_updated_timestamp( + store::EmulationModelStore, + key::OptimizationContainerKey, +) + container = get_data_field(store, get_store_container_type(key)) + return get_update_timestamp(container[key]) +end +function write_optimizer_stats!( + store::EmulationModelStore, + stats::OptimizerStats, + index::EmulationModelIndexType, +) + @assert !(index in keys(store.optimizer_stats)) + store.optimizer_stats[index] = stats + return +end + +function read_optimizer_stats(store::EmulationModelStore) + return DataFrames.DataFrame([ + IS.to_namedtuple(x) for x in values(store.optimizer_stats) + ]) +end + +function get_last_recorded_row(x::EmulationModelStore, key::OptimizationContainerKey) + return get_last_recorded_row(x.data_container, key) +end diff --git a/src/operation/initial_conditions_update_in_memory_store.jl b/src/operation/initial_conditions_update_in_memory_store.jl new file mode 100644 index 0000000..1d2e7c9 --- /dev/null +++ b/src/operation/initial_conditions_update_in_memory_store.jl @@ -0,0 +1,28 @@ + +################## ic updates from store for emulation problems simulation ################# + +""" + update_initial_conditions!(model, key, source) + +Update initial conditions for a specific key from the model store. +Dispatches to the per-IC-type `update_initial_conditions!(ics, store, resolution)` method. +""" +function update_initial_conditions!( + model::OperationModel, + key::InitialConditionKey{T, U}, + source, +) where {T <: InitialConditionType, U <: IS.InfrastructureSystemsComponent} + if get_execution_count(model) < 1 + return + end + container = get_optimization_container(model) + model_resolution = get_resolution(get_store_params(model)) + ini_conditions_vector = get_initial_condition(container, key) + update_initial_conditions!(ini_conditions_vector, source, model_resolution) + return +end + +# NOTE: The 3-arg method (Vector{<:InitialCondition{T}}, ::EmulationModelStore, +# ::Dates.Millisecond) lives in src/initial_conditions/update_initial_conditions.jl. +# It used to be a stub here when this file lived in IOM; POM provides the +# concrete implementation now. diff --git a/src/operation/model_numerical_analysis_utils.jl b/src/operation/model_numerical_analysis_utils.jl new file mode 100644 index 0000000..b27324a --- /dev/null +++ b/src/operation/model_numerical_analysis_utils.jl @@ -0,0 +1,152 @@ +# The Numerical stability checks code in this file is based on the code from the SDDP.jl package, +# from the below mentioned commit and file. +# commit :8cd305188caffc50a1734913053fc81bba613778 +# link to file :https://github.com/odow/SDDP.jl/blob/d353fe5a2903421e7fed6d609eb9377c35d715a1/src/print.jl#L190 + +mutable struct NumericalBounds + min::Float64 + max::Float64 + min_index::Any + max_index::Any +end + +NumericalBounds() = NumericalBounds(Inf, -Inf, nothing, nothing) + +set_min!(v::NumericalBounds, value::Real) = v.min = value +set_max!(v::NumericalBounds, value::Real) = v.max = value +set_min_index!(v::NumericalBounds, idx) = v.min_index = idx +set_max_index!(v::NumericalBounds, idx) = v.max_index = idx + +mutable struct ConstraintBounds + coefficient::NumericalBounds + rhs::NumericalBounds + function ConstraintBounds() + return new(NumericalBounds(), NumericalBounds()) + end +end + +function update_coefficient_bounds( + v::ConstraintBounds, + constraint::JuMP.ScalarConstraint, + idx, +) + update_numerical_bounds(v.coefficient, constraint.func, idx) + return +end + +function update_rhs_bounds(v::ConstraintBounds, constraint::JuMP.ScalarConstraint, idx) + update_numerical_bounds(v.rhs, constraint.set, idx) + return +end + +mutable struct VariableBounds + bounds::NumericalBounds + function VariableBounds() + return new(NumericalBounds()) + end +end + +function update_variable_bounds(v::VariableBounds, variable::JuMP.VariableRef, idx) + if JuMP.is_binary(variable) + set_min!(v.bounds, 0.0) + update_numerical_bounds(v.bounds, 1.0, idx) + else + if JuMP.has_lower_bound(variable) + update_numerical_bounds(v.bounds, JuMP.lower_bound(variable), idx) + end + if JuMP.has_upper_bound(variable) + update_numerical_bounds(v.bounds, JuMP.upper_bound(variable), idx) + end + end + return +end + +function update_numerical_bounds(v::NumericalBounds, value::Real, idx) + if !isapprox(value, 0.0) + if v.min > abs(value) + set_min!(v, value) + set_min_index!(v, idx) + elseif v.max < abs(value) + set_max!(v, value) + set_max_index!(v, idx) + end + end + return +end + +function update_numerical_bounds(bonuds::NumericalBounds, func::JuMP.GenericAffExpr, idx) + for coefficient in values(func.terms) + update_numerical_bounds(bonuds, coefficient, idx) + end + return +end + +function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.LessThan, idx) + return update_numerical_bounds(bonuds, func.upper, idx) +end + +function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.GreaterThan, idx) + return update_numerical_bounds(bonuds, func.lower, idx) +end + +function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.EqualTo, idx) + return update_numerical_bounds(bonuds, func.value, idx) +end + +function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.Interval, idx) + update_numerical_bounds(bonuds, func.upper, idx) + return update_numerical_bounds(bonuds, func.lower, idx) +end + +# Default fallbacks for unsupported constraints. +update_numerical_bounds(::NumericalBounds, func, idx) = nothing +update_coefficient_bounds(::ConstraintBounds, func, idx) = nothing +update_rhs_bounds(::ConstraintBounds, func, idx) = nothing + +function get_constraint_numerical_bounds(model::OperationModel) + if !is_built(model) + error("Model not built, can't calculate constraint numerical bounds") + end + bounds = ConstraintBounds() + for (const_key, constraint_array) in get_constraints(get_optimization_container(model)) + # TODO: handle this at compile and not at run time + if isa(constraint_array, SparseAxisArray) + for idx in eachindex(constraint_array) + constraint_array[idx] == 0.0 && continue + con_obj = JuMP.constraint_object(constraint_array[idx]) + update_coefficient_bounds(bounds, con_obj, (const_key, idx)) + update_rhs_bounds(bounds, con_obj, (const_key, idx)) + end + else + for idx in Iterators.product(constraint_array.axes...) + !isassigned(constraint_array, idx...) && continue + con_obj = JuMP.constraint_object(constraint_array[idx...]) + update_coefficient_bounds(bounds, con_obj, (const_key, idx)) + update_rhs_bounds(bounds, con_obj, (const_key, idx)) + end + end + end + return bounds +end + +function get_variable_numerical_bounds(model::OperationModel) + if !is_built(model) + error("Model not built, can't calculate variable numerical bounds") + end + bounds = VariableBounds() + for (variable_key, variable_array) in get_variables(get_optimization_container(model)) + if isa(variable_array, SparseAxisArray) + for idx in eachindex(variable_array) + var = variable_array[idx] + var == 0.0 && continue + update_variable_bounds(bounds, var, (variable_key, idx)) + end + else + for idx in Iterators.product(variable_array.axes...) + var = variable_array[idx...] + update_variable_bounds(bounds, var, (variable_key, idx)) + end + end + end + return bounds +end diff --git a/src/operation/operation_model_glue.jl b/src/operation/operation_model_glue.jl new file mode 100644 index 0000000..d18de2a --- /dev/null +++ b/src/operation/operation_model_glue.jl @@ -0,0 +1,99 @@ +# OperationModel methods that dispatch on the abstract type but call into +# POM-defined functions on concrete stores or numerical-bounds types. Moved +# out of IOM's operation_model_interface.jl because the call chain terminates +# in POM-only methods (read_outputs, list_keys, get_variable_numerical_bounds, +# instantiate_network_model!, …). + +function _check_numerical_bounds(model::OperationModel) + variable_bounds = get_variable_numerical_bounds(model) + if variable_bounds.bounds.max - variable_bounds.bounds.min > 1e9 + @warn "Variable bounds range is $(variable_bounds.bounds.max - variable_bounds.bounds.min) and can result in numerical problems for the solver. \\ + max_bound_variable = $(encode_key_as_string(variable_bounds.bounds.max_index)) \\ + min_bound_variable = $(encode_key_as_string(variable_bounds.bounds.min_index)) \\ + Run get_detailed_variable_numerical_bounds on the model for a deeper analysis" + else + @info "Variable bounds range is [$(variable_bounds.bounds.min) $(variable_bounds.bounds.max)]" + end + + constraint_bounds = get_constraint_numerical_bounds(model) + if constraint_bounds.coefficient.max - constraint_bounds.coefficient.min > 1e9 + @warn "Constraint coefficient bounds range is $(constraint_bounds.coefficient.max - constraint_bounds.coefficient.min) and can result in numerical problems for the solver. \\ + max_bound_constraint = $(encode_key_as_string(constraint_bounds.coefficient.max_index)) \\ + min_bound_constraint = $(encode_key_as_string(constraint_bounds.coefficient.min_index)) \\ + Run get_detailed_constraint_numerical_bounds on the model for a deeper analysis" + else + @info "Constraint coefficient bounds range is [$(constraint_bounds.coefficient.min) $(constraint_bounds.coefficient.max)]" + end + + if constraint_bounds.rhs.max - constraint_bounds.rhs.min > 1e9 + @warn "Constraint right-hand-side bounds range is $(constraint_bounds.rhs.max - constraint_bounds.rhs.min) and can result in numerical problems for the solver. \\ + max_bound_constraint = $(encode_key_as_string(constraint_bounds.rhs.max_index)) \\ + min_bound_constraint = $(encode_key_as_string(constraint_bounds.rhs.min_index)) \\ + Run get_detailed_constraint_numerical_bounds on the model for a deeper analysis" + else + @info "Constraint right-hand-side bounds [$(constraint_bounds.rhs.min) $(constraint_bounds.rhs.max)]" + end + return +end + +function _pre_solve_model_checks(model::OperationModel, optimizer = nothing) + jump_model = get_jump_model(model) + if optimizer !== nothing + JuMP.set_optimizer(jump_model, optimizer) + end + + if JuMP.mode(jump_model) != JuMP.DIRECT + if JuMP.backend(jump_model).state == MOIU.NO_OPTIMIZER + error("No Optimizer has been defined, can't solve the operational problem") + end + else + @assert get_direct_mode_optimizer(get_settings(model)) + end + + optimizer_name = JuMP.solver_name(jump_model) + @info "$(get_name(model)) optimizer set to: $optimizer_name" + settings = get_settings(model) + if get_check_numerical_bounds(settings) + @info "Checking Numerical Bounds" + TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Numerical Bounds Check" begin + _check_numerical_bounds(model) + end + end + return +end + +function list_names(model::OperationModel, ::Type{T}) where {T <: OptimizationKeyType} + return encode_keys_as_strings( + list_keys(get_store(model), T), + ) +end + +read_dual(model::OperationModel, key::ConstraintKey) = _read_outputs(model, key) +read_parameter(model::OperationModel, key::ParameterKey) = _read_outputs(model, key) +read_aux_variable(model::OperationModel, key::AuxVarKey) = _read_outputs(model, key) +read_variable(model::OperationModel, key::VariableKey) = _read_outputs(model, key) +read_expression(model::OperationModel, key::ExpressionKey) = _read_outputs(model, key) + +function _read_outputs(model::OperationModel, key::OptimizationContainerKey) + array = read_outputs(get_store(model), key) + return to_outputs_dataframe(array, nothing, Val(TableFormat.LONG)) +end + +read_optimizer_stats(model::OperationModel) = read_optimizer_stats(get_store(model)) + +list_aux_variable_keys(x::OperationModel) = list_keys(get_store(x), AuxVariableType) +list_aux_variable_names(x::OperationModel) = list_names(x, AuxVariableType) +list_variable_keys(x::OperationModel) = list_keys(get_store(x), VariableType) +list_variable_names(x::OperationModel) = list_names(x, VariableType) +list_parameter_keys(x::OperationModel) = list_keys(get_store(x), ParameterType) +list_parameter_names(x::OperationModel) = list_names(x, ParameterType) +list_dual_keys(x::OperationModel) = list_keys(get_store(x), ConstraintType) +list_dual_names(x::OperationModel) = list_names(x, ConstraintType) +list_expression_keys(x::OperationModel) = list_keys(get_store(x), ExpressionType) +list_expression_names(x::OperationModel) = list_names(x, ExpressionType) + +function list_all_keys(x::OperationModel) + return Iterators.flatten( + list_fields(get_store(x), T) for T in STORE_CONTAINER_TYPES + ) +end diff --git a/src/operation/optimization_debugging.jl b/src/operation/optimization_debugging.jl new file mode 100644 index 0000000..b67d7b1 --- /dev/null +++ b/src/operation/optimization_debugging.jl @@ -0,0 +1,110 @@ +""" +Each Tuple corresponds to (con_name, internal_index, moi_index) +""" +function get_all_constraint_index(model::OperationModel) + con_index = Vector{Tuple{ConstraintKey, Int, Int}}() + container = get_optimization_container(model) + for (key, value) in get_constraints(container) + for (idx, constraint) in enumerate(value) + moi_index = JuMP.optimizer_index(constraint) + push!(con_index, (key, idx, moi_index.value)) + end + end + return con_index +end + +""" +Each Tuple corresponds to (con_name, internal_index, moi_index) +""" +function get_all_variable_index(model::OperationModel) + var_keys = get_all_variable_keys(model) + return [(encode_key(v[1]), v[2], v[3]) for v in var_keys] +end + +function get_all_variable_keys(model::OperationModel) + var_index = Vector{Tuple{VariableKey, Int, Int}}() + container = get_optimization_container(model) + for (key, value) in get_variables(container) + for (idx, variable) in enumerate(value) + moi_index = JuMP.optimizer_index(variable) + push!(var_index, (key, idx, moi_index.value)) + end + end + return var_index +end + +function get_constraint_index(model::OperationModel, index::Int) + container = get_optimization_container(model) + constraints = get_constraints(container) + for i in get_all_constraint_index(model) + if i[3] == index + return constraints[i[1]].data[i[2]] + end + end + @info "Index not found" + return +end + +function get_variable_index(model::OperationModel, index::Int) + container = get_optimization_container(model) + variables = get_variables(container) + for i in get_all_variable_keys(model) + if i[3] == index + return variables[i[1]].data[i[2]] + end + end + @info "Index not found" + return +end + +function get_detailed_constraint_numerical_bounds(model::OperationModel) + if !is_built(model) + error("Model not built, can't calculate constraint numerical bounds") + end + constraint_bounds = Dict() + for (const_key, constraint_array) in get_constraints(get_optimization_container(model)) + if isa(constraint_array, SparseAxisArray) + bounds = ConstraintBounds() + for idx in eachindex(constraint_array) + constraint_array[idx] == 0.0 && continue + con_obj = JuMP.constraint_object(constraint_array[idx]) + update_coefficient_bounds(bounds, con_obj, idx) + update_rhs_bounds(bounds, con_obj, idx) + end + constraint_bounds[const_key] = bounds + else + bounds = ConstraintBounds() + for idx in Iterators.product(constraint_array.axes...) + con_obj = JuMP.constraint_object(constraint_array[idx...]) + update_coefficient_bounds(bounds, con_obj, idx) + update_rhs_bounds(bounds, con_obj, idx) + end + constraint_bounds[const_key] = bounds + end + end + return constraint_bounds +end + +function get_detailed_variable_numerical_bounds(model::OperationModel) + if !is_built(model) + error("Model not built, can't calculate variable numerical bounds") + end + variable_bounds = Dict() + for (variable_key, variable_array) in get_variables(get_optimization_container(model)) + bounds = VariableBounds() + if isa(variable_array, SparseAxisArray) + for idx in eachindex(variable_array) + var = variable_array[idx] + var == 0.0 && continue + update_variable_bounds(bounds, var, idx) + end + else + for idx in Iterators.product(variable_array.axes...) + var = variable_array[idx...] + update_variable_bounds(bounds, var, idx) + end + end + variable_bounds[variable_key] = bounds + end + return variable_bounds +end diff --git a/src/operation/optimization_problem_outputs.jl b/src/operation/optimization_problem_outputs.jl new file mode 100644 index 0000000..2db94f1 --- /dev/null +++ b/src/operation/optimization_problem_outputs.jl @@ -0,0 +1,1116 @@ +""" + mutable struct OptimizationProblemOutputs <: Outputs + +Container for the outputs of an optimization problem, including variable values, dual values, +parameter values, expression values, and optimizer statistics. + +This type stores all output data from solving an optimization problem and provides methods +to read, export, and serialize the outputs. Instead of accessing the output dictionary +fields directly, use the `read_foo` functions. + +# Fields +- `base_power::Float64`: Base power used for per-unit conversion +- `timestamps::Vector{Dates.DateTime}`: Time stamps for each step in the outputs +- `source_data::Union{Nothing, InfrastructureSystemsType}`: Reference to the source data (e.g., system) +- `source_data_uuid::Base.UUID`: UUID of the source data for validation. Internal usage. +- `aux_variable_values::Dict{AuxVarKey, DataFrame}`: Auxiliary variable outputs. See [`read_aux_variable`](@ref) and [`read_aux_variables`](@ref) +- `variable_values::Dict{VariableKey, DataFrame}`: Decision variable outputs. See [`read_variable`](@ref) and [`read_variables`](@ref) +- `dual_values::Dict{ConstraintKey, DataFrame}`: Dual outputs. See [`read_dual`](@ref) and [`read_duals`](@ref) +- `parameter_values::Dict{ParameterKey, DataFrame}`: Parameter outputs. See [`read_parameter`](@ref) and [`read_parameters`](@ref) +- `expression_values::Dict{ExpressionKey, DataFrame}`: Expression outputs. See [`read_expression`](@ref) and [`read_expressions`](@ref) +- `optimizer_stats::DataFrame`: Optimizer statistics for each solve +- `optimization_container_metadata::OptimizationContainerMetadata`: Metadata about the optimization container. Internal usage. +- `model_type::String`: Type of optimization model. Internal usage. +- `outputs_dir::String`: Directory where outputs are stored +- `output_dir::String`: Directory for exported output + +See also: [`OptimizerStats`](@ref), [`OptimizationProblemOutputsExport`](@ref) +""" +mutable struct OptimizationProblemOutputs <: Outputs + base_power::Float64 + timestamps::Vector{Dates.DateTime} + source_data::Union{Nothing, InfrastructureSystemsType} + source_data_uuid::Base.UUID + aux_variable_values::Dict{AuxVarKey, DataFrame} + variable_values::Dict{VariableKey, DataFrame} + dual_values::Dict{ConstraintKey, DataFrame} + parameter_values::Dict{ParameterKey, DataFrame} + expression_values::Dict{ExpressionKey, DataFrame} + optimizer_stats::DataFrame + optimization_container_metadata::OptimizationContainerMetadata + model_type::String + outputs_dir::String + output_dir::String +end + +function OptimizationProblemOutputs( + base_power, + timestamps::StepRange{Dates.DateTime, Dates.Millisecond}, + source_data, + source_data_uuid, + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + optimization_container_metadata, + model_type, + outputs_dir, + output_dir, +) + return OptimizationProblemOutputs( + base_power, + collect(timestamps), + source_data, + source_data_uuid, + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + optimization_container_metadata, + model_type, + outputs_dir, + output_dir, + ) +end + +list_aux_variable_keys(res::OptimizationProblemOutputs) = + collect(keys(res.aux_variable_values)) +list_aux_variable_names(res::OptimizationProblemOutputs) = + encode_keys_as_strings(keys(res.aux_variable_values)) +list_variable_keys(res::OptimizationProblemOutputs) = collect(keys(res.variable_values)) +list_variable_names(res::OptimizationProblemOutputs) = + encode_keys_as_strings(keys(res.variable_values)) +list_parameter_keys(res::OptimizationProblemOutputs) = collect(keys(res.parameter_values)) +list_parameter_names(res::OptimizationProblemOutputs) = + encode_keys_as_strings(keys(res.parameter_values)) +list_dual_keys(res::OptimizationProblemOutputs) = collect(keys(res.dual_values)) +list_dual_names(res::OptimizationProblemOutputs) = + encode_keys_as_strings(keys(res.dual_values)) +list_expression_keys(res::OptimizationProblemOutputs) = collect(keys(res.expression_values)) +list_expression_names(res::OptimizationProblemOutputs) = + encode_keys_as_strings(keys(res.expression_values)) +get_timestamps(res::OptimizationProblemOutputs) = res.timestamps +get_model_base_power(res::OptimizationProblemOutputs) = res.base_power +get_dual_values(res::OptimizationProblemOutputs) = res.dual_values +get_expression_values(res::OptimizationProblemOutputs) = res.expression_values +get_variable_values(res::OptimizationProblemOutputs) = res.variable_values +get_aux_variable_values(res::OptimizationProblemOutputs) = res.aux_variable_values +get_total_cost(res::OptimizationProblemOutputs) = get_objective_value(res) +get_optimizer_stats(res::OptimizationProblemOutputs) = res.optimizer_stats +get_parameter_values(res::OptimizationProblemOutputs) = res.parameter_values +get_source_data(res::OptimizationProblemOutputs) = res.source_data + +make_system_filename(sys::IS.InfrastructureSystemsContainer) = + make_system_filename(get_system_uuid(sys)) +make_system_filename(sys_uuid::Union{Base.UUID, AbstractString}) = "system-$(sys_uuid).json" + +""" +Load the system from disk if not already set, and return it. + +Currently only used in the tests, not downstream in POM. +""" +function load_system(res::OptimizationProblemOutputs; kwargs...) + !isnothing(get_source_data(res)) && return + file = joinpath(get_outputs_dir(res), make_system_filename(get_source_data_uuid(res))) + if isfile(file) + sys = IS.InfrastructureSystemsContainer(file; time_series_read_only = true) + @info "De-serialized the system from files." + else + error("Could not locate system file: $file") + end + set_source_data!(res, sys) + return +end + +get_forecast_horizon(res::OptimizationProblemOutputs) = length(get_timestamps(res)) +get_output_dir(res::OptimizationProblemOutputs) = res.output_dir +get_outputs_dir(res::OptimizationProblemOutputs) = res.outputs_dir +get_source_data_uuid(res::OptimizationProblemOutputs) = res.source_data_uuid + +get_output_values(x::OptimizationProblemOutputs, ::AuxVarKey) = x.aux_variable_values +get_output_values(x::OptimizationProblemOutputs, ::ConstraintKey) = x.dual_values +get_output_values(x::OptimizationProblemOutputs, ::ExpressionKey) = x.expression_values +get_output_values(x::OptimizationProblemOutputs, ::ParameterKey) = x.parameter_values +get_output_values(x::OptimizationProblemOutputs, ::VariableKey) = x.variable_values + +function get_objective_value(res::OptimizationProblemOutputs, execution = 1) + return res.optimizer_stats[execution, :objective_value] +end + +function get_resolution(res::OptimizationProblemOutputs) + # Method return the resolution between timestamps. + # If multiple resolutions are present it returns the first observed. + # If single timestamp is used, it return. + diff_res = diff(get_timestamps(res)) + if !isempty(diff_res) + unique!(diff_res) + if length(diff_res) == 1 + return only(diff_res) + else + @warn "Multiple resolutions detected, returning the first resolution." + return first(diff_res) + end + end + return +end + +function get_realized_timestamps( + res::IS.Outputs; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, +) + timestamps = get_timestamps(res) + resolution = get_resolution(res) + intervals = diff(timestamps) + if isempty(intervals) && isnothing(resolution) + interval = Dates.Millisecond(1) + resolution = Dates.Millisecond(1) + elseif !isempty(intervals) && isnothing(resolution) + interval = first(intervals) + resolution = interval + elseif isempty(intervals) && !isnothing(resolution) + interval = resolution + else + interval = first(intervals) + end + horizon = get_forecast_horizon(res) + start_time = isnothing(start_time) ? first(timestamps) : start_time + end_time = + if isnothing(len) + last(timestamps) + interval - resolution + else + start_time + (len - 1) * resolution + end + + requested_range = start_time:resolution:end_time + available_range = + first(timestamps):resolution:(last(timestamps) + (horizon - 1) * resolution) + invalid_timestamps = setdiff(requested_range, available_range) + + if !isempty(invalid_timestamps) + msg = "Requested time does not match available outputs" + @error msg + throw(IS.InvalidValue(msg)) + end + + return requested_range +end + +function export_output( + ::Type{CSV.File}, + path, + key::OptimizationContainerKey, + timestamp::Dates.DateTime, + df::DataFrame, +) + name = encode_key_as_string(key) + export_output(CSV.File, path, name, timestamp, df) + return +end + +function export_output( + ::Type{CSV.File}, + path, + name::AbstractString, + timestamp::Dates.DateTime, + df::DataFrame, +) + filename = joinpath(path, name * "_" * convert_for_path(timestamp) * ".csv") + export_output(CSV.File, filename, df) + return +end + +function export_output( + ::Type{CSV.File}, + path, + key::OptimizationContainerKey, + df::DataFrame, +) + name = encode_key_as_string(key) + export_output(CSV.File, path, name, df) + return +end + +function export_output( + ::Type{CSV.File}, + path, + name::AbstractString, + df::DataFrame, +) + filename = joinpath(path, name * ".csv") + export_output(CSV.File, filename, df) + return +end + +function export_output(::Type{CSV.File}, filename, df::DataFrame) + open(filename, "w") do io + CSV.write(io, df) + end + + @debug "Exported $filename" + return +end + +""" +Exports all outputs from the operations problem. +""" +function export_outputs(outputs::OptimizationProblemOutputs; kwargs...) + exports = OptimizationProblemOutputsExport( + "Problem"; + store_all_duals = true, + store_all_parameters = true, + store_all_variables = true, + store_all_aux_variables = true, + ) + return export_outputs(outputs, exports; kwargs...) +end + +function export_outputs( + outputs::OptimizationProblemOutputs, + exports::OptimizationProblemOutputsExport; + file_type = CSV.File, +) + file_type != CSV.File && error("only CSV.File is currently supported") + for (source, decider, label) in [ + (outputs.variable_values, should_export_variable, "variables"), + (outputs.aux_variable_values, should_export_aux_variable, "aux_variables"), + (outputs.dual_values, should_export_dual, "duals"), + (outputs.parameter_values, should_export_parameter, "parameters"), + (outputs.expression_values, should_export_expression, "expressions"), + ] + export_path = mkpath(joinpath(get_output_dir(outputs), label)) + for (key, df) in source + if decider(exports, key) + export_output(file_type, export_path, key, df) + end + end + end + + if exports.optimizer_stats + export_output( + file_type, + joinpath(get_output_dir(outputs), "optimizer_stats.csv"), + outputs.optimizer_stats, + ) + end + + @info "Exported OptimizationProblemOutputs to $(get_output_dir(outputs))" +end + +function _deserialize_key( + ::Type{<:OptimizationContainerKey}, + outputs::OptimizationProblemOutputs, + name::AbstractString, +) + return deserialize_key(outputs.optimization_container_metadata, name) +end + +function _deserialize_key( + ::Type{T}, + ::OptimizationProblemOutputs, + args..., +) where {T <: OptimizationContainerKey} + return make_key(T, args...) +end + +function _validate_keys(existing_keys, output_keys) + diff = setdiff(output_keys, existing_keys) + if !isempty(diff) + throw(InvalidValue("These keys are not stored: $diff")) + end + return +end + +read_optimizer_stats(res::OptimizationProblemOutputs) = res.optimizer_stats + +""" +Set the system in the outputs instance. + +Throws InvalidValue if the source UUID is incorrect. +""" +function set_source_data!( + res::OptimizationProblemOutputs, + source::InfrastructureSystemsType, +) + source_uuid = get_uuid(source) + if source_uuid != res.source_data_uuid + throw( + InvalidValue( + "System mismatch. $sys_uuid does not match the stored value of $(res.source_uuid)", + ), + ) + end + + res.source_data = source + return +end + +const _PROBLEM_OUTPUTS_FILENAME = "problem_outputs.bin" + +# TODO test this in IS +""" +Serialize the outputs to a binary file. + +It is recommended that `directory` be the directory that contains a serialized +OperationModel. That will allow automatic deserialization of the PowerSystems.System. +The `OptimizationProblemOutputs` instance can be deserialized with `OptimizationProblemOutputs(directory)`. +""" +function serialize_outputs(res::OptimizationProblemOutputs, directory::AbstractString) + mkpath(directory) + filename = joinpath(directory, _PROBLEM_OUTPUTS_FILENAME) + isfile(filename) && rm(filename) + Serialization.serialize(filename, _copy_for_serialization(res)) + @info "Serialize OptimizationProblemOutputs to $filename" +end + +""" +Construct a OptimizationProblemOutputs instance from a serialized directory. It is up to the +user or a higher-level package to set the source data using [`set_source_data!`](@ref). +""" +function OptimizationProblemOutputs(directory::AbstractString) + filename = joinpath(directory, _PROBLEM_OUTPUTS_FILENAME) + isfile(filename) || error("No outputs file exists in $directory") + return Serialization.deserialize(filename) +end + +function _copy_for_serialization(res::OptimizationProblemOutputs) + return OptimizationProblemOutputs( + res.base_power, + res.timestamps, + nothing, + res.source_data_uuid, + res.aux_variable_values, + res.variable_values, + res.dual_values, + res.parameter_values, + res.expression_values, + res.optimizer_stats, + res.optimization_container_metadata, + res.model_type, + res.outputs_dir, + res.output_dir, + ) +end + +function _read_outputs( + output_values::Dict{<:OptimizationContainerKey, DataFrame}, + container_keys, + timestamps::Vector{Dates.DateTime}, + time_ids, + base_power::Float64, + base_timestamps::Vector{Dates.DateTime}, + table_format::TableFormat, +) + existing_keys = keys(output_values) + container_keys = container_keys === nothing ? existing_keys : container_keys + _validate_keys(existing_keys, container_keys) + outputs = Dict{OptimizationContainerKey, DataFrame}() + IS.@assert_op length(time_ids) == length(timestamps) + df_timestamps = DataFrame(:DateTime => timestamps, :time_index => time_ids) + filter_timestamps = timestamps != base_timestamps + + for (key, df) in output_values + if !in(key, container_keys) + continue + end + if filter_timestamps + df = @subset(df, :time_index .∈ Ref(time_ids)) + end + first_dim_col = get_first_dimension_output_column_name(key) + second_dim_col = get_second_dimension_output_column_name(key) + component_cols = [first_dim_col] + if second_dim_col in names(df) + push!(component_cols, second_dim_col) + if table_format == TableFormat.WIDE + error( + "Wide format is not supported with 3-dimensional outputs", + ) + end + end + num_components = DataFrames.nrow(unique(df[:, component_cols])) + num_rows = DataFrames.nrow(df) + if num_rows % num_components != 0 + error( + "num_rows = $num_rows is not divisible by num_components = $num_components", + ) + end + num_rows_per_component = num_rows ÷ num_components + if num_rows_per_component == length(time_ids) == length(timestamps) + tmp_df = innerjoin(df, df_timestamps; on = :time_index) + if DataFrames.nrow(tmp_df) != DataFrames.nrow(df) + error( + "Bug: Unexpectedly dropped rows: df2 = $tmp_df orig = $(outputs[key])", + ) + end + outputs[key] = select(tmp_df, [:DateTime, Symbol.(component_cols)..., :value]) + else + @warn "Length of variables is different than timestamps. Ignoring timestamps." + outputs[key] = deepcopy(df) + end + outputs[key] = _handle_natural_units(outputs[key], base_power, key) + if table_format == TableFormat.WIDE + outputs[key] = DataFrames.unstack(outputs[key], first_dim_col, "value") + end + end + return outputs +end + +""" +Convert the value column to natural units, if required by the key. +Does not mutate the input dataframe. +""" +function _handle_natural_units( + df::DataFrame, + base_power::Float64, + key::OptimizationContainerKey, +) + return if convert_output_to_natural_units(key) + @transform(df, :value = :value * base_power) + else + df + end +end + +function _process_timestamps( + res::OptimizationProblemOutputs, + start_time::Union{Nothing, Dates.DateTime}, + len::Union{Int, Nothing}, +) + if start_time === nothing + start_time = first(get_timestamps(res)) + elseif start_time ∉ get_timestamps(res) + throw(InvalidValue("start_time not in output timestamps")) + end + + if startswith(res.model_type, "EmulationModel{") + def_len = DataFrames.nrow(get_optimizer_stats(res)) + requested_range = + collect(findfirst(x -> x >= start_time, get_timestamps(res)):def_len) + timestamps = repeat(get_timestamps(res), def_len) + else + timestamps = get_timestamps(res) + requested_range = findall(x -> x >= start_time, timestamps) + def_len = length(requested_range) + end + actual_len = if len === nothing + def_len + elseif len < 0 + throw(InvalidValue("len cannot be negative: $len")) + elseif len > def_len + throw(InvalidValue("requested outputs have less than $len values")) + else + len + end + timestamp_ids = requested_range[1:actual_len] + return timestamp_ids, timestamps[timestamp_ids] +end + +""" +Return the values for the requested variable key for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + +- `res::OptimizationProblemOutputs`: Optimization problem outputs +- `variable::Tuple{Type{<:VariableType}, Type{<:IS.InfrastructureSystemsComponent}`: Tuple with variable type + and device type for the desired outputs +- `start_time::Dates.DateTime`: Start time of the requested outputs +- `len::Int`: length of outputs +- `table_format::TableFormat`: Format of the table to be returned. Default is + `TableFormat.LONG` where the columns are `DateTime`, `name`, and `value` when the data + has two dimensions and `DateTime`, `name`, `name2`, and `value` when the data has three + dimensions. + Set to it `TableFormat.WIDE` to pivot the names as columns. + Note: `TableFormat.WIDE` is not supported when the data has more than two dimensions. +""" +function read_variable( + res::OptimizationProblemOutputs, + args...; + kwargs..., +) + key = VariableKey(args...) + return read_variable(res, key; kwargs...) +end + +function read_variable(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) + return read_variable(res, _deserialize_key(VariableKey, res, key); kwargs...) +end + +function read_variable( + res::OptimizationProblemOutputs, + key::VariableKey; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + return read_outputs_with_keys( + res, + [key]; + start_time = start_time, + len = len, + table_format = table_format, + )[key] +end + +""" +Return the values for the requested variable keys for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `variables::Vector{Tuple{Type{<:VariableType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with variable type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_variables(res::OptimizationProblemOutputs, variables; kwargs...) + return read_variables(res, [VariableKey(x...) for x in variables]; kwargs...) +end + +function read_variables( + res::OptimizationProblemOutputs, + variables::Vector{<:AbstractString}; + kwargs..., +) + return read_variables( + res, + [_deserialize_key(VariableKey, res, x) for x in variables]; + kwargs..., + ) +end + +function read_variables( + res::OptimizationProblemOutputs, + variables::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + output_values = + read_outputs_with_keys( + res, + variables; + start_time = start_time, + len = len, + table_format = table_format, + ) + return Dict(encode_key_as_string(k) => v for (k, v) in output_values) +end + +""" +Return the values for all variables. +""" +function read_variables(res::Outputs; kwargs...) + return Dict(x => read_variable(res, x; kwargs...) for x in list_variable_names(res)) +end + +""" +Return the values for the requested dual key for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `dual::Tuple{Type{<:ConstraintType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with dual type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_dual( + res::OptimizationProblemOutputs, + args...; + kwargs..., +) + key = ConstraintKey(args...) + return read_dual(res, key; kwargs...) +end + +function read_dual(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) + return read_dual(res, _deserialize_key(ConstraintKey, res, key); kwargs...) +end + +function read_dual( + res::OptimizationProblemOutputs, + key::ConstraintKey; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + return read_outputs_with_keys( + res, + [key]; + start_time = start_time, + len = len, + table_format = table_format, + )[key] +end + +""" +Return the values for the requested dual keys for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `duals::Vector{Tuple{Type{<:ConstraintType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with dual type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_duals(res::OptimizationProblemOutputs, duals; kwargs...) + return read_duals(res, [ConstraintKey(x...) for x in duals]; kwargs...) +end + +function read_duals( + res::OptimizationProblemOutputs, + duals::Vector{<:AbstractString}; + kwargs..., +) + return read_duals( + res, + [_deserialize_key(ConstraintKey, res, x) for x in duals]; + kwargs..., + ) +end + +function read_duals( + res::OptimizationProblemOutputs, + duals::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + output_values = read_outputs_with_keys( + res, + duals; + start_time = start_time, + len = len, + table_format = table_format, + ) + return Dict(encode_key_as_string(k) => v for (k, v) in output_values) +end + +""" +Return the values for all duals. +""" +function read_duals(res::Outputs; kwargs...) + duals = Dict(x => read_dual(res, x; kwargs...) for x in list_dual_names(res)) +end + +""" +Return the values for the requested parameter key for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `parameter::Tuple{Type{<:ParameterType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with parameter type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_parameter( + res::OptimizationProblemOutputs, + args...; + kwargs..., +) + key = ParameterKey(args...) + return read_parameter(res, key; kwargs...) +end + +function read_parameter(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) + return read_parameter(res, _deserialize_key(ParameterKey, res, key); kwargs...) +end + +function read_parameter( + res::OptimizationProblemOutputs, + key::ParameterKey; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + return read_outputs_with_keys( + res, + [key]; + start_time = start_time, + len = len, + table_format = table_format, + )[key] +end + +""" +Return the values for the requested parameter keys for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `parameters::Vector{Tuple{Type{<:ParameterType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with parameter type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_parameters(res::OptimizationProblemOutputs, parameters; kwargs...) + return read_parameters(res, [ParameterKey(x...) for x in parameters]; kwargs...) +end + +function read_parameters( + res::OptimizationProblemOutputs, + parameters::Vector{<:AbstractString}; + kwargs..., +) + return read_parameters( + res, + [_deserialize_key(ParameterKey, res, x) for x in parameters]; + kwargs..., + ) +end + +function read_parameters( + res::OptimizationProblemOutputs, + parameters::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + output_values = + read_outputs_with_keys( + res, + parameters; + start_time = start_time, + len = len, + table_format = table_format, + ) + return Dict(encode_key_as_string(k) => v for (k, v) in output_values) +end + +""" +Return the values for all parameters. +""" +function read_parameters(res::Outputs; kwargs...) + parameters = + Dict(x => read_parameter(res, x; kwargs...) for x in list_parameter_names(res)) +end + +""" +Return the values for the requested aux_variable key for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `aux_variable::Tuple{Type{<:AuxVariableType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with aux_variable type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_aux_variable( + res::OptimizationProblemOutputs, + args...; + kwargs..., +) + key = AuxVarKey(args...) + return read_aux_variable(res, key; kwargs...) +end + +function read_aux_variable(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) + return read_aux_variable(res, _deserialize_key(AuxVarKey, res, key); kwargs...) +end + +function read_aux_variable( + res::OptimizationProblemOutputs, + key::AuxVarKey; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + return read_outputs_with_keys( + res, + [key]; + start_time = start_time, + len = len, + table_format = table_format, + )[key] +end + +""" +Return the values for the requested aux_variable keys for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `aux_variables::Vector{Tuple{Type{<:AuxVariableType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with aux_variable type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_aux_variables(res::OptimizationProblemOutputs, aux_variables; kwargs...) + return read_aux_variables(res, [AuxVarKey(x...) for x in aux_variables]; kwargs...) +end + +function read_aux_variables( + res::OptimizationProblemOutputs, + aux_variables::Vector{<:AbstractString}; + kwargs..., +) + return read_aux_variables( + res, + [_deserialize_key(AuxVarKey, res, x) for x in aux_variables]; + kwargs..., + ) +end + +function read_aux_variables( + res::OptimizationProblemOutputs, + aux_variables::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + output_values = + read_outputs_with_keys( + res, + aux_variables; + start_time = start_time, + len = len, + table_format = table_format, + ) + return Dict(encode_key_as_string(k) => v for (k, v) in output_values) +end + +""" +Return the values for all auxiliary variables. +""" +function read_aux_variables(res::Outputs; kwargs...) + return Dict( + x => read_aux_variable(res, x; kwargs...) for x in list_aux_variable_names(res) + ) +end + +""" +Return the values for the requested expression key for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `expression::Tuple{Type{<:ExpressionType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with expression type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_expression( + res::OptimizationProblemOutputs, + args...; + kwargs..., +) + key = ExpressionKey(args...) + return read_expression(res, key; kwargs...) +end + +function read_expression(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) + return read_expression(res, _deserialize_key(ExpressionKey, res, key); kwargs...) +end + +function read_expression( + res::OptimizationProblemOutputs, + key::ExpressionKey; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + return read_outputs_with_keys( + res, + [key]; + start_time = start_time, + len = len, + table_format = table_format, + )[key] +end + +""" +Return the values for the requested expression keys for a problem. +Accepts a vector of keys for the return of the values. + +# Arguments + + - `expressions::Vector{Tuple{Type{<:ExpressionType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with expression type and device type for the desired outputs + - `start_time::Dates.DateTime` : initial time of the requested outputs + - `len::Int`: length of outputs +""" +function read_expressions(res::OptimizationProblemOutputs; kwargs...) + return read_expressions(res, collect(keys(res.expression_values)); kwargs...) +end + +function read_expressions(res::OptimizationProblemOutputs, expressions; kwargs...) + return read_expressions(res, [ExpressionKey(x...) for x in expressions]; kwargs...) +end + +function read_expressions( + res::OptimizationProblemOutputs, + expressions::Vector{<:AbstractString}; + kwargs..., +) + return read_expressions( + res, + [_deserialize_key(ExpressionKey, res, x) for x in expressions]; + kwargs..., + ) +end + +function read_expressions( + res::OptimizationProblemOutputs, + expressions::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + output_values = + read_outputs_with_keys( + res, + expressions; + start_time = start_time, + len = len, + table_format = table_format, + ) + return Dict(encode_key_as_string(k) => v for (k, v) in output_values) +end + +""" +Return the values for all expressions. +""" +function read_expressions(res::Outputs; kwargs...) + return Dict(x => read_expression(res, x) for x in list_expression_names(res)) +end + +function read_outputs_with_keys( + res::OptimizationProblemOutputs, + output_keys::Vector{<:OptimizationContainerKey}; + start_time::Union{Nothing, Dates.DateTime} = nothing, + len::Union{Int, Nothing} = nothing, + table_format::TableFormat = TableFormat.LONG, +) + isempty(output_keys) && return Dict{OptimizationContainerKey, DataFrame}() + (timestamp_ids, timestamps) = _process_timestamps(res, start_time, len) + + base_timestamps = get_timestamps(res) + return _read_outputs( + get_output_values(res, first(output_keys)), + output_keys, + timestamps, + timestamp_ids, + get_model_base_power(res), + base_timestamps, + table_format, + ) +end + +""" +Save the realized outputs to CSV files for all variables, paramaters, duals, auxiliary variables, +expressions, and optimizer statistics. + +# Arguments + + - `res::Outputs`: Outputs + - `save_path::AbstractString` : path to save outputs (defaults to simulation path) +""" +function export_realized_outputs(res::Outputs) + save_path = mkpath(joinpath(get_output_dir(res), "export")) + return export_realized_outputs(res, save_path) +end + +function export_realized_outputs( + res::Outputs, + save_path::AbstractString, +) + if !isdir(save_path) + throw(IS.ConflictingInputsError("Specified path is not valid.")) + end + write_data(read_outputs_with_keys(res, list_variable_keys(res)), save_path) + !isempty(list_dual_keys(res)) && + write_data( + read_outputs_with_keys(res, list_dual_keys(res)), + save_path; + name = "dual", + ) + !isempty(list_parameter_keys(res)) && write_data( + read_outputs_with_keys(res, list_parameter_keys(res)), + save_path; + name = "parameter", + ) + !isempty(list_aux_variable_keys(res)) && write_data( + read_outputs_with_keys(res, list_aux_variable_keys(res)), + save_path; + name = "aux_variable", + ) + !isempty(list_expression_keys(res)) && write_data( + read_outputs_with_keys(res, list_expression_keys(res)), + save_path; + name = "expression", + ) + export_optimizer_stats(res, save_path) + files = readdir(save_path) + compute_file_hash(save_path, files) + @info("Files written to $save_path folder.") + return save_path +end + +""" +Save the optimizer statistics to CSV or JSON + +# Arguments + + - `res::Union{OptimizationProblemOutputs, SimulationProblmeOutputs`: Outputs + - `directory::AbstractString` : target directory + - `format = "CSV"` : can be "csv" or "json +""" +function export_optimizer_stats( + res::Outputs, + directory::AbstractString; + format = "csv", +) + data = read_optimizer_stats(res) + isnothing(data) && return + if uppercase(format) == "CSV" + CSV.write(joinpath(directory, "optimizer_stats.csv"), data) + elseif uppercase(format) == "JSON" + JSON.write(joinpath(directory, "optimizer_stats.json"), JSON.json(to_dict(data))) + else + throw(error("writing optimizer stats only supports csv or json formats")) + end +end + +function write_data( + vars_outputs::Dict, + time::DataFrame, + save_path::AbstractString, +) + for (k, v) in vars_outputs + var = DataFrame() + if size(time, 1) == size(v, 1) + var = hcat(time, v) + else + var = v + end + file_path = joinpath(save_path, "$(k).csv") + CSV.write(file_path, var) + end +end + +function write_data( + data::DataFrame, + save_path::AbstractString, + file_name::String, +) + if isfile(save_path) + save_path = dirname(save_path) + end + file_path = joinpath(save_path, "$(file_name).csv") + CSV.write(file_path, data) + return +end + +# writing a dictionary of dataframes to files +function write_data(vars_outputs::Dict, save_path::String; kwargs...) + name = get(kwargs, :name, "") + for (k, v) in vars_outputs + keyname = encode_key_as_string(k) + file_path = joinpath(save_path, "$name$keyname.csv") + @debug "writing" file_path + if isempty(vars_outputs[k]) + @debug "$name$k is empty, not writing $file_path" + else + CSV.write(file_path, vars_outputs[k]) + end + end +end diff --git a/src/operation/optimization_problem_outputs_export.jl b/src/operation/optimization_problem_outputs_export.jl new file mode 100644 index 0000000..8744722 --- /dev/null +++ b/src/operation/optimization_problem_outputs_export.jl @@ -0,0 +1,130 @@ +""" + struct OptimizationProblemOutputsExport + +Configuration for exporting optimization problem outputs to files. + +Specifies which variables, duals, parameters, expressions, and auxiliary variables +should be exported when calling [`export_outputs`](@ref) on an +[`OptimizationProblemOutputs`](@ref) instance. + +# Fields +- `name::Symbol`: Name identifier for this export configuration +- `duals::Set{ConstraintKey}`: Specific dual values to export +- `expressions::Set{ExpressionKey}`: Specific expression values to export +- `parameters::Set{ParameterKey}`: Specific parameter values to export +- `variables::Set{VariableKey}`: Specific variable values to export +- `aux_variables::Set{AuxVarKey}`: Specific auxiliary variable values to export +- `optimizer_stats::Bool`: Whether to export optimizer statistics +- `store_all_flags::Dict{Symbol, Bool}`: Flags indicating whether to export all values + of each type (e.g., all variables, all duals). Set via constructor keyword arguments + like `store_all_variables = true`. When a flag is true, all values of that type are + exported regardless of what specific keys are passed in the corresponding set. + +# Example +```julia +export_config = OptimizationProblemOutputsExport( + "MyExport"; + store_all_variables = true, + store_all_duals = false, + optimizer_stats = true, +) +export_outputs(outputs, export_config) +``` + +See also: [`OptimizationProblemOutputs`](@ref), [`export_outputs`](@ref) +""" +struct OptimizationProblemOutputsExport + name::Symbol + duals::Set{ConstraintKey} + expressions::Set{ExpressionKey} + parameters::Set{ParameterKey} + variables::Set{VariableKey} + aux_variables::Set{AuxVarKey} + optimizer_stats::Bool + store_all_flags::Dict{Symbol, Bool} + + function OptimizationProblemOutputsExport( + name, + duals, + expressions, + parameters, + variables, + aux_variables, + optimizer_stats, + store_all_flags, + ) + duals = _check_fields(duals) + expressions = _check_fields(expressions) + parameters = _check_fields(parameters) + variables = _check_fields(variables) + aux_variables = _check_fields(aux_variables) + new( + name, + duals, + expressions, + parameters, + variables, + aux_variables, + optimizer_stats, + store_all_flags, + ) + end +end + +function OptimizationProblemOutputsExport( + name; + duals = Set{ConstraintKey}(), + expressions = Set{ExpressionKey}(), + parameters = Set{ParameterKey}(), + variables = Set{VariableKey}(), + aux_variables = Set{AuxVarKey}(), + optimizer_stats = true, + store_all_duals = false, + store_all_expressions = false, + store_all_parameters = false, + store_all_variables = false, + store_all_aux_variables = false, +) + store_all_flags = Dict( + :duals => store_all_duals, + :expressions => store_all_expressions, + :parameters => store_all_parameters, + :variables => store_all_variables, + :aux_variables => store_all_aux_variables, + ) + return OptimizationProblemOutputsExport( + Symbol(name), + duals, + expressions, + parameters, + variables, + aux_variables, + optimizer_stats, + store_all_flags, + ) +end + +function _check_fields(fields) + if !(typeof(fields) <: Set) + fields = Set(fields) + end + + return fields +end + +should_export_dual(x::OptimizationProblemOutputsExport, key) = + _should_export(x, :duals, key) +should_export_expression(x::OptimizationProblemOutputsExport, key) = + _should_export(x, :expressions, key) +should_export_parameter(x::OptimizationProblemOutputsExport, key) = + _should_export(x, :parameters, key) +should_export_variable(x::OptimizationProblemOutputsExport, key) = + _should_export(x, :variables, key) +should_export_aux_variable(x::OptimizationProblemOutputsExport, key) = + _should_export(x, :aux_variables, key) + +function _should_export(exports::OptimizationProblemOutputsExport, field_name, key) + exports.store_all_flags[field_name] && return true + container = getproperty(exports, field_name) + return key in container +end diff --git a/src/operation/problem_outputs.jl b/src/operation/problem_outputs.jl new file mode 100644 index 0000000..c4a70d3 --- /dev/null +++ b/src/operation/problem_outputs.jl @@ -0,0 +1,88 @@ +""" +Construct OptimizationProblemOutputs from a solved DecisionModel. +""" +function OptimizationProblemOutputs(model::DecisionModel) + status = get_run_status(model) + status != RunStatus.SUCCESSFULLY_FINALIZED && + error("problem was not solved successfully: $status") + + model_store = get_store(model) + + if isempty(model_store) + error("Model Solved as part of a Simulation.") + end + + timestamps = get_timestamps(model) + optimizer_stats = to_dataframe(get_optimizer_stats(model)) + + aux_variable_values = + Dict(x => read_aux_variable(model, x) for x in list_aux_variable_keys(model)) + variable_values = Dict(x => read_variable(model, x) for x in list_variable_keys(model)) + dual_values = Dict(x => read_dual(model, x) for x in list_dual_keys(model)) + parameter_values = + Dict(x => read_parameter(model, x) for x in list_parameter_keys(model)) + expression_values = + Dict(x => read_expression(model, x) for x in list_expression_keys(model)) + + sys = get_system(model) + + return OptimizationProblemOutputs( + get_problem_base_power(model), + timestamps, + sys, + get_uuid(sys), + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + get_metadata(get_optimization_container(model)), + IS.strip_module_name(typeof(model)), + get_output_dir(model), + mkpath(joinpath(get_output_dir(model), "outputs")), + ) +end + +""" +Construct OptimizationProblemOutputs from a solved EmulationModel. +""" +function OptimizationProblemOutputs(model::EmulationModel) + status = get_run_status(model) + status != RunStatus.SUCCESSFULLY_FINALIZED && + error("problem was not solved successfully: $status") + + model_store = get_store(model) + + if isempty(model_store) + error("Model Solved as part of a Simulation.") + end + + aux_variables = + Dict(x => read_aux_variable(model, x) for x in list_aux_variable_keys(model)) + variables = Dict(x => read_variable(model, x) for x in list_variable_keys(model)) + duals = Dict(x => read_dual(model, x) for x in list_dual_keys(model)) + parameters = Dict(x => read_parameter(model, x) for x in list_parameter_keys(model)) + expression = Dict(x => read_expression(model, x) for x in list_expression_keys(model)) + optimizer_stats = read_optimizer_stats(model) + initial_time = get_initial_time(model) + container = get_optimization_container(model) + sys = get_system(model) + + return OptimizationProblemOutputs( + get_problem_base_power(model), + StepRange(initial_time, get_resolution(model), initial_time), + sys, + get_uuid(sys), + aux_variables, + variables, + duals, + parameters, + expression, + optimizer_stats, + get_metadata(container), + IS.strip_module_name(typeof(model)), + get_output_dir(model), + mkpath(joinpath(get_output_dir(model), "outputs")), + ) +end diff --git a/src/operation/store_common.jl b/src/operation/store_common.jl new file mode 100644 index 0000000..eb09754 --- /dev/null +++ b/src/operation/store_common.jl @@ -0,0 +1,236 @@ +"""Typed container for export configuration parameters used during model output writing.""" +struct ExportParameters{E} + exports::E + exports_path::String + file_type::Type + resolution::Dates.Millisecond + horizon_count::Int +end + +function _export_container_output!( + export_params::ExportParameters, + exports_path, + key, + index, + data, +) + df = to_dataframe(data, key) + time_col = + range(index; length = export_params.horizon_count, step = export_params.resolution) + DataFrames.insertcols!(df, 1, :DateTime => time_col) + ISOPT.export_output(export_params.file_type, exports_path, key, index, df) + return +end + +"""Sanitize a model name for use as a filesystem path component. +Replaces path separators, null bytes, and control characters with underscores.""" +function _sanitize_model_name(name::AbstractString) + _is_path_safe(c::AbstractChar) = + isprint(c) && c ∉ ('/', '\\', ':', '*', '?', '"', '<', '>', '|') + sanitized = map(c -> _is_path_safe(c) ? c : '_', name) + if isempty(sanitized) || sanitized == "." || sanitized == ".." + throw( + IS.InvalidValue("Model name '$name' is not valid for use as a path component"), + ) + end + return sanitized +end + +# Aliases used for clarity in the method dispatches so it is possible to know if writing to +# DecisionModel data or EmulationModel data +# Note: DecisionModelIndexType and EmulationModelIndexType are defined in core/definitions.jl + +function write_outputs!( + store::AbstractModelStore, + model::OperationModel, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime; + exports = nothing, +) + if exports !== nothing + export_params = ExportParameters( + exports, + joinpath(exports.path, _sanitize_model_name(string(get_name(model)))), + get_export_file_type(exports), + get_resolution(model), + get_horizon(get_settings(model)) ÷ get_resolution(model), + ) + else + export_params = nothing + end + + write_model_dual_outputs!(store, model, index, update_timestamp, export_params) + write_model_parameter_outputs!(store, model, index, update_timestamp, export_params) + write_model_variable_outputs!(store, model, index, update_timestamp, export_params) + write_model_aux_variable_outputs!(store, model, index, update_timestamp, export_params) + write_model_expression_outputs!(store, model, index, update_timestamp, export_params) + return +end + +function write_model_dual_outputs!( + store, + model::T, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime, + export_params::Union{ExportParameters, Nothing}, +) where {T <: OperationModel} + container = get_optimization_container(model) + model_name = get_name(model) + if export_params !== nothing + exports_path = joinpath(export_params.exports_path, "duals") + mkpath(exports_path) + end + + for (key, constraint) in get_duals(container) + !should_write_resulting_value(key) && continue + data = jump_value.(constraint) + write_output!(store, model_name, key, index, update_timestamp, data) + + if export_params !== nothing && + should_export_dual(export_params.exports, update_timestamp, model_name, key) + _export_container_output!(export_params, exports_path, key, index, data) + end + end + return +end + +function write_model_parameter_outputs!( + store, + model::T, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime, + export_params::Union{ExportParameters, Nothing}, +) where {T <: OperationModel} + container = get_optimization_container(model) + model_name = get_name(model) + if export_params !== nothing + exports_path = joinpath(export_params.exports_path, "parameters") + mkpath(exports_path) + end + + parameters = get_parameters(container) + for (key, param_container) in parameters + !should_write_resulting_value(key) && continue + data = calculate_parameter_values(param_container) + write_output!(store, model_name, key, index, update_timestamp, data) + + if export_params !== nothing && + should_export_parameter( + export_params.exports, + update_timestamp, + model_name, + key, + ) + _export_container_output!(export_params, exports_path, key, index, data) + end + end + return +end + +function write_model_variable_outputs!( + store, + model::T, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime, + export_params::Union{ExportParameters, Nothing}, +) where {T <: OperationModel} + container = get_optimization_container(model) + model_name = get_name(model) + if export_params !== nothing + exports_path = joinpath(export_params.exports_path, "variables") + mkpath(exports_path) + end + + if !isempty(container.primal_values_cache) + variables = container.primal_values_cache.variables_cache + else + variables = get_variables(container) + end + + for (key, variable) in variables + !should_write_resulting_value(key) && continue + data = jump_value.(variable) + write_output!(store, model_name, key, index, update_timestamp, data) + + if export_params !== nothing && + should_export_variable( + export_params.exports, + update_timestamp, + model_name, + key, + ) + _export_container_output!(export_params, exports_path, key, index, data) + end + end + return +end + +function write_model_aux_variable_outputs!( + store, + model::T, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime, + export_params::Union{ExportParameters, Nothing}, +) where {T <: OperationModel} + container = get_optimization_container(model) + model_name = get_name(model) + if export_params !== nothing + exports_path = joinpath(export_params.exports_path, "aux_variables") + mkpath(exports_path) + end + + for (key, variable) in get_aux_variables(container) + !should_write_resulting_value(key) && continue + data = jump_value.(variable) + write_output!(store, model_name, key, index, update_timestamp, data) + + if export_params !== nothing && + should_export_aux_variable( + export_params.exports, + update_timestamp, + model_name, + key, + ) + _export_container_output!(export_params, exports_path, key, index, data) + end + end + return +end + +function write_model_expression_outputs!( + store, + model::T, + index::Union{DecisionModelIndexType, EmulationModelIndexType}, + update_timestamp::Dates.DateTime, + export_params::Union{ExportParameters, Nothing}, +) where {T <: OperationModel} + container = get_optimization_container(model) + model_name = get_name(model) + if export_params !== nothing + exports_path = joinpath(export_params.exports_path, "expressions") + mkpath(exports_path) + end + + if !isempty(container.primal_values_cache) + expressions = container.primal_values_cache.expressions_cache + else + expressions = get_expressions(container) + end + + for (key, expression) in expressions + !should_write_resulting_value(key) && continue + data = jump_value.(expression) + write_output!(store, model_name, key, index, update_timestamp, data) + + if export_params !== nothing && + should_export_expression( + export_params.exports, + update_timestamp, + model_name, + key, + ) + _export_container_output!(export_params, exports_path, key, index, data) + end + end + return +end diff --git a/src/operation/template_validation.jl b/src/operation/template_validation.jl index 00fbe63..cf35825 100644 --- a/src/operation/template_validation.jl +++ b/src/operation/template_validation.jl @@ -1,5 +1,18 @@ const _TEMPLATE_VALIDATION_EXCLUSIONS = [PSY.Arc, PSY.Area, PSY.ACBus, PSY.LoadZone] +# Default behavior: a model parameterized by a Default*Problem runs the impl. +# Custom problems error out by default — they must override validate_template. +function validate_template( + model::OperationModel{<:Union{DefaultDecisionProblem, DefaultEmulationProblem}}, +) + validate_template_impl!(model) + return +end + +function validate_template(::OperationModel{M}) where {M <: OperationProblem} + error("validate_template is not implemented for OperationModel{$M}") +end + function validate_template_impl!(model::IOM.OperationModel) template = get_template(model) settings = get_settings(model) diff --git a/src/operation/time_series_interface.jl b/src/operation/time_series_interface.jl new file mode 100644 index 0000000..fbc3b51 --- /dev/null +++ b/src/operation/time_series_interface.jl @@ -0,0 +1,93 @@ +function get_time_series_values!( + time_series_type::Type{T}, + model::DecisionModel, + component, + name::String, + initial_time::Dates.DateTime, + horizon::Int; + ignore_scaling_factors = true, + interval::Dates.Millisecond = UNSET_INTERVAL, +) where {T <: IS.Forecast} + is_interval = _to_is_interval(interval) + settings = get_settings(model) + resolution = get_resolution(settings) + if !use_time_series_cache(settings) + return IS.get_time_series_values( + T, + component, + name; + start_time = initial_time, + len = horizon, + ignore_scaling_factors = ignore_scaling_factors, + interval = is_interval, + ) + end + + cache = get_time_series_cache(model) + key = IS.TimeSeriesCacheKey(IS.get_uuid(component), T, name, resolution, is_interval) + if haskey(cache, key) + ts_cache = cache[key] + else + ts_cache = IS.make_time_series_cache( + time_series_type, + component, + name, + initial_time, + horizon; + ignore_scaling_factors = ignore_scaling_factors, + interval = is_interval, + resolution = resolution, + ) + cache[key] = ts_cache + end + + ts = IS.get_time_series_array!(ts_cache, initial_time) + return TimeSeries.values(ts) +end + +function get_time_series_values!( + ::Type{T}, + model::EmulationModel, + component::U, + name::String, + initial_time::Dates.DateTime, + len::Int = 1; + ignore_scaling_factors = true, + resolution::Dates.Millisecond = UNSET_RESOLUTION, +) where {T <: IS.StaticTimeSeries, U <: IS.InfrastructureSystemsComponent} + settings = get_settings(model) + key_resolution = + resolution == UNSET_RESOLUTION ? get_resolution(settings) : resolution + is_resolution = _to_is_resolution(key_resolution) + if !use_time_series_cache(settings) + return IS.get_time_series_values( + T, + component, + name; + start_time = initial_time, + len = len, + ignore_scaling_factors = ignore_scaling_factors, + resolution = is_resolution, + ) + end + + cache = get_time_series_cache(model) + key = IS.TimeSeriesCacheKey(IS.get_uuid(component), T, name, key_resolution, nothing) + if haskey(cache, key) + ts_cache = cache[key] + else + ts_cache = IS.make_time_series_cache( + T, + component, + name, + initial_time, + len; + ignore_scaling_factors = ignore_scaling_factors, + resolution = is_resolution, + ) + cache[key] = ts_cache + end + + ts = IS.get_time_series_array!(ts_cache, initial_time) + return TimeSeries.values(ts) +end diff --git a/src/utils/print.jl b/src/utils/print.jl index 84de17f..4e98e93 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -113,3 +113,81 @@ function _show_method( end return end +ProblemOutputsTypes = Union{OptimizationProblemOutputs, SimulationProblemOutputs} +function Base.show(io::IO, ::MIME"text/plain", input::ProblemOutputsTypes) + _show_method(io, input, :auto) +end + +function Base.show(io::IO, ::MIME"text/html", input::ProblemOutputsTypes) + # The tf_html_simple format was eliminated from PrettyTables and it was added to PowerSystems + _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) +end + +function _show_method( + io::IO, + outputs::T, + backend::Symbol; + kwargs..., +) where {T <: ProblemOutputsTypes} + timestamps = get_timestamps(outputs) + + if backend == :html + println(io, "

Start: $(first(timestamps))

") + println(io, "

End: $(last(timestamps))

") + println( + io, + "

Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))

", + ) + else + println(io, "Start: $(first(timestamps))") + println(io, "End: $(last(timestamps))") + println(io, "Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))") + end + + values = Dict{String, Vector{String}}( + "Variables" => list_variable_names(outputs), + "Auxiliary variables" => list_aux_variable_names(outputs), + "Duals" => list_dual_names(outputs), + "Expressions" => list_expression_names(outputs), + "Parameters" => list_parameter_names(outputs), + ) + + if hasfield(T, :problem) + name = outputs.problem + else + name = "InfrastructureOptimizationModels" + end + + for (k, val) in values + if !isempty(val) + println(io) + PrettyTables.pretty_table( + io, + val; + show_column_labels = false, + backend = backend, + title = "$name Problem $k Outputs", + alignment = :l, + kwargs..., + ) + end + end +end + +function Base.show(io::IO, ::MIME"text/plain", bounds::ConstraintBounds) + println(io, "ConstraintBounds:") + println(io, "Constraint Coefficient") + show(io, MIME"text/plain"(), bounds.coefficient) + println(io, "Constraint RHS") + show(io, MIME"text/plain"(), bounds.rhs) +end + +function Base.show(io::IO, ::MIME"text/plain", bounds::VariableBounds) + println(io, "VariableBounds:") + show(io, MIME"text/plain"(), bounds.bounds) +end + +function Base.show(io::IO, ::MIME"text/plain", bounds::NumericalBounds) + println(io, rpad(" Minimum", 20), "Maximum") + println(io, rpad(" $(bounds.min)", 20), "$(bounds.max)") +end diff --git a/test/Project.toml b/test/Project.toml index 9445668..91525e4 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -34,7 +34,7 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" [sources] InfrastructureSystems = {url = "https://github.com/NREL-Sienna/InfrastructureSystems.jl", rev = "IS4"} PowerSystems = {url = "https://github.com/NREL-Sienna/PowerSystems.jl", rev = "psy6"} -InfrastructureOptimizationModels = {url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl", rev = "lk/pom-test-fixes"} +InfrastructureOptimizationModels = {url = "https://github.com/NREL-Sienna/InfrastructureOptimizationModels.jl", rev = "lk/move-operation-to-pom"} PowerSystemCaseBuilder = {url = "https://github.com/NREL-Sienna/PowerSystemCaseBuilder.jl", rev = "psy6"} [compat] diff --git a/test/test_model_decision.jl b/test/test_model_decision.jl index 0e4ccd2..12340ad 100644 --- a/test/test_model_decision.jl +++ b/test/test_model_decision.jl @@ -163,7 +163,7 @@ end ), ) == "TimeDurationOff__ThermalStandard" - export_outputs(res) + POM.export_outputs(res) outputs_dir = joinpath(output_dir, "outputs") @test isfile(joinpath(outputs_dir, "optimizer_stats.csv")) variables_dir = joinpath(outputs_dir, "variables") @@ -186,7 +186,7 @@ end end @test get_constraint_index(model, length(constraint_indices) + 1) === nothing - var_keys = IOM.get_all_variable_keys(model) + var_keys = POM.get_all_variable_keys(model) var_index = get_all_variable_index(model) for (ix, (key, index, moi_index)) in enumerate(var_keys) index_tuple = var_index[ix] @@ -268,7 +268,7 @@ end container = IOM.get_optimization_container(model) constraint_key = IOM.ConstraintKey(CopperPlateBalanceConstraint, PSY.System) constraints = IOM.get_constraints(container)[constraint_key] - dual_outputs = IOM.read_duals(container)[constraint_key] + dual_outputs = POM.read_duals(container)[constraint_key] dual_outputs_read = read_dual(res, constraint_key; table_format = TableFormat.WIDE) realized_dual_outputs = read_duals(res, [constraint_key]; table_format = TableFormat.WIDE)[IOM.encode_key_as_string( @@ -296,7 +296,7 @@ end system = IOM.get_system(model) parameter_key = IOM.ParameterKey(ActivePowerTimeSeriesParameter, PSY.PowerLoad) - param_vals = IOM.read_parameters(container)[parameter_key] + param_vals = POM.read_parameters(container)[parameter_key] for load in get_components(PowerLoad, system) name = get_name(load) vals = get_time_series_values(Deterministic, load, "max_active_power") @@ -305,8 +305,8 @@ end end res = OptimizationProblemOutputs(model) - @test length(list_variable_names(res)) == 1 - @test length(list_dual_names(res)) == 1 + @test length(POM.list_variable_names(res)) == 1 + @test length(POM.list_dual_names(res)) == 1 @test get_model_base_power(res) == 100.0 @test isa(get_objective_value(res), Float64) @test isa(res.variable_values, Dict{IOM.VariableKey, DataFrames.DataFrame}) @@ -328,7 +328,7 @@ end ) @test isa(IOM.get_resolution(res), Dates.TimePeriod) @test isa(IOM.get_forecast_horizon(res), Int64) - @test isa(get_realized_timestamps(res), StepRange{DateTime}) + @test isa(POM.get_realized_timestamps(res), StepRange{DateTime}) @test isa(IOM.get_source_data(res), PSY.System) @test length(get_timestamps(res)) == 24 @@ -402,12 +402,12 @@ end # Serialize to a new directory with the exported function. outputs_path = joinpath(path, "outputs") serialize_outputs(outputs1, outputs_path) - @test isfile(joinpath(outputs_path, IOM._PROBLEM_OUTPUTS_FILENAME)) + @test isfile(joinpath(outputs_path, POM._PROBLEM_OUTPUTS_FILENAME)) outputs3 = OptimizationProblemOutputs(outputs_path) var3 = read_variable(outputs3, ActivePowerVariable, ThermalStandard) @test var1_a == var3 @test get_source_data(outputs3) === nothing - set_source_data!(outputs3, get_source_data(outputs1)) + POM.set_source_data!(outputs3, get_source_data(outputs1)) @test get_source_data(outputs3) isa PSY.System exp_file = @@ -416,7 +416,7 @@ end # Manually Multiply by the base power var1_a has natural units and export writes directly from the solver @test var1_a.value == var4.value .* 100.0 - @test length(readdir(IOM.export_realized_outputs(outputs1))) === 7 + @test length(readdir(POM.export_realized_outputs(outputs1))) === 7 end @testset "Test Numerical Stability of Constraints" begin @@ -428,10 +428,10 @@ end @test build!(model; output_dir = mktempdir(; cleanup = true)) == IOM.ModelBuildStatus.BUILT - bounds = IOM.get_constraint_numerical_bounds(model) + bounds = POM.get_constraint_numerical_bounds(model) _check_constraint_bounds(bounds, valid_bounds) - model_bounds = IOM.get_detailed_constraint_numerical_bounds(model) + model_bounds = POM.get_detailed_constraint_numerical_bounds(model) valid_model_bounds = Dict( :CopperPlateBalanceConstraint__System => ( coefficient = (min = 1.0, max = 1.0), @@ -458,10 +458,10 @@ end @test build!(model; output_dir = mktempdir(; cleanup = true)) == IOM.ModelBuildStatus.BUILT - bounds = IOM.get_variable_numerical_bounds(model) + bounds = POM.get_variable_numerical_bounds(model) _check_variable_bounds(bounds, valid_bounds) - model_bounds = IOM.get_detailed_variable_numerical_bounds(model) + model_bounds = POM.get_detailed_variable_numerical_bounds(model) valid_model_bounds = Dict( :StopVariable__ThermalStandard => (min = 0.0, max = 1.0), :StartVariable__ThermalStandard => (min = 0.0, max = 1.0), diff --git a/test/test_optimization_outputs.jl b/test/test_optimization_outputs.jl new file mode 100644 index 0000000..59e8e6e --- /dev/null +++ b/test/test_optimization_outputs.jl @@ -0,0 +1,278 @@ +import PowerOperationsModels: + OptimizationContainerMetadata, + OptimizationProblemOutputs, + VariableKey, + ExpressionKey, + read_variable, + read_expression +import Dates: + DateTime, + Millisecond +import InfrastructureSystems as IS +import InfrastructureSystems.Optimization: VariableType, ExpressionType + +# Minimal mock types — originally in IOM's test/test_utils/test_types.jl. +# Kept inline here so this test stays self-contained in POM. +struct MockVariable <: VariableType end +struct MockVariable2 <: VariableType end +struct MockExpression <: ExpressionType end +struct MockExpression2 <: ExpressionType end +# MockThermalGen is used purely as a component-type tag — TestComponent suffices. +const MockThermalGen = IS.TestComponent + +# Override ISOPT's default `convert_output_to_natural_units(::Type) = false` for +# the two "2"-suffixed mock types so the conversion logic in OPO is exercised. +POM.convert_output_to_natural_units(::Type{MockVariable2}) = true +POM.convert_output_to_natural_units(::Type{MockExpression2}) = true + +@testset "Test OptimizationProblemOutputs long format" begin + base_power = 10.0 + # 2 hours timestamp range + timestamp_range = + StepRange( + DateTime("2024-01-01T00:00:00"), + Millisecond(3600000), + DateTime("2024-01-01T03:00:00"), + ) + timestamp_vec = collect(timestamp_range) + data = IS.SystemData() + uuid = IS.make_uuid() + aux_variable_values = Dict() + @test !POM.convert_output_to_natural_units(MockVariable) + @test POM.convert_output_to_natural_units(MockVariable2) + var_key1 = VariableKey(MockVariable, IS.TestComponent) + var_key2 = VariableKey(MockVariable2, IS.TestComponent) + vals = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + variable_values = Dict( + var_key1 => DataFrame( + "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], + "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], + "value" => vals, + ), + var_key2 => DataFrame( + "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], + "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], + "value" => vals, + ), + ) + dual_values = Dict() + parameter_values = Dict() + @test !POM.convert_output_to_natural_units(MockExpression) + @test POM.convert_output_to_natural_units(MockExpression2) + exp_key1 = ExpressionKey(MockExpression, IS.TestComponent) + exp_key2 = ExpressionKey(MockExpression2, MockThermalGen) + # Expression only 1 time-step + expression_values = Dict( + exp_key1 => DataFrame( + "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], + "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], + "value" => vals, + ), + exp_key2 => DataFrame( + "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], + "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], + "value" => vals, + ), + ) + optimizer_stats = DataFrames.DataFrame() + metadata = OptimizationContainerMetadata() + # Test with StepRange + opt_res1 = OptimizationProblemOutputs( + base_power, + timestamp_range, + data, + uuid, + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + metadata, + "test_model", + mktempdir(), + mktempdir(), + ) + # Test with Vector{DateTime} + opt_res2 = OptimizationProblemOutputs( + base_power, + timestamp_vec, + data, + uuid, + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + metadata, + "test_model", + mktempdir(), + mktempdir(), + ) + opt_res3 = OptimizationProblemOutputs( + base_power, + [timestamp_vec[1]], + data, + uuid, + aux_variable_values, + variable_values, + dual_values, + parameter_values, + expression_values, + optimizer_stats, + metadata, + "test_model", + mktempdir(), + mktempdir(), + ) + + var_res = read_variable(opt_res1, var_key1) + @test sort!(unique(var_res.DateTime)) == timestamp_vec + @test @rsubset(var_res, :name == "c1")[!, :value] == [1.0, 2.0, 3.0, 4.0] + @test @rsubset(var_res, :name == "c2")[!, :value] == [5.0, 6.0, 7.0, 8.0] + + var_res = read_variable(opt_res1, var_key2) + @test @rsubset(var_res, :name == "c1")[!, :value] == [10.0, 20.0, 30.0, 40.0] + @test @rsubset(var_res, :name == "c2")[!, :value] == [50.0, 60.0, 70.0, 80.0] + + var_res2 = read_variable( + opt_res1, + var_key1; + start_time = DateTime("2024-01-01T01:00:00"), + len = 2, + ) + @test @rsubset(var_res2, :name == "c1")[!, :value] == [2.0, 3.0] + @test @rsubset(var_res2, :name == "c2")[!, :value] == [6.0, 7.0] + + var_res2 = read_variable( + opt_res1, + var_key2; + start_time = DateTime("2024-01-01T01:00:00"), + len = 2, + ) + @test @rsubset(var_res2, :name == "c1")[!, :value] == [20.0, 30.0] + @test @rsubset(var_res2, :name == "c2")[!, :value] == [60.0, 70.0] + + var_res = read_variable(opt_res1, var_key2; table_format = IS.TableFormat.WIDE) + @test var_res[!, :c1] == [10.0, 20.0, 30.0, 40.0] + @test var_res[!, :c2] == [50.0, 60.0, 70.0, 80.0] + + exp_res = read_expression(opt_res2, exp_key1) + @test @rsubset(exp_res, :name == "c1")[!, :value] == [1.0, 2.0, 3.0, 4.0] + @test @rsubset(exp_res, :name == "c2")[!, :value] == [5.0, 6.0, 7.0, 8.0] + exp_res = read_expression(opt_res2, exp_key2) + @test @rsubset(exp_res, :name == "c1")[!, :value] == [10.0, 20.0, 30.0, 40.0] + @test @rsubset(exp_res, :name == "c2")[!, :value] == [50.0, 60.0, 70.0, 80.0] + + @test POM.get_resolution(opt_res1) == Millisecond(3600000) + @test POM.get_resolution(opt_res2) == Millisecond(3600000) + @test isnothing(POM.get_resolution(opt_res3)) +end + +@testset "Test OptimizationProblemOutputs 3d long format" begin + timestamps = StepRange( + DateTime("2024-01-01T00:00:00"), + Millisecond(3600000), + DateTime("2024-01-01T01:00:00"), + ) + data = IS.SystemData() + aux_variable_values = Dict() + var_key = VariableKey(MockVariable, IS.TestComponent) + vals = [1.0, 2.0, 3.0, 4.0] + variable_values = Dict( + var_key => DataFrame( + "time_index" => [1, 2, 1, 2], + "name" => ["c1", "c2", "c1", "c2"], + "name2" => ["c3", "c4", "c3", "c4"], + "value" => vals, + ), + ) + optimizer_stats = DataFrames.DataFrame() + res = OptimizationProblemOutputs( + 100.0, + timestamps, + data, + IS.make_uuid(), + Dict(), + variable_values, + Dict(), + Dict(), + Dict(), + optimizer_stats, + OptimizationContainerMetadata(), + "test_model", + mktempdir(), + mktempdir(), + ) + + var_res = read_variable(res, var_key) + @test @rsubset(var_res, :name == "c1" && :name2 == "c3")[!, :value] == [1.0, 3.0] + @test @rsubset(var_res, :name == "c2" && :name2 == "c4")[!, :value] == [2.0, 4.0] +end + +@testset "Test OptimizationProblemOutputs _process_timestamps" begin + time_ids = [1, 2, 3, 4] + timestamps = [ + DateTime("2024-01-01T00:00:00"), + DateTime("2024-01-01T01:00:00"), + DateTime("2024-01-01T02:00:00"), + DateTime("2024-01-01T03:00:00"), + ] + data = IS.SystemData() + var_key = VariableKey(MockVariable, IS.TestComponent) + vals = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] + variable_values = Dict( + var_key => DataFrame( + "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], + "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], + "value" => vals, + ), + ) + optimizer_stats = DataFrames.DataFrame() + metadata = OptimizationContainerMetadata() + opt_res = OptimizationProblemOutputs( + 100.0, + timestamps, + data, + IS.make_uuid(), + Dict(), + variable_values, + Dict(), + Dict(), + Dict(), + optimizer_stats, + metadata, + "DecisionModel", + mktempdir(), + mktempdir(), + ) + @test POM._process_timestamps(opt_res, nothing, nothing) == + (time_ids, timestamps) + @test POM._process_timestamps(opt_res, timestamps[2], nothing) == + (time_ids[2:end], timestamps[2:end]) + @test POM._process_timestamps(opt_res, timestamps[4], nothing) == + ([time_ids[4]], [timestamps[4]]) + @test POM._process_timestamps(opt_res, nothing, 3) == + (time_ids[1:3], timestamps[1:3]) + @test POM._process_timestamps(opt_res, timestamps[2], 2) == + (time_ids[2:3], timestamps[2:3]) + + @test_throws IS.InvalidValue POM._process_timestamps( + opt_res, + timestamps[1] - Hour(1), + nothing, + ) + @test_throws IS.InvalidValue POM._process_timestamps( + opt_res, + timestamps[4] + Hour(1), + nothing, + ) + @test_throws IS.InvalidValue POM._process_timestamps(opt_res, nothing, -1) + @test_throws IS.InvalidValue POM._process_timestamps(opt_res, nothing, 10) + @test_throws IS.InvalidValue POM._process_timestamps( + opt_res, + timestamps[2], + 10, + ) +end diff --git a/test/test_utils/mock_operation_models.jl b/test/test_utils/mock_operation_models.jl index d14347b..15eda20 100644 --- a/test/test_utils/mock_operation_models.jl +++ b/test/test_utils/mock_operation_models.jl @@ -1,12 +1,12 @@ # NOTE: None of the models and function in this file are functional. All of these are used for testing purposes and do not represent valid examples either to develop custom # models. Please refer to the documentation. -struct MockOperationProblem <: IOM.DefaultDecisionProblem end -struct MockEmulationProblem <: IOM.DefaultEmulationProblem end -struct EconomicDispatchProblem <: IOM.DefaultDecisionProblem end -struct UnitCommitmentProblem <: IOM.DefaultDecisionProblem end +struct MockOperationProblem <: POM.DefaultDecisionProblem end +struct MockEmulationProblem <: POM.DefaultEmulationProblem end +struct EconomicDispatchProblem <: POM.DefaultDecisionProblem end +struct UnitCommitmentProblem <: POM.DefaultDecisionProblem end -function IOM.DecisionModel( +function POM.DecisionModel( ::Type{MockOperationProblem}, ::Type{T}, sys::PSY.System; @@ -57,7 +57,7 @@ function make_mock_singletimeseries(horizon, resolution) return SingleTimeSeries(; name = "mock_timeseries", data = timeseries_data) end -function IOM.DecisionModel(::Type{MockOperationProblem}; name = nothing, kwargs...) +function POM.DecisionModel(::Type{MockOperationProblem}; name = nothing, kwargs...) sys = System(100.0) add_component!(sys, ACBus(nothing)) l = PowerLoad(nothing) @@ -85,7 +85,7 @@ function IOM.DecisionModel(::Type{MockOperationProblem}; name = nothing, kwargs. ) end -function IOM.EmulationModel(::Type{MockEmulationProblem}; name = nothing, kwargs...) +function POM.EmulationModel(::Type{MockEmulationProblem}; name = nothing, kwargs...) sys = System(100.0) add_component!(sys, ACBus(nothing)) l = PowerLoad(nothing) @@ -114,7 +114,7 @@ end # Only used for testing function mock_construct_device!( - problem::IOM.DecisionModel{MockOperationProblem}, + problem::POM.DecisionModel{MockOperationProblem}, model; built_for_recurrent_solves = false, add_event_model = false, @@ -127,7 +127,7 @@ function mock_construct_device!( set_device_model!(problem.template, model) template = IOM.get_template(problem) IOM.finalize_template!(template, IOM.get_system(problem)) - IOM.validate_time_series!(problem) + POM.validate_time_series!(problem) IOM.init_optimization_container!( IOM.get_optimization_container(problem), IOM.get_network_model(template), @@ -171,7 +171,7 @@ function mock_construct_device!( return end -function mock_construct_network!(problem::IOM.DecisionModel{MockOperationProblem}, model) +function mock_construct_network!(problem::POM.DecisionModel{MockOperationProblem}, model) IOM.set_network_model!(problem.template, model) IOM.construct_network!( IOM.get_optimization_container(problem), @@ -229,7 +229,7 @@ function setup_ic_model_container!(model::DecisionModel) IOM.get_system(model), ) - IOM.init_model_store_params!(model) + POM.init_model_store_params!(model) @info "Make Initial Conditions Model" IOM.set_output_dir!(model, mktempdir(; cleanup = true)) diff --git a/test/test_utils/model_checks.jl b/test/test_utils/model_checks.jl index 64174a7..b877029 100644 --- a/test/test_utils/model_checks.jl +++ b/test/test_utils/model_checks.jl @@ -320,14 +320,14 @@ function IOM.jump_value(int::Int) return int end -function _check_constraint_bounds(bounds::IOM.ConstraintBounds, valid_bounds::NamedTuple) +function _check_constraint_bounds(bounds::POM.ConstraintBounds, valid_bounds::NamedTuple) @test bounds.coefficient.min == valid_bounds.coefficient.min @test bounds.coefficient.max == valid_bounds.coefficient.max @test bounds.rhs.min == valid_bounds.rhs.min @test bounds.rhs.max == valid_bounds.rhs.max end -function _check_variable_bounds(bounds::IOM.VariableBounds, valid_bounds::NamedTuple) +function _check_variable_bounds(bounds::POM.VariableBounds, valid_bounds::NamedTuple) @test bounds.bounds.min == valid_bounds.min @test bounds.bounds.max == valid_bounds.max end From dde965a85dfecb53b1faa68e138eece611946bab Mon Sep 17 00:00:00 2001 From: Luke Kiernan Date: Wed, 13 May 2026 13:32:59 -0600 Subject: [PATCH 2/3] Address CoPilot review of PR #121 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All 10 CoPilot comments — real bugs in code carried over from IOM: fix undefined `template` reference in EmulationModel constructor (mirror DecisionModel's pattern), add missing `throw()` around an IS.ArgumentError, remove dead empty!/isempty branches that referenced fields DatasetContainer doesn't have, import the missing STORE_CONTAINER_TYPES, correct the undefined `sys_uuid`/`res.source_uuid` in set_source_data!'s error message, swap JSON for JSON3, fix the bare tf_html_simple and stray ISOPT.get_resolution in print.jl, and patch docstring typos. New regression tests caught two more: show(::OptimizationProblemOutputs) referenced an undefined `T` and a missing `outputs.problem` field, and export_optimizer_stats JSON path passed a DataFrame into to_dict which has no method. Also fix the `bonuds` typo in 5 update_numerical_bounds methods and FIXME the dead PSI-leftover should_export_* call paths. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/PowerOperationsModels.jl | 4 +- src/operation/decision_model.jl | 6 +- src/operation/emulation_model.jl | 8 ++- src/operation/emulation_model_store.jl | 39 ++---------- .../model_numerical_analysis_utils.jl | 22 +++---- src/operation/optimization_problem_outputs.jl | 11 ++-- src/operation/store_common.jl | 5 ++ src/utils/print.jl | 21 +++---- test/test_default_problem_constructors.jl | 9 +++ test/test_optimization_outputs.jl | 59 +++++++++++++++++++ test/test_show.jl | 29 +++++++++ 11 files changed, 142 insertions(+), 71 deletions(-) create mode 100644 test/test_default_problem_constructors.jl create mode 100644 test/test_show.jl diff --git a/src/PowerOperationsModels.jl b/src/PowerOperationsModels.jl index 34457ba..84782a4 100644 --- a/src/PowerOperationsModels.jl +++ b/src/PowerOperationsModels.jl @@ -262,12 +262,10 @@ import InfrastructureOptimizationModels: LOG_GROUP_MODEL_STORE, ModelInternal, ModelStoreParams, - Simulation, - SimulationProblemOutputs, - SimulationOutputs, SimulationSequence, SimulationModels, STORE_CONTAINERS, + STORE_CONTAINER_TYPES, auto_transform_time_series!, calculate_parameter_values, cost_function_unsynch, diff --git a/src/operation/decision_model.jl b/src/operation/decision_model.jl index edc3d72..fb0314a 100644 --- a/src/operation/decision_model.jl +++ b/src/operation/decision_model.jl @@ -210,8 +210,10 @@ function DecisionModel{M}( jump_model::Union{Nothing, JuMP.Model} = nothing; kwargs..., ) where {M <: DefaultDecisionProblem} - IS.ArgumentError( - "DefaultDecisionProblem subtypes require a template. Use DecisionModel subtyping instead.", + throw( + IS.ArgumentError( + "DefaultDecisionProblem subtypes require a template. Use DecisionModel subtyping instead.", + ), ) end diff --git a/src/operation/emulation_model.jl b/src/operation/emulation_model.jl index 1f85184..fa51156 100644 --- a/src/operation/emulation_model.jl +++ b/src/operation/emulation_model.jl @@ -192,8 +192,12 @@ function EmulationModel{M}( sys::IS.InfrastructureSystemsContainer, jump_model::Union{Nothing, JuMP.Model} = nothing; kwargs..., -) where {M <: EmulationProblem} - return EmulationModel{M}(template, sys, jump_model; kwargs...) +) where {M <: DefaultEmulationProblem} + throw( + IS.ArgumentError( + "DefaultEmulationProblem subtypes require a template. Use EmulationModel subtyping instead.", + ), + ) end # get_problem_type lifted to OperationModel{T} in IOM/operation_model_abstract_types.jl diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl index 8dd0bd4..5e7a1fc 100644 --- a/src/operation/emulation_model_store.jl +++ b/src/operation/emulation_model_store.jl @@ -29,47 +29,16 @@ end Empty the [`EmulationModelStore`](@ref) """ function Base.empty!(store::EmulationModelStore) - stype = DatasetContainer - for (name, _) in zip(fieldnames(stype), fieldtypes(stype)) - if name ∉ [:values, :timestamps] - val = get_data_field(store, name) - try - empty!(val) - catch - @error "Base.empty! must be customized for type $stype or skipped" - rethrow() - end - elseif name == :update_timestamp - store.update_timestamp = UNSET_INI_TIME - else - setfield!( - store.data_container, - name, - zero(fieldtype(store.data_container, name)), - ) - end + for name in fieldnames(DatasetContainer) + empty!(get_data_field(store, name)) end empty!(store.optimizer_stats) return end function Base.isempty(store::EmulationModelStore) - stype = DatasetContainer - for (name, type) in zip(fieldnames(stype), fieldtypes(stype)) - if name ∉ [:values, :timestamps] - val = get_data_field(store, name) - try - !isempty(val) && return false - catch - @error "Base.isempty must be customized for type $stype or skipped" - rethrow() - end - elseif name == :update_timestamp - store.update_timestamp != UNSET_INI_TIME && return false - else - val = get_data_field(store, name) - iszero(val) && return false - end + for name in fieldnames(DatasetContainer) + !isempty(get_data_field(store, name)) && return false end return isempty(store.optimizer_stats) end diff --git a/src/operation/model_numerical_analysis_utils.jl b/src/operation/model_numerical_analysis_utils.jl index b27324a..d8712b5 100644 --- a/src/operation/model_numerical_analysis_utils.jl +++ b/src/operation/model_numerical_analysis_utils.jl @@ -74,28 +74,28 @@ function update_numerical_bounds(v::NumericalBounds, value::Real, idx) return end -function update_numerical_bounds(bonuds::NumericalBounds, func::JuMP.GenericAffExpr, idx) +function update_numerical_bounds(bounds::NumericalBounds, func::JuMP.GenericAffExpr, idx) for coefficient in values(func.terms) - update_numerical_bounds(bonuds, coefficient, idx) + update_numerical_bounds(bounds, coefficient, idx) end return end -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.LessThan, idx) - return update_numerical_bounds(bonuds, func.upper, idx) +function update_numerical_bounds(bounds::NumericalBounds, func::MOI.LessThan, idx) + return update_numerical_bounds(bounds, func.upper, idx) end -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.GreaterThan, idx) - return update_numerical_bounds(bonuds, func.lower, idx) +function update_numerical_bounds(bounds::NumericalBounds, func::MOI.GreaterThan, idx) + return update_numerical_bounds(bounds, func.lower, idx) end -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.EqualTo, idx) - return update_numerical_bounds(bonuds, func.value, idx) +function update_numerical_bounds(bounds::NumericalBounds, func::MOI.EqualTo, idx) + return update_numerical_bounds(bounds, func.value, idx) end -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.Interval, idx) - update_numerical_bounds(bonuds, func.upper, idx) - return update_numerical_bounds(bonuds, func.lower, idx) +function update_numerical_bounds(bounds::NumericalBounds, func::MOI.Interval, idx) + update_numerical_bounds(bounds, func.upper, idx) + return update_numerical_bounds(bounds, func.lower, idx) end # Default fallbacks for unsupported constraints. diff --git a/src/operation/optimization_problem_outputs.jl b/src/operation/optimization_problem_outputs.jl index 2db94f1..8ad1c38 100644 --- a/src/operation/optimization_problem_outputs.jl +++ b/src/operation/optimization_problem_outputs.jl @@ -340,7 +340,7 @@ function set_source_data!( if source_uuid != res.source_data_uuid throw( InvalidValue( - "System mismatch. $sys_uuid does not match the stored value of $(res.source_uuid)", + "System mismatch. $source_uuid does not match the stored value of $(res.source_data_uuid)", ), ) end @@ -1050,9 +1050,9 @@ Save the optimizer statistics to CSV or JSON # Arguments - - `res::Union{OptimizationProblemOutputs, SimulationProblmeOutputs`: Outputs - - `directory::AbstractString` : target directory - - `format = "CSV"` : can be "csv" or "json + - `res::OptimizationProblemOutputs`: Outputs + - `directory::AbstractString`: target directory + - `format = "CSV"`: can be `"csv"` or `"json"` """ function export_optimizer_stats( res::Outputs, @@ -1064,7 +1064,8 @@ function export_optimizer_stats( if uppercase(format) == "CSV" CSV.write(joinpath(directory, "optimizer_stats.csv"), data) elseif uppercase(format) == "JSON" - JSON.write(joinpath(directory, "optimizer_stats.json"), JSON.json(to_dict(data))) + cols = Dict(string(n) => data[!, n] for n in names(data)) + write(joinpath(directory, "optimizer_stats.json"), JSON3.write(cols)) else throw(error("writing optimizer stats only supports csv or json formats")) end diff --git a/src/operation/store_common.jl b/src/operation/store_common.jl index eb09754..8f97f34 100644 --- a/src/operation/store_common.jl +++ b/src/operation/store_common.jl @@ -86,6 +86,7 @@ function write_model_dual_outputs!( data = jump_value.(constraint) write_output!(store, model_name, key, index, update_timestamp, data) + # FIXME undefined function. leftover from PSI if export_params !== nothing && should_export_dual(export_params.exports, update_timestamp, model_name, key) _export_container_output!(export_params, exports_path, key, index, data) @@ -114,6 +115,7 @@ function write_model_parameter_outputs!( data = calculate_parameter_values(param_container) write_output!(store, model_name, key, index, update_timestamp, data) + # FIXME undefined function. leftover from PSI if export_params !== nothing && should_export_parameter( export_params.exports, @@ -152,6 +154,7 @@ function write_model_variable_outputs!( data = jump_value.(variable) write_output!(store, model_name, key, index, update_timestamp, data) + # FIXME undefined function. leftover from PSI if export_params !== nothing && should_export_variable( export_params.exports, @@ -184,6 +187,7 @@ function write_model_aux_variable_outputs!( data = jump_value.(variable) write_output!(store, model_name, key, index, update_timestamp, data) + # FIXME undefined function. leftover from PSI if export_params !== nothing && should_export_aux_variable( export_params.exports, @@ -222,6 +226,7 @@ function write_model_expression_outputs!( data = jump_value.(expression) write_output!(store, model_name, key, index, update_timestamp, data) + # FIXME undefined function. leftover from PSI if export_params !== nothing && should_export_expression( export_params.exports, diff --git a/src/utils/print.jl b/src/utils/print.jl index 4e98e93..b7e8093 100644 --- a/src/utils/print.jl +++ b/src/utils/print.jl @@ -113,22 +113,21 @@ function _show_method( end return end -ProblemOutputsTypes = Union{OptimizationProblemOutputs, SimulationProblemOutputs} -function Base.show(io::IO, ::MIME"text/plain", input::ProblemOutputsTypes) +function Base.show(io::IO, ::MIME"text/plain", input::OptimizationProblemOutputs) _show_method(io, input, :auto) end -function Base.show(io::IO, ::MIME"text/html", input::ProblemOutputsTypes) +function Base.show(io::IO, ::MIME"text/html", input::OptimizationProblemOutputs) # The tf_html_simple format was eliminated from PrettyTables and it was added to PowerSystems - _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) + _show_method(io, input, :html; stand_alone = false, table_format = PSY.tf_html_simple) end function _show_method( io::IO, - outputs::T, + outputs::OptimizationProblemOutputs, backend::Symbol; kwargs..., -) where {T <: ProblemOutputsTypes} +) timestamps = get_timestamps(outputs) if backend == :html @@ -136,12 +135,12 @@ function _show_method( println(io, "

End: $(last(timestamps))

") println( io, - "

Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))

", + "

Resolution: $(Dates.Minute(get_resolution(outputs)))

", ) else println(io, "Start: $(first(timestamps))") println(io, "End: $(last(timestamps))") - println(io, "Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))") + println(io, "Resolution: $(Dates.Minute(get_resolution(outputs)))") end values = Dict{String, Vector{String}}( @@ -152,11 +151,7 @@ function _show_method( "Parameters" => list_parameter_names(outputs), ) - if hasfield(T, :problem) - name = outputs.problem - else - name = "InfrastructureOptimizationModels" - end + name = outputs.model_type for (k, val) in values if !isempty(val) diff --git a/test/test_default_problem_constructors.jl b/test/test_default_problem_constructors.jl new file mode 100644 index 0000000..1a112c5 --- /dev/null +++ b/test/test_default_problem_constructors.jl @@ -0,0 +1,9 @@ +@testset "DecisionModel{GenericOpProblem}(sys) throws ArgumentError" begin + sys = System(100.0) + @test_throws IS.ArgumentError DecisionModel{POM.GenericOpProblem}(sys) +end + +@testset "EmulationModel{GenericEmulationProblem}(sys) throws ArgumentError" begin + sys = System(100.0) + @test_throws IS.ArgumentError EmulationModel{POM.GenericEmulationProblem}(sys) +end diff --git a/test/test_optimization_outputs.jl b/test/test_optimization_outputs.jl index 59e8e6e..22ddc36 100644 --- a/test/test_optimization_outputs.jl +++ b/test/test_optimization_outputs.jl @@ -276,3 +276,62 @@ end 10, ) end + +# Smoke fixture for the regression tests below — minimal OPO with one variable row. +function _make_minimal_opo() + base_power = 100.0 + ts = collect( + StepRange( + DateTime("2024-01-01T00:00:00"), + Millisecond(3600000), + DateTime("2024-01-01T01:00:00"), + ), + ) + var_key = VariableKey(MockVariable, IS.TestComponent) + variable_values = Dict( + var_key => DataFrame( + "time_index" => [1, 2], + "name" => ["c1", "c1"], + "value" => [1.0, 2.0], + ), + ) + optimizer_stats = DataFrame(; objective_value = [1.0]) + return OptimizationProblemOutputs( + base_power, + ts, + IS.SystemData(), + IS.make_uuid(), + Dict(), + variable_values, + Dict(), + Dict(), + Dict(), + optimizer_stats, + OptimizationContainerMetadata(), + "DecisionModel", + mktempdir(), + mktempdir(), + ) +end + +@testset "export_optimizer_stats JSON path" begin + res = _make_minimal_opo() + dir = mktempdir() + POM.export_optimizer_stats(res, dir; format = "json") + @test isfile(joinpath(dir, "optimizer_stats.json")) +end + +@testset "set_source_data! throws InvalidValue on UUID mismatch" begin + res = _make_minimal_opo() + wrong_source = IS.TestComponent("wrong", 0) + @test IS.get_uuid(wrong_source) != res.source_data_uuid + @test_throws IS.InvalidValue POM.set_source_data!(res, wrong_source) +end + +@testset "show(OptimizationProblemOutputs) does not throw" begin + res = _make_minimal_opo() + plain = sprint(show, MIME"text/plain"(), res) + @test occursin("Resolution", plain) + html = sprint(show, MIME"text/html"(), res) + @test occursin("Resolution", html) +end diff --git a/test/test_show.jl b/test/test_show.jl new file mode 100644 index 0000000..6df5978 --- /dev/null +++ b/test/test_show.jl @@ -0,0 +1,29 @@ +@testset "show(SimulationModels) decision-only does not throw" begin + sim_models = IOM.SimulationModels(; + decision_models = [ + DecisionModel(MockOperationProblem; horizon = Hour(24), name = "UC"), + DecisionModel( + MockOperationProblem; + horizon = Hour(12), + resolution = Minute(5), + name = "ED", + ), + ], + ) + out = sprint(show, MIME"text/plain"(), sim_models) + @test occursin("UC", out) + @test occursin("ED", out) + @test occursin("Decision Models", out) +end + +@testset "show(SimulationModels) with emulator row does not throw" begin + sim_models = IOM.SimulationModels(; + decision_models = [ + DecisionModel(MockOperationProblem; horizon = Hour(24), name = "UC"), + ], + emulation_model = EmulationModel(MockEmulationProblem; name = "AGC"), + ) + out = sprint(show, MIME"text/plain"(), sim_models) + @test occursin("UC", out) + @test occursin("AGC", out) +end From f6eb4bdddc746607a4956632bec79066debed7bb Mon Sep 17 00:00:00 2001 From: Luke Kiernan Date: Wed, 13 May 2026 19:46:19 -0600 Subject: [PATCH 3/3] bump PNM compat --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 672e64c..9dc394d 100644 --- a/Project.toml +++ b/Project.toml @@ -52,7 +52,7 @@ InfrastructureSystems = "3" InteractiveUtils = "1.11.0" JuMP = "^1.28" MathOptInterface = "1.51.0" -PowerNetworkMatrices = "^0.19" +PowerNetworkMatrices = "^0.20" PowerSystems = "5.3" PrettyTables = "3.3.2" ProgressMeter = "1.11.0"