diff --git a/Project.toml b/Project.toml index 4419271f..936f17f5 100644 --- a/Project.toml +++ b/Project.toml @@ -39,7 +39,7 @@ JuMP = "^1.28" LinearAlgebra = "1" Logging = "1" MathOptInterface = "1" -PowerNetworkMatrices = "^0.19" +PowerNetworkMatrices = "^0.20" PrettyTables = "3.1" Random = "^1.10" Serialization = "1" diff --git a/src/InfrastructureOptimizationModels.jl b/src/InfrastructureOptimizationModels.jl index 1fa5f860..5d3ab929 100644 --- a/src/InfrastructureOptimizationModels.jl +++ b/src/InfrastructureOptimizationModels.jl @@ -159,9 +159,7 @@ using DocStringExtensions ################################################################################# # Exports -# Base Models -export DecisionModel -export EmulationModel +# Base Models — concrete DecisionModel/EmulationModel now defined and exported by POM. export AbstractProblemTemplate export ServicesModelContainer, DevicesModelContainer, BranchModelContainer export InitialCondition @@ -197,11 +195,9 @@ export EventParametersAttributes export ValidDataParamEltypes # Functions -export validate_time_series! export init_optimization_container! ## Op Model Exports export get_initial_conditions -export serialize_outputs export serialize_optimization_model export get_device_models @@ -231,51 +227,17 @@ export add_pwl_normalization_constraint! export add_pwl_sos2_constraint! export get_pwl_cost_expression_delta -## Outputs interfaces -export get_variable_values -export get_dual_values +## Outputs interfaces — OptimizationProblemOutputs and its readers now live in POM. export get_parameter_values -export get_aux_variable_values export get_expression_values export get_timestamps export get_system -export list_variable_keys -export list_dual_keys -export list_parameter_keys -export list_aux_variable_keys -export list_expression_keys -export list_variable_names -export list_dual_names -export list_parameter_names -export list_aux_variable_names -export list_expression_names -export read_variable -export read_dual -export read_parameter -export read_aux_variable -export read_expression -export read_variables -export read_duals -export read_parameters -export read_aux_variables -export read_expressions -export get_realized_timestamps export get_problem_base_power -export get_objective_value -export read_optimizer_stats ## Utils Exports -export OptimizationProblemOutputs -export OptimizationProblemOutputsExport export OptimizerStats -export get_all_constraint_index -export get_all_variable_index -export get_constraint_index -export get_variable_index export list_recorder_events export jump_value -export ConstraintBounds -export VariableBounds # Internal accessors needed by downstream packages export get_network_model @@ -297,9 +259,6 @@ export get_service_name export get_default_time_series_type export add_expression_container! -# Initial condition infrastructure (extension points for POM) -export update_initial_conditions! - # Key Types (defined in IOM) export OptimizationContainerKey export VariableKey @@ -370,7 +329,6 @@ export add_service_variables!, requires_initialization # End bulk-added # more extension points -export write_outputs! export built_for_recurrent_solves export get_incompatible_devices @@ -378,7 +336,7 @@ export get_incompatible_devices # Core types export OptimizationContainer, OperationModel, AbstractPowerFlowEvaluationModel export ArgumentConstructStage, ModelConstructStage -export EmulationModelStore, DeviceModelForBranches +export DeviceModelForBranches export SOSStatusVariable # Parameter types export FuelCostParameter, VariableValueParameter, FixValueParameter @@ -476,11 +434,12 @@ export ModelBuildStatus export RunStatus export SimulationBuildStatus -# Problem Types +# Problem Types — abstract types stay in IOM; default concrete subtypes +# (DefaultDecisionProblem, DefaultEmulationProblem, GenericOpProblem, +# GenericEmulationProblem) are exported by POM. +export OperationProblem export DecisionProblem export EmulationProblem -export DefaultDecisionProblem -export DefaultEmulationProblem # Settings and Data Types export Settings @@ -517,9 +476,7 @@ export get_contributing_devices export get_contributing_devices_map export get_parameter_column_values export update_container_parameter_values! -export export_outputs export get_source_data -export set_source_data! ## Note: Concrete PowerModels types (ACPPowerModel, DCPPowerModel, etc.) are now ## defined and exported by PowerOperationsModels, not IOM. @@ -538,8 +495,6 @@ include("core/parameter_container.jl") # Parameter container infr include("core/abstract_model_store.jl") # Store depends on keys include("core/optimizer_stats.jl") # Stats standalone include("core/optimization_container_metadata.jl") # Metadata depends on keys -include("core/optimization_problem_outputs_export.jl") # Export config -include("core/optimization_problem_outputs.jl") # Outputs depends on all above include("core/model_internal.jl") # Internal state (needs ModelBuildStatus) include("core/time_series_parameter_types.jl") @@ -625,16 +580,6 @@ include("bilinear_approximations/nmdt.jl") include("common_models/add_param_container.jl") include("operation/operation_model_interface.jl") -include("operation/decision_model_store.jl") -include("operation/emulation_model_store.jl") -include("operation/store_common.jl") -include("operation/initial_conditions_update_in_memory_store.jl") -include("operation/decision_model.jl") -include("operation/emulation_model.jl") -include("operation/problem_outputs.jl") -include("operation/time_series_interface.jl") -include("operation/optimization_debugging.jl") -include("operation/model_numerical_analysis_utils.jl") include("initial_conditions/calculate_initial_condition.jl") diff --git a/src/core/operation_model_abstract_types.jl b/src/core/operation_model_abstract_types.jl index fd98bbec..7ece6eb9 100644 --- a/src/core/operation_model_abstract_types.jl +++ b/src/core/operation_model_abstract_types.jl @@ -1,7 +1,14 @@ """ -Abstract type for Decision Model and Emulation Model. OperationModel structs are parameterized with DecisionProblem or Emulation Problem structs +Common supertype for problems that an `OperationModel` can solve. +Both `DecisionProblem` and `EmulationProblem` are subtypes. """ -abstract type OperationModel end +abstract type OperationProblem end + +""" +Abstract type for Decision Model and Emulation Model. `OperationModel` +subtypes are parameterized by the problem they solve (`<: OperationProblem`). +""" +abstract type OperationModel{T <: OperationProblem} end #TODO: Document the required interfaces for custom types """ @@ -13,7 +20,7 @@ import InfrastructureOptimizationModels const POM = InfrastructureOptimizationModels struct MyCustomProblem <: POM.DecisionProblem """ -abstract type DecisionProblem end +abstract type DecisionProblem <: OperationProblem end """ Abstract type for Emulation Problems @@ -24,7 +31,12 @@ import InfrastructureOptimizationModels const POM = InfrastructureOptimizationModels struct MyCustomEmulator <: POM.EmulationProblem """ -abstract type EmulationProblem end +abstract type EmulationProblem <: OperationProblem end + +""" +Return the concrete `OperationProblem` subtype that parameterizes a model. +""" +get_problem_type(::OperationModel{M}) where {M <: OperationProblem} = M ################################################################################# # Simulation Models Container @@ -55,21 +67,3 @@ end # Placeholder accessor function for simulation sequence get_step_resolution(::SimulationSequence) = Dates.Hour(1) - -################################################################################# -# Simulation Type -# Abstract type for simulation objects -# Concrete implementation should be in PowerSimulations -abstract type Simulation end - -################################################################################# -# Simulation Outputs Type -# Abstract type for simulation outputs -# Concrete implementation should be in PowerSimulations -abstract type SimulationOutputs end - -################################################################################# -# Simulation Problem Outputs Type -# Abstract type for individual problem outputs within a simulation -# Concrete implementation should be in PowerSimulations -abstract type SimulationProblemOutputs end diff --git a/src/core/optimization_problem_outputs.jl b/src/core/optimization_problem_outputs.jl deleted file mode 100644 index 2db94f1d..00000000 --- a/src/core/optimization_problem_outputs.jl +++ /dev/null @@ -1,1116 +0,0 @@ -""" - mutable struct OptimizationProblemOutputs <: Outputs - -Container for the outputs of an optimization problem, including variable values, dual values, -parameter values, expression values, and optimizer statistics. - -This type stores all output data from solving an optimization problem and provides methods -to read, export, and serialize the outputs. Instead of accessing the output dictionary -fields directly, use the `read_foo` functions. - -# Fields -- `base_power::Float64`: Base power used for per-unit conversion -- `timestamps::Vector{Dates.DateTime}`: Time stamps for each step in the outputs -- `source_data::Union{Nothing, InfrastructureSystemsType}`: Reference to the source data (e.g., system) -- `source_data_uuid::Base.UUID`: UUID of the source data for validation. Internal usage. -- `aux_variable_values::Dict{AuxVarKey, DataFrame}`: Auxiliary variable outputs. See [`read_aux_variable`](@ref) and [`read_aux_variables`](@ref) -- `variable_values::Dict{VariableKey, DataFrame}`: Decision variable outputs. See [`read_variable`](@ref) and [`read_variables`](@ref) -- `dual_values::Dict{ConstraintKey, DataFrame}`: Dual outputs. See [`read_dual`](@ref) and [`read_duals`](@ref) -- `parameter_values::Dict{ParameterKey, DataFrame}`: Parameter outputs. See [`read_parameter`](@ref) and [`read_parameters`](@ref) -- `expression_values::Dict{ExpressionKey, DataFrame}`: Expression outputs. See [`read_expression`](@ref) and [`read_expressions`](@ref) -- `optimizer_stats::DataFrame`: Optimizer statistics for each solve -- `optimization_container_metadata::OptimizationContainerMetadata`: Metadata about the optimization container. Internal usage. -- `model_type::String`: Type of optimization model. Internal usage. -- `outputs_dir::String`: Directory where outputs are stored -- `output_dir::String`: Directory for exported output - -See also: [`OptimizerStats`](@ref), [`OptimizationProblemOutputsExport`](@ref) -""" -mutable struct OptimizationProblemOutputs <: Outputs - base_power::Float64 - timestamps::Vector{Dates.DateTime} - source_data::Union{Nothing, InfrastructureSystemsType} - source_data_uuid::Base.UUID - aux_variable_values::Dict{AuxVarKey, DataFrame} - variable_values::Dict{VariableKey, DataFrame} - dual_values::Dict{ConstraintKey, DataFrame} - parameter_values::Dict{ParameterKey, DataFrame} - expression_values::Dict{ExpressionKey, DataFrame} - optimizer_stats::DataFrame - optimization_container_metadata::OptimizationContainerMetadata - model_type::String - outputs_dir::String - output_dir::String -end - -function OptimizationProblemOutputs( - base_power, - timestamps::StepRange{Dates.DateTime, Dates.Millisecond}, - source_data, - source_data_uuid, - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - optimization_container_metadata, - model_type, - outputs_dir, - output_dir, -) - return OptimizationProblemOutputs( - base_power, - collect(timestamps), - source_data, - source_data_uuid, - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - optimization_container_metadata, - model_type, - outputs_dir, - output_dir, - ) -end - -list_aux_variable_keys(res::OptimizationProblemOutputs) = - collect(keys(res.aux_variable_values)) -list_aux_variable_names(res::OptimizationProblemOutputs) = - encode_keys_as_strings(keys(res.aux_variable_values)) -list_variable_keys(res::OptimizationProblemOutputs) = collect(keys(res.variable_values)) -list_variable_names(res::OptimizationProblemOutputs) = - encode_keys_as_strings(keys(res.variable_values)) -list_parameter_keys(res::OptimizationProblemOutputs) = collect(keys(res.parameter_values)) -list_parameter_names(res::OptimizationProblemOutputs) = - encode_keys_as_strings(keys(res.parameter_values)) -list_dual_keys(res::OptimizationProblemOutputs) = collect(keys(res.dual_values)) -list_dual_names(res::OptimizationProblemOutputs) = - encode_keys_as_strings(keys(res.dual_values)) -list_expression_keys(res::OptimizationProblemOutputs) = collect(keys(res.expression_values)) -list_expression_names(res::OptimizationProblemOutputs) = - encode_keys_as_strings(keys(res.expression_values)) -get_timestamps(res::OptimizationProblemOutputs) = res.timestamps -get_model_base_power(res::OptimizationProblemOutputs) = res.base_power -get_dual_values(res::OptimizationProblemOutputs) = res.dual_values -get_expression_values(res::OptimizationProblemOutputs) = res.expression_values -get_variable_values(res::OptimizationProblemOutputs) = res.variable_values -get_aux_variable_values(res::OptimizationProblemOutputs) = res.aux_variable_values -get_total_cost(res::OptimizationProblemOutputs) = get_objective_value(res) -get_optimizer_stats(res::OptimizationProblemOutputs) = res.optimizer_stats -get_parameter_values(res::OptimizationProblemOutputs) = res.parameter_values -get_source_data(res::OptimizationProblemOutputs) = res.source_data - -make_system_filename(sys::IS.InfrastructureSystemsContainer) = - make_system_filename(get_system_uuid(sys)) -make_system_filename(sys_uuid::Union{Base.UUID, AbstractString}) = "system-$(sys_uuid).json" - -""" -Load the system from disk if not already set, and return it. - -Currently only used in the tests, not downstream in POM. -""" -function load_system(res::OptimizationProblemOutputs; kwargs...) - !isnothing(get_source_data(res)) && return - file = joinpath(get_outputs_dir(res), make_system_filename(get_source_data_uuid(res))) - if isfile(file) - sys = IS.InfrastructureSystemsContainer(file; time_series_read_only = true) - @info "De-serialized the system from files." - else - error("Could not locate system file: $file") - end - set_source_data!(res, sys) - return -end - -get_forecast_horizon(res::OptimizationProblemOutputs) = length(get_timestamps(res)) -get_output_dir(res::OptimizationProblemOutputs) = res.output_dir -get_outputs_dir(res::OptimizationProblemOutputs) = res.outputs_dir -get_source_data_uuid(res::OptimizationProblemOutputs) = res.source_data_uuid - -get_output_values(x::OptimizationProblemOutputs, ::AuxVarKey) = x.aux_variable_values -get_output_values(x::OptimizationProblemOutputs, ::ConstraintKey) = x.dual_values -get_output_values(x::OptimizationProblemOutputs, ::ExpressionKey) = x.expression_values -get_output_values(x::OptimizationProblemOutputs, ::ParameterKey) = x.parameter_values -get_output_values(x::OptimizationProblemOutputs, ::VariableKey) = x.variable_values - -function get_objective_value(res::OptimizationProblemOutputs, execution = 1) - return res.optimizer_stats[execution, :objective_value] -end - -function get_resolution(res::OptimizationProblemOutputs) - # Method return the resolution between timestamps. - # If multiple resolutions are present it returns the first observed. - # If single timestamp is used, it return. - diff_res = diff(get_timestamps(res)) - if !isempty(diff_res) - unique!(diff_res) - if length(diff_res) == 1 - return only(diff_res) - else - @warn "Multiple resolutions detected, returning the first resolution." - return first(diff_res) - end - end - return -end - -function get_realized_timestamps( - res::IS.Outputs; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, -) - timestamps = get_timestamps(res) - resolution = get_resolution(res) - intervals = diff(timestamps) - if isempty(intervals) && isnothing(resolution) - interval = Dates.Millisecond(1) - resolution = Dates.Millisecond(1) - elseif !isempty(intervals) && isnothing(resolution) - interval = first(intervals) - resolution = interval - elseif isempty(intervals) && !isnothing(resolution) - interval = resolution - else - interval = first(intervals) - end - horizon = get_forecast_horizon(res) - start_time = isnothing(start_time) ? first(timestamps) : start_time - end_time = - if isnothing(len) - last(timestamps) + interval - resolution - else - start_time + (len - 1) * resolution - end - - requested_range = start_time:resolution:end_time - available_range = - first(timestamps):resolution:(last(timestamps) + (horizon - 1) * resolution) - invalid_timestamps = setdiff(requested_range, available_range) - - if !isempty(invalid_timestamps) - msg = "Requested time does not match available outputs" - @error msg - throw(IS.InvalidValue(msg)) - end - - return requested_range -end - -function export_output( - ::Type{CSV.File}, - path, - key::OptimizationContainerKey, - timestamp::Dates.DateTime, - df::DataFrame, -) - name = encode_key_as_string(key) - export_output(CSV.File, path, name, timestamp, df) - return -end - -function export_output( - ::Type{CSV.File}, - path, - name::AbstractString, - timestamp::Dates.DateTime, - df::DataFrame, -) - filename = joinpath(path, name * "_" * convert_for_path(timestamp) * ".csv") - export_output(CSV.File, filename, df) - return -end - -function export_output( - ::Type{CSV.File}, - path, - key::OptimizationContainerKey, - df::DataFrame, -) - name = encode_key_as_string(key) - export_output(CSV.File, path, name, df) - return -end - -function export_output( - ::Type{CSV.File}, - path, - name::AbstractString, - df::DataFrame, -) - filename = joinpath(path, name * ".csv") - export_output(CSV.File, filename, df) - return -end - -function export_output(::Type{CSV.File}, filename, df::DataFrame) - open(filename, "w") do io - CSV.write(io, df) - end - - @debug "Exported $filename" - return -end - -""" -Exports all outputs from the operations problem. -""" -function export_outputs(outputs::OptimizationProblemOutputs; kwargs...) - exports = OptimizationProblemOutputsExport( - "Problem"; - store_all_duals = true, - store_all_parameters = true, - store_all_variables = true, - store_all_aux_variables = true, - ) - return export_outputs(outputs, exports; kwargs...) -end - -function export_outputs( - outputs::OptimizationProblemOutputs, - exports::OptimizationProblemOutputsExport; - file_type = CSV.File, -) - file_type != CSV.File && error("only CSV.File is currently supported") - for (source, decider, label) in [ - (outputs.variable_values, should_export_variable, "variables"), - (outputs.aux_variable_values, should_export_aux_variable, "aux_variables"), - (outputs.dual_values, should_export_dual, "duals"), - (outputs.parameter_values, should_export_parameter, "parameters"), - (outputs.expression_values, should_export_expression, "expressions"), - ] - export_path = mkpath(joinpath(get_output_dir(outputs), label)) - for (key, df) in source - if decider(exports, key) - export_output(file_type, export_path, key, df) - end - end - end - - if exports.optimizer_stats - export_output( - file_type, - joinpath(get_output_dir(outputs), "optimizer_stats.csv"), - outputs.optimizer_stats, - ) - end - - @info "Exported OptimizationProblemOutputs to $(get_output_dir(outputs))" -end - -function _deserialize_key( - ::Type{<:OptimizationContainerKey}, - outputs::OptimizationProblemOutputs, - name::AbstractString, -) - return deserialize_key(outputs.optimization_container_metadata, name) -end - -function _deserialize_key( - ::Type{T}, - ::OptimizationProblemOutputs, - args..., -) where {T <: OptimizationContainerKey} - return make_key(T, args...) -end - -function _validate_keys(existing_keys, output_keys) - diff = setdiff(output_keys, existing_keys) - if !isempty(diff) - throw(InvalidValue("These keys are not stored: $diff")) - end - return -end - -read_optimizer_stats(res::OptimizationProblemOutputs) = res.optimizer_stats - -""" -Set the system in the outputs instance. - -Throws InvalidValue if the source UUID is incorrect. -""" -function set_source_data!( - res::OptimizationProblemOutputs, - source::InfrastructureSystemsType, -) - source_uuid = get_uuid(source) - if source_uuid != res.source_data_uuid - throw( - InvalidValue( - "System mismatch. $sys_uuid does not match the stored value of $(res.source_uuid)", - ), - ) - end - - res.source_data = source - return -end - -const _PROBLEM_OUTPUTS_FILENAME = "problem_outputs.bin" - -# TODO test this in IS -""" -Serialize the outputs to a binary file. - -It is recommended that `directory` be the directory that contains a serialized -OperationModel. That will allow automatic deserialization of the PowerSystems.System. -The `OptimizationProblemOutputs` instance can be deserialized with `OptimizationProblemOutputs(directory)`. -""" -function serialize_outputs(res::OptimizationProblemOutputs, directory::AbstractString) - mkpath(directory) - filename = joinpath(directory, _PROBLEM_OUTPUTS_FILENAME) - isfile(filename) && rm(filename) - Serialization.serialize(filename, _copy_for_serialization(res)) - @info "Serialize OptimizationProblemOutputs to $filename" -end - -""" -Construct a OptimizationProblemOutputs instance from a serialized directory. It is up to the -user or a higher-level package to set the source data using [`set_source_data!`](@ref). -""" -function OptimizationProblemOutputs(directory::AbstractString) - filename = joinpath(directory, _PROBLEM_OUTPUTS_FILENAME) - isfile(filename) || error("No outputs file exists in $directory") - return Serialization.deserialize(filename) -end - -function _copy_for_serialization(res::OptimizationProblemOutputs) - return OptimizationProblemOutputs( - res.base_power, - res.timestamps, - nothing, - res.source_data_uuid, - res.aux_variable_values, - res.variable_values, - res.dual_values, - res.parameter_values, - res.expression_values, - res.optimizer_stats, - res.optimization_container_metadata, - res.model_type, - res.outputs_dir, - res.output_dir, - ) -end - -function _read_outputs( - output_values::Dict{<:OptimizationContainerKey, DataFrame}, - container_keys, - timestamps::Vector{Dates.DateTime}, - time_ids, - base_power::Float64, - base_timestamps::Vector{Dates.DateTime}, - table_format::TableFormat, -) - existing_keys = keys(output_values) - container_keys = container_keys === nothing ? existing_keys : container_keys - _validate_keys(existing_keys, container_keys) - outputs = Dict{OptimizationContainerKey, DataFrame}() - IS.@assert_op length(time_ids) == length(timestamps) - df_timestamps = DataFrame(:DateTime => timestamps, :time_index => time_ids) - filter_timestamps = timestamps != base_timestamps - - for (key, df) in output_values - if !in(key, container_keys) - continue - end - if filter_timestamps - df = @subset(df, :time_index .∈ Ref(time_ids)) - end - first_dim_col = get_first_dimension_output_column_name(key) - second_dim_col = get_second_dimension_output_column_name(key) - component_cols = [first_dim_col] - if second_dim_col in names(df) - push!(component_cols, second_dim_col) - if table_format == TableFormat.WIDE - error( - "Wide format is not supported with 3-dimensional outputs", - ) - end - end - num_components = DataFrames.nrow(unique(df[:, component_cols])) - num_rows = DataFrames.nrow(df) - if num_rows % num_components != 0 - error( - "num_rows = $num_rows is not divisible by num_components = $num_components", - ) - end - num_rows_per_component = num_rows ÷ num_components - if num_rows_per_component == length(time_ids) == length(timestamps) - tmp_df = innerjoin(df, df_timestamps; on = :time_index) - if DataFrames.nrow(tmp_df) != DataFrames.nrow(df) - error( - "Bug: Unexpectedly dropped rows: df2 = $tmp_df orig = $(outputs[key])", - ) - end - outputs[key] = select(tmp_df, [:DateTime, Symbol.(component_cols)..., :value]) - else - @warn "Length of variables is different than timestamps. Ignoring timestamps." - outputs[key] = deepcopy(df) - end - outputs[key] = _handle_natural_units(outputs[key], base_power, key) - if table_format == TableFormat.WIDE - outputs[key] = DataFrames.unstack(outputs[key], first_dim_col, "value") - end - end - return outputs -end - -""" -Convert the value column to natural units, if required by the key. -Does not mutate the input dataframe. -""" -function _handle_natural_units( - df::DataFrame, - base_power::Float64, - key::OptimizationContainerKey, -) - return if convert_output_to_natural_units(key) - @transform(df, :value = :value * base_power) - else - df - end -end - -function _process_timestamps( - res::OptimizationProblemOutputs, - start_time::Union{Nothing, Dates.DateTime}, - len::Union{Int, Nothing}, -) - if start_time === nothing - start_time = first(get_timestamps(res)) - elseif start_time ∉ get_timestamps(res) - throw(InvalidValue("start_time not in output timestamps")) - end - - if startswith(res.model_type, "EmulationModel{") - def_len = DataFrames.nrow(get_optimizer_stats(res)) - requested_range = - collect(findfirst(x -> x >= start_time, get_timestamps(res)):def_len) - timestamps = repeat(get_timestamps(res), def_len) - else - timestamps = get_timestamps(res) - requested_range = findall(x -> x >= start_time, timestamps) - def_len = length(requested_range) - end - actual_len = if len === nothing - def_len - elseif len < 0 - throw(InvalidValue("len cannot be negative: $len")) - elseif len > def_len - throw(InvalidValue("requested outputs have less than $len values")) - else - len - end - timestamp_ids = requested_range[1:actual_len] - return timestamp_ids, timestamps[timestamp_ids] -end - -""" -Return the values for the requested variable key for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - -- `res::OptimizationProblemOutputs`: Optimization problem outputs -- `variable::Tuple{Type{<:VariableType}, Type{<:IS.InfrastructureSystemsComponent}`: Tuple with variable type - and device type for the desired outputs -- `start_time::Dates.DateTime`: Start time of the requested outputs -- `len::Int`: length of outputs -- `table_format::TableFormat`: Format of the table to be returned. Default is - `TableFormat.LONG` where the columns are `DateTime`, `name`, and `value` when the data - has two dimensions and `DateTime`, `name`, `name2`, and `value` when the data has three - dimensions. - Set to it `TableFormat.WIDE` to pivot the names as columns. - Note: `TableFormat.WIDE` is not supported when the data has more than two dimensions. -""" -function read_variable( - res::OptimizationProblemOutputs, - args...; - kwargs..., -) - key = VariableKey(args...) - return read_variable(res, key; kwargs...) -end - -function read_variable(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) - return read_variable(res, _deserialize_key(VariableKey, res, key); kwargs...) -end - -function read_variable( - res::OptimizationProblemOutputs, - key::VariableKey; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - return read_outputs_with_keys( - res, - [key]; - start_time = start_time, - len = len, - table_format = table_format, - )[key] -end - -""" -Return the values for the requested variable keys for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `variables::Vector{Tuple{Type{<:VariableType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with variable type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_variables(res::OptimizationProblemOutputs, variables; kwargs...) - return read_variables(res, [VariableKey(x...) for x in variables]; kwargs...) -end - -function read_variables( - res::OptimizationProblemOutputs, - variables::Vector{<:AbstractString}; - kwargs..., -) - return read_variables( - res, - [_deserialize_key(VariableKey, res, x) for x in variables]; - kwargs..., - ) -end - -function read_variables( - res::OptimizationProblemOutputs, - variables::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - output_values = - read_outputs_with_keys( - res, - variables; - start_time = start_time, - len = len, - table_format = table_format, - ) - return Dict(encode_key_as_string(k) => v for (k, v) in output_values) -end - -""" -Return the values for all variables. -""" -function read_variables(res::Outputs; kwargs...) - return Dict(x => read_variable(res, x; kwargs...) for x in list_variable_names(res)) -end - -""" -Return the values for the requested dual key for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `dual::Tuple{Type{<:ConstraintType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with dual type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_dual( - res::OptimizationProblemOutputs, - args...; - kwargs..., -) - key = ConstraintKey(args...) - return read_dual(res, key; kwargs...) -end - -function read_dual(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) - return read_dual(res, _deserialize_key(ConstraintKey, res, key); kwargs...) -end - -function read_dual( - res::OptimizationProblemOutputs, - key::ConstraintKey; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - return read_outputs_with_keys( - res, - [key]; - start_time = start_time, - len = len, - table_format = table_format, - )[key] -end - -""" -Return the values for the requested dual keys for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `duals::Vector{Tuple{Type{<:ConstraintType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with dual type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_duals(res::OptimizationProblemOutputs, duals; kwargs...) - return read_duals(res, [ConstraintKey(x...) for x in duals]; kwargs...) -end - -function read_duals( - res::OptimizationProblemOutputs, - duals::Vector{<:AbstractString}; - kwargs..., -) - return read_duals( - res, - [_deserialize_key(ConstraintKey, res, x) for x in duals]; - kwargs..., - ) -end - -function read_duals( - res::OptimizationProblemOutputs, - duals::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - output_values = read_outputs_with_keys( - res, - duals; - start_time = start_time, - len = len, - table_format = table_format, - ) - return Dict(encode_key_as_string(k) => v for (k, v) in output_values) -end - -""" -Return the values for all duals. -""" -function read_duals(res::Outputs; kwargs...) - duals = Dict(x => read_dual(res, x; kwargs...) for x in list_dual_names(res)) -end - -""" -Return the values for the requested parameter key for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `parameter::Tuple{Type{<:ParameterType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with parameter type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_parameter( - res::OptimizationProblemOutputs, - args...; - kwargs..., -) - key = ParameterKey(args...) - return read_parameter(res, key; kwargs...) -end - -function read_parameter(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) - return read_parameter(res, _deserialize_key(ParameterKey, res, key); kwargs...) -end - -function read_parameter( - res::OptimizationProblemOutputs, - key::ParameterKey; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - return read_outputs_with_keys( - res, - [key]; - start_time = start_time, - len = len, - table_format = table_format, - )[key] -end - -""" -Return the values for the requested parameter keys for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `parameters::Vector{Tuple{Type{<:ParameterType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with parameter type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_parameters(res::OptimizationProblemOutputs, parameters; kwargs...) - return read_parameters(res, [ParameterKey(x...) for x in parameters]; kwargs...) -end - -function read_parameters( - res::OptimizationProblemOutputs, - parameters::Vector{<:AbstractString}; - kwargs..., -) - return read_parameters( - res, - [_deserialize_key(ParameterKey, res, x) for x in parameters]; - kwargs..., - ) -end - -function read_parameters( - res::OptimizationProblemOutputs, - parameters::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - output_values = - read_outputs_with_keys( - res, - parameters; - start_time = start_time, - len = len, - table_format = table_format, - ) - return Dict(encode_key_as_string(k) => v for (k, v) in output_values) -end - -""" -Return the values for all parameters. -""" -function read_parameters(res::Outputs; kwargs...) - parameters = - Dict(x => read_parameter(res, x; kwargs...) for x in list_parameter_names(res)) -end - -""" -Return the values for the requested aux_variable key for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `aux_variable::Tuple{Type{<:AuxVariableType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with aux_variable type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_aux_variable( - res::OptimizationProblemOutputs, - args...; - kwargs..., -) - key = AuxVarKey(args...) - return read_aux_variable(res, key; kwargs...) -end - -function read_aux_variable(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) - return read_aux_variable(res, _deserialize_key(AuxVarKey, res, key); kwargs...) -end - -function read_aux_variable( - res::OptimizationProblemOutputs, - key::AuxVarKey; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - return read_outputs_with_keys( - res, - [key]; - start_time = start_time, - len = len, - table_format = table_format, - )[key] -end - -""" -Return the values for the requested aux_variable keys for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `aux_variables::Vector{Tuple{Type{<:AuxVariableType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with aux_variable type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_aux_variables(res::OptimizationProblemOutputs, aux_variables; kwargs...) - return read_aux_variables(res, [AuxVarKey(x...) for x in aux_variables]; kwargs...) -end - -function read_aux_variables( - res::OptimizationProblemOutputs, - aux_variables::Vector{<:AbstractString}; - kwargs..., -) - return read_aux_variables( - res, - [_deserialize_key(AuxVarKey, res, x) for x in aux_variables]; - kwargs..., - ) -end - -function read_aux_variables( - res::OptimizationProblemOutputs, - aux_variables::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - output_values = - read_outputs_with_keys( - res, - aux_variables; - start_time = start_time, - len = len, - table_format = table_format, - ) - return Dict(encode_key_as_string(k) => v for (k, v) in output_values) -end - -""" -Return the values for all auxiliary variables. -""" -function read_aux_variables(res::Outputs; kwargs...) - return Dict( - x => read_aux_variable(res, x; kwargs...) for x in list_aux_variable_names(res) - ) -end - -""" -Return the values for the requested expression key for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `expression::Tuple{Type{<:ExpressionType}, Type{<:IS.InfrastructureSystemsComponent}` : Tuple with expression type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_expression( - res::OptimizationProblemOutputs, - args...; - kwargs..., -) - key = ExpressionKey(args...) - return read_expression(res, key; kwargs...) -end - -function read_expression(res::OptimizationProblemOutputs, key::AbstractString; kwargs...) - return read_expression(res, _deserialize_key(ExpressionKey, res, key); kwargs...) -end - -function read_expression( - res::OptimizationProblemOutputs, - key::ExpressionKey; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - return read_outputs_with_keys( - res, - [key]; - start_time = start_time, - len = len, - table_format = table_format, - )[key] -end - -""" -Return the values for the requested expression keys for a problem. -Accepts a vector of keys for the return of the values. - -# Arguments - - - `expressions::Vector{Tuple{Type{<:ExpressionType}, Type{<:IS.InfrastructureSystemsComponent}}` : Tuple with expression type and device type for the desired outputs - - `start_time::Dates.DateTime` : initial time of the requested outputs - - `len::Int`: length of outputs -""" -function read_expressions(res::OptimizationProblemOutputs; kwargs...) - return read_expressions(res, collect(keys(res.expression_values)); kwargs...) -end - -function read_expressions(res::OptimizationProblemOutputs, expressions; kwargs...) - return read_expressions(res, [ExpressionKey(x...) for x in expressions]; kwargs...) -end - -function read_expressions( - res::OptimizationProblemOutputs, - expressions::Vector{<:AbstractString}; - kwargs..., -) - return read_expressions( - res, - [_deserialize_key(ExpressionKey, res, x) for x in expressions]; - kwargs..., - ) -end - -function read_expressions( - res::OptimizationProblemOutputs, - expressions::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - output_values = - read_outputs_with_keys( - res, - expressions; - start_time = start_time, - len = len, - table_format = table_format, - ) - return Dict(encode_key_as_string(k) => v for (k, v) in output_values) -end - -""" -Return the values for all expressions. -""" -function read_expressions(res::Outputs; kwargs...) - return Dict(x => read_expression(res, x) for x in list_expression_names(res)) -end - -function read_outputs_with_keys( - res::OptimizationProblemOutputs, - output_keys::Vector{<:OptimizationContainerKey}; - start_time::Union{Nothing, Dates.DateTime} = nothing, - len::Union{Int, Nothing} = nothing, - table_format::TableFormat = TableFormat.LONG, -) - isempty(output_keys) && return Dict{OptimizationContainerKey, DataFrame}() - (timestamp_ids, timestamps) = _process_timestamps(res, start_time, len) - - base_timestamps = get_timestamps(res) - return _read_outputs( - get_output_values(res, first(output_keys)), - output_keys, - timestamps, - timestamp_ids, - get_model_base_power(res), - base_timestamps, - table_format, - ) -end - -""" -Save the realized outputs to CSV files for all variables, paramaters, duals, auxiliary variables, -expressions, and optimizer statistics. - -# Arguments - - - `res::Outputs`: Outputs - - `save_path::AbstractString` : path to save outputs (defaults to simulation path) -""" -function export_realized_outputs(res::Outputs) - save_path = mkpath(joinpath(get_output_dir(res), "export")) - return export_realized_outputs(res, save_path) -end - -function export_realized_outputs( - res::Outputs, - save_path::AbstractString, -) - if !isdir(save_path) - throw(IS.ConflictingInputsError("Specified path is not valid.")) - end - write_data(read_outputs_with_keys(res, list_variable_keys(res)), save_path) - !isempty(list_dual_keys(res)) && - write_data( - read_outputs_with_keys(res, list_dual_keys(res)), - save_path; - name = "dual", - ) - !isempty(list_parameter_keys(res)) && write_data( - read_outputs_with_keys(res, list_parameter_keys(res)), - save_path; - name = "parameter", - ) - !isempty(list_aux_variable_keys(res)) && write_data( - read_outputs_with_keys(res, list_aux_variable_keys(res)), - save_path; - name = "aux_variable", - ) - !isempty(list_expression_keys(res)) && write_data( - read_outputs_with_keys(res, list_expression_keys(res)), - save_path; - name = "expression", - ) - export_optimizer_stats(res, save_path) - files = readdir(save_path) - compute_file_hash(save_path, files) - @info("Files written to $save_path folder.") - return save_path -end - -""" -Save the optimizer statistics to CSV or JSON - -# Arguments - - - `res::Union{OptimizationProblemOutputs, SimulationProblmeOutputs`: Outputs - - `directory::AbstractString` : target directory - - `format = "CSV"` : can be "csv" or "json -""" -function export_optimizer_stats( - res::Outputs, - directory::AbstractString; - format = "csv", -) - data = read_optimizer_stats(res) - isnothing(data) && return - if uppercase(format) == "CSV" - CSV.write(joinpath(directory, "optimizer_stats.csv"), data) - elseif uppercase(format) == "JSON" - JSON.write(joinpath(directory, "optimizer_stats.json"), JSON.json(to_dict(data))) - else - throw(error("writing optimizer stats only supports csv or json formats")) - end -end - -function write_data( - vars_outputs::Dict, - time::DataFrame, - save_path::AbstractString, -) - for (k, v) in vars_outputs - var = DataFrame() - if size(time, 1) == size(v, 1) - var = hcat(time, v) - else - var = v - end - file_path = joinpath(save_path, "$(k).csv") - CSV.write(file_path, var) - end -end - -function write_data( - data::DataFrame, - save_path::AbstractString, - file_name::String, -) - if isfile(save_path) - save_path = dirname(save_path) - end - file_path = joinpath(save_path, "$(file_name).csv") - CSV.write(file_path, data) - return -end - -# writing a dictionary of dataframes to files -function write_data(vars_outputs::Dict, save_path::String; kwargs...) - name = get(kwargs, :name, "") - for (k, v) in vars_outputs - keyname = encode_key_as_string(k) - file_path = joinpath(save_path, "$name$keyname.csv") - @debug "writing" file_path - if isempty(vars_outputs[k]) - @debug "$name$k is empty, not writing $file_path" - else - CSV.write(file_path, vars_outputs[k]) - end - end -end diff --git a/src/core/optimization_problem_outputs_export.jl b/src/core/optimization_problem_outputs_export.jl deleted file mode 100644 index 87447226..00000000 --- a/src/core/optimization_problem_outputs_export.jl +++ /dev/null @@ -1,130 +0,0 @@ -""" - struct OptimizationProblemOutputsExport - -Configuration for exporting optimization problem outputs to files. - -Specifies which variables, duals, parameters, expressions, and auxiliary variables -should be exported when calling [`export_outputs`](@ref) on an -[`OptimizationProblemOutputs`](@ref) instance. - -# Fields -- `name::Symbol`: Name identifier for this export configuration -- `duals::Set{ConstraintKey}`: Specific dual values to export -- `expressions::Set{ExpressionKey}`: Specific expression values to export -- `parameters::Set{ParameterKey}`: Specific parameter values to export -- `variables::Set{VariableKey}`: Specific variable values to export -- `aux_variables::Set{AuxVarKey}`: Specific auxiliary variable values to export -- `optimizer_stats::Bool`: Whether to export optimizer statistics -- `store_all_flags::Dict{Symbol, Bool}`: Flags indicating whether to export all values - of each type (e.g., all variables, all duals). Set via constructor keyword arguments - like `store_all_variables = true`. When a flag is true, all values of that type are - exported regardless of what specific keys are passed in the corresponding set. - -# Example -```julia -export_config = OptimizationProblemOutputsExport( - "MyExport"; - store_all_variables = true, - store_all_duals = false, - optimizer_stats = true, -) -export_outputs(outputs, export_config) -``` - -See also: [`OptimizationProblemOutputs`](@ref), [`export_outputs`](@ref) -""" -struct OptimizationProblemOutputsExport - name::Symbol - duals::Set{ConstraintKey} - expressions::Set{ExpressionKey} - parameters::Set{ParameterKey} - variables::Set{VariableKey} - aux_variables::Set{AuxVarKey} - optimizer_stats::Bool - store_all_flags::Dict{Symbol, Bool} - - function OptimizationProblemOutputsExport( - name, - duals, - expressions, - parameters, - variables, - aux_variables, - optimizer_stats, - store_all_flags, - ) - duals = _check_fields(duals) - expressions = _check_fields(expressions) - parameters = _check_fields(parameters) - variables = _check_fields(variables) - aux_variables = _check_fields(aux_variables) - new( - name, - duals, - expressions, - parameters, - variables, - aux_variables, - optimizer_stats, - store_all_flags, - ) - end -end - -function OptimizationProblemOutputsExport( - name; - duals = Set{ConstraintKey}(), - expressions = Set{ExpressionKey}(), - parameters = Set{ParameterKey}(), - variables = Set{VariableKey}(), - aux_variables = Set{AuxVarKey}(), - optimizer_stats = true, - store_all_duals = false, - store_all_expressions = false, - store_all_parameters = false, - store_all_variables = false, - store_all_aux_variables = false, -) - store_all_flags = Dict( - :duals => store_all_duals, - :expressions => store_all_expressions, - :parameters => store_all_parameters, - :variables => store_all_variables, - :aux_variables => store_all_aux_variables, - ) - return OptimizationProblemOutputsExport( - Symbol(name), - duals, - expressions, - parameters, - variables, - aux_variables, - optimizer_stats, - store_all_flags, - ) -end - -function _check_fields(fields) - if !(typeof(fields) <: Set) - fields = Set(fields) - end - - return fields -end - -should_export_dual(x::OptimizationProblemOutputsExport, key) = - _should_export(x, :duals, key) -should_export_expression(x::OptimizationProblemOutputsExport, key) = - _should_export(x, :expressions, key) -should_export_parameter(x::OptimizationProblemOutputsExport, key) = - _should_export(x, :parameters, key) -should_export_variable(x::OptimizationProblemOutputsExport, key) = - _should_export(x, :variables, key) -should_export_aux_variable(x::OptimizationProblemOutputsExport, key) = - _should_export(x, :aux_variables, key) - -function _should_export(exports::OptimizationProblemOutputsExport, field_name, key) - exports.store_all_flags[field_name] && return true - container = getproperty(exports, field_name) - return key in container -end diff --git a/src/operation/decision_model.jl b/src/operation/decision_model.jl deleted file mode 100644 index 5585c422..00000000 --- a/src/operation/decision_model.jl +++ /dev/null @@ -1,322 +0,0 @@ -function get_deterministic_time_series_type(sys::IS.InfrastructureSystemsContainer) - time_series_types = get_time_series_counts_by_type(sys) - existing_types = Set(d["type"] for d in time_series_types) - if ("Deterministic" in existing_types) && - ("DeterministicSingleTimeSeries" in existing_types) - error( - "The System contains a combination of forecast data and transformed time series data. Currently this is not supported.", - ) - end - if "Deterministic" ∈ existing_types - return IS.Deterministic - elseif "DeterministicSingleTimeSeries" ∈ existing_types - return IS.DeterministicSingleTimeSeries - else - error( - "The System does not contain any forecast data or transformed time series data.", - ) - end -end - -""" -Abstract type for models that use default InfrastructureOptimizationModels formulations. For custom decision problems - use DecisionProblem as the super type. -""" -abstract type DefaultDecisionProblem <: DecisionProblem end - -""" -Generic InfrastructureOptimizationModels Operation Problem Type for unspecified models -""" -struct GenericOpProblem <: DefaultDecisionProblem end - -mutable struct DecisionModel{M <: DecisionProblem} <: OperationModel - name::Symbol - template::AbstractProblemTemplate - sys::IS.InfrastructureSystemsContainer - internal::Union{Nothing, ModelInternal} - simulation_info::Union{Nothing, SimulationInfo} - store::DecisionModelStore - ext::Dict{String, Any} -end - -""" - DecisionModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model}=nothing; - kwargs...) where {M<:DecisionProblem} - -Build the optimization problem of type M with the specific system and template. - -# Arguments - - - `::Type{M} where M<:DecisionProblem`: The abstract operation model type - - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. - - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems - - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care - - `name = nothing`: name of model, string or symbol; defaults to the type of template converted to a symbol. - - `optimizer::Union{Nothing,MOI.OptimizerWithAttributes} = nothing` : The optimizer does - not get serialized. Callers should pass whatever they passed to the original problem. - - `horizon::Dates.Period = UNSET_HORIZON`: Manually specify the length of the forecast Horizon - - `resolution::Dates.Period = UNSET_RESOLUTION`: Manually specify the model's resolution - - `warm_start::Bool = true`: True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true - - `check_components::Bool = true`: True to check the components valid fields when building - - `initialize_model::Bool = true`: Option to decide to initialize the model or not. - - `initialization_file::String = ""`: This allows to pass pre-existing initialization values to avoid the solution of an optimization problem to find feasible initial conditions. - - `deserialize_initial_conditions::Bool = false`: Option to deserialize conditions - - `export_pwl_vars::Bool = false`: True to export all the pwl intermediate variables. It can slow down significantly the build and solve time. - - `allow_fails::Bool = false`: True to allow the simulation to continue even if the optimization step fails. Use with care. - - `optimizer_solve_log_print::Bool = false`: Uses JuMP.unset_silent() to print the optimizer's log. By default all solvers are set to MOI.Silent() - - `detailed_optimizer_stats::Bool = false`: True to save detailed optimizer stats log. - - `calculate_conflict::Bool = false`: True to use solver to calculate conflicts for infeasible problems. Only specific solvers are able to calculate conflicts. - - `direct_mode_optimizer::Bool = false`: True to use the solver in direct mode. Creates a [JuMP.direct_model](https://jump.dev/JuMP.jl/dev/reference/models/#JuMP.direct_model). - - `store_variable_names::Bool = false`: to store variable names in optimization model. Decreases the build times. - - `rebuild_model::Bool = false`: It will force the rebuild of the underlying JuMP model with each call to update the model. It increases solution times, use only if the model can't be updated in memory. - - `initial_time::Dates.DateTime = UNSET_INI_TIME`: Initial Time for the model solve. - - `time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES`: Size in bytes to cache for each time array. Default is 1 MiB. Set to 0 to disable. - -# Example - -```julia -template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) -OpModel = DecisionModel(MockOperationProblem, template, system) -``` -""" -function DecisionModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - settings::Settings, - jump_model::Union{Nothing, JuMP.Model} = nothing; - name = nothing, -) where {M <: DecisionProblem} - if name === nothing - name = nameof(M) - elseif name isa String - name = Symbol(name) - end - auto_transform_time_series!(sys, settings) - ts_type = get_deterministic_time_series_type(sys) - internal = ModelInternal( - OptimizationContainer(sys, settings, jump_model, ts_type), - ) - - template_ = deepcopy(template) - finalize_template!(template_, sys) - model = DecisionModel{M}( - name, - template_, - sys, - internal, - SimulationInfo(), - DecisionModelStore(), - Dict{String, Any}(), - ) - validate_time_series!(model) - return model -end - -function DecisionModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - name = nothing, - optimizer = nothing, - horizon = UNSET_HORIZON, - resolution = UNSET_RESOLUTION, - interval = UNSET_INTERVAL, - warm_start = true, - check_components = true, - initialize_model = true, - initialization_file = "", - deserialize_initial_conditions = false, - export_pwl_vars = false, - allow_fails = false, - optimizer_solve_log_print = false, - detailed_optimizer_stats = false, - calculate_conflict = false, - direct_mode_optimizer = false, - store_variable_names = false, - rebuild_model = false, - export_optimization_model = false, - check_numerical_bounds = true, - initial_time = UNSET_INI_TIME, - time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES, -) where {M <: DecisionProblem} - settings = Settings( - sys; - horizon = horizon, - resolution = resolution, - interval = interval, - initial_time = initial_time, - optimizer = optimizer, - time_series_cache_size = time_series_cache_size, - warm_start = warm_start, - check_components = check_components, - initialize_model = initialize_model, - initialization_file = initialization_file, - deserialize_initial_conditions = deserialize_initial_conditions, - export_pwl_vars = export_pwl_vars, - allow_fails = allow_fails, - calculate_conflict = calculate_conflict, - optimizer_solve_log_print = optimizer_solve_log_print, - detailed_optimizer_stats = detailed_optimizer_stats, - direct_mode_optimizer = direct_mode_optimizer, - check_numerical_bounds = check_numerical_bounds, - store_variable_names = store_variable_names, - rebuild_model = rebuild_model, - export_optimization_model = export_optimization_model, - ) - return DecisionModel{M}(template, sys, settings, jump_model; name = name) -end - -""" -Build the optimization problem of type M with the specific system and template - -# Arguments - - - `::Type{M} where M<:DecisionProblem`: The abstract operation model type - - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. - - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems - - `jump_model::Union{Nothing, JuMP.Model}` = nothing: Enables passing a custom JuMP model. Use with care. - -# Example - -```julia -template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) -problem = DecisionModel(MyOpProblemType, template, system, optimizer) -``` -""" -function DecisionModel( - ::Type{M}, - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) where {M <: DecisionProblem} - return DecisionModel{M}(template, sys, jump_model; kwargs...) -end - -function DecisionModel( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) - return DecisionModel{GenericOpProblem}(template, sys, jump_model; kwargs...) -end - -function DecisionModel{M}( - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) where {M <: DefaultDecisionProblem} - IS.ArgumentError( - "DefaultDecisionProblem subtypes require a template. Use DecisionModel subtyping instead.", - ) -end - -get_problem_type(::DecisionModel{M}) where {M <: DecisionProblem} = M - -function validate_template(::DecisionModel{M}) where {M <: DecisionProblem} - error("validate_template is not implemented for DecisionModel{$M}") -end - -function validate_template(::DecisionModel{<:DefaultDecisionProblem}) - return nothing -end - -# Probably could be more efficient by storing the info in the internal -function get_current_time(model::DecisionModel) - execution_count = get_execution_count(model) - initial_time = get_initial_time(model) - interval = get_interval(model) - return initial_time + interval * execution_count -end - -function init_model_store_params!(model::DecisionModel) - num_executions = get_executions(model) - horizon = get_horizon(model) - system = get_system(model) - settings = get_settings(model) - model_interval = get_interval(settings) - if model_interval != UNSET_INTERVAL - interval = model_interval - else - interval = get_forecast_interval(system) - end - resolution = get_resolution(model) - base_power = get_base_power(system) - sys_uuid = get_system_uuid(system) - store_params = ModelStoreParams( - num_executions, - horizon, - iszero(interval) ? resolution : interval, - resolution, - base_power, - sys_uuid, - get_metadata(get_optimization_container(model)), - ) - set_store_params!(get_internal(model), store_params) - return -end - -function validate_time_series!(model::DecisionModel{<:DefaultDecisionProblem}) - sys = get_system(model) - settings = get_settings(model) - available_resolutions = get_time_series_resolutions(sys) - - if get_resolution(settings) == UNSET_RESOLUTION && length(available_resolutions) != 1 - throw( - IS.ConflictingInputsError( - "Data contains multiple resolutions, the resolution keyword argument must be added to the Model. Time Series Resolutions: $(available_resolutions)", - ), - ) - elseif get_resolution(settings) != UNSET_RESOLUTION && length(available_resolutions) > 1 - if get_resolution(settings) ∉ available_resolutions - throw( - IS.ConflictingInputsError( - "Resolution $(get_resolution(settings)) is not available in the system data. Time Series Resolutions: $(available_resolutions)", - ), - ) - end - else - set_resolution!(settings, first(available_resolutions)) - end - - model_interval = get_interval(settings) - available_intervals = get_forecast_intervals(sys) - if model_interval == UNSET_INTERVAL && length(available_intervals) > 1 - throw( - IS.ConflictingInputsError( - "The system contains multiple forecast intervals $(available_intervals). " * - "The `interval` keyword argument must be provided to the DecisionModel constructor " * - "to select which interval to use.", - ), - ) - elseif model_interval != UNSET_INTERVAL && !isempty(available_intervals) - if model_interval ∉ available_intervals - throw( - IS.ConflictingInputsError( - "Interval $(Dates.canonicalize(model_interval)) is not available in the system data. " * - "Available forecast intervals: $(available_intervals)", - ), - ) - end - end - if get_horizon(settings) == UNSET_HORIZON - set_horizon!( - settings, - get_forecast_horizon(sys; interval = _to_is_interval(model_interval)), - ) - end - - counts = get_time_series_counts(sys) - if counts.forecast_count < 1 - error( - "The system does not contain forecast data. A DecisionModel can't be built.", - ) - end - return -end - -get_horizon(model::DecisionModel) = get_horizon(get_settings(model)) diff --git a/src/operation/decision_model_store.jl b/src/operation/decision_model_store.jl deleted file mode 100644 index e9f65c64..00000000 --- a/src/operation/decision_model_store.jl +++ /dev/null @@ -1,168 +0,0 @@ -""" -Stores outputs data for one DecisionModel -""" -mutable struct DecisionModelStore <: AbstractModelStore - # All DenseAxisArrays have axes (column names, row indexes) - duals::Dict{ConstraintKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} - parameters::Dict{ParameterKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} - variables::Dict{VariableKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} - aux_variables::Dict{AuxVarKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} - expressions::Dict{ExpressionKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}} - optimizer_stats::OrderedDict{Dates.DateTime, OptimizerStats} -end - -function DecisionModelStore() - return DecisionModelStore( - Dict{ConstraintKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), - Dict{ParameterKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), - Dict{VariableKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), - Dict{AuxVarKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), - Dict{ExpressionKey, OrderedDict{Dates.DateTime, DenseAxisArray{Float64}}}(), - OrderedDict{Dates.DateTime, OptimizerStats}(), - ) -end - -function initialize_storage!( - store::DecisionModelStore, - container::AbstractOptimizationContainer, - params::ModelStoreParams, -) - num_of_executions = get_num_executions(params) - if length(get_time_steps(container)) < 1 - error("The time step count in the optimization container is not defined") - end - time_steps_count = get_time_steps(container)[end] - initial_time = get_initial_time(container) - model_interval = get_interval(params) - for type in STORE_CONTAINERS - field_containers = getfield(container, type) - outputs_container = getfield(store, type) - for (key, field_container) in field_containers - !should_write_resulting_value(key) && continue - @debug "Adding $(encode_key_as_string(key)) to DecisionModelStore" _group = - LOG_GROUP_MODEL_STORE - column_names = get_column_names(container, type, field_container, key) - data = OrderedDict{ - Dates.DateTime, - DenseAxisArray{Float64, length(column_names) + 1}, - }() - for timestamp in - range(initial_time; step = model_interval, length = num_of_executions) - data[timestamp] = fill!( - DenseAxisArray{Float64}(undef, column_names..., 1:time_steps_count), - NaN, - ) - end - outputs_container[key] = data - end - end - return -end - -function write_output!( - store::DecisionModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::DecisionModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{T, 2, <:Tuple{Vector{String}, UnitRange}}, -) where {T} - container = getfield(store, get_store_container_type(key)) - container[key][index] = array - return -end - -function write_output!( - store::DecisionModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::DecisionModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{T, 2, <:Tuple{Vector{Int}, UnitRange}}, -) where {T} - columns = get_column_names_from_axis_array(array) - container = getfield(store, get_store_container_type(key)) - container[key][index] = DenseAxisArray(array.data, columns..., 1:size(array, 2)) - return -end - -function write_output!( - store::DecisionModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::DecisionModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{T, 1, <:Tuple{Vector{String}}}, -) where {T} - container = getfield(store, get_store_container_type(key)) - container[key][index] = array - return -end - -function write_output!( - store::DecisionModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::DecisionModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{T, 3, <:Tuple{Vector{String}, Vector{String}, UnitRange{Int}}}, -) where {T} - container = getfield(store, get_store_container_type(key)) - container[key][index] = array - return -end - -function write_output!( - store::DecisionModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::DecisionModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{T, 3, <:Tuple{Vector{String}, UnitRange, UnitRange}}, -) where {T} - container = getfield(store, get_store_container_type(key)) - container[key][index] = array - return -end - -function read_outputs( - store::DecisionModelStore, - key::OptimizationContainerKey; - index::Union{DecisionModelIndexType, Nothing} = nothing, -) - container = getfield(store, get_store_container_type(key)) - data = container[key] - if isnothing(index) - @assert_op length(data) == 1 - index = first(keys(data)) - end - - # Return a copy because callers may mutate it. - return deepcopy(data[index]) -end - -function write_optimizer_stats!( - store::DecisionModelStore, - stats::OptimizerStats, - index::DecisionModelIndexType, -) - # TODO: This check is incompatible with test calls to psi_checksolve_test - # Overwriting should not be allowed in normal operation. - # if index in keys(store.optimizer_stats) - # error("Bug: Overwriting optimizer stats for index = $index") - # end - store.optimizer_stats[index] = stats - return -end - -function read_optimizer_stats(store::DecisionModelStore) - stats = [IS.to_namedtuple(x) for x in values(store.optimizer_stats)] - df = DataFrames.DataFrame(stats) - DataFrames.insertcols!(df, 1, :DateTime => keys(store.optimizer_stats)) - return df -end - -function get_column_names(store::DecisionModelStore, key::OptimizationContainerKey) - container = getfield(store, get_store_container_type(key)) - return get_column_names_from_axis_array(key, first(values(container[key]))) -end diff --git a/src/operation/emulation_model.jl b/src/operation/emulation_model.jl deleted file mode 100644 index 1898e5b0..00000000 --- a/src/operation/emulation_model.jl +++ /dev/null @@ -1,354 +0,0 @@ -# FIXME untested. Moved to accommodate a few methods dispatching on EmulationModelStore, -# but not run in the tests and not yet refactored for IOM-POM split. -""" -Abstract type for models that use default InfrastructureOptimizationModels formulations. For custom emulation problems - use EmulationProblem as the super type. -""" -abstract type DefaultEmulationProblem <: EmulationProblem end - -""" -Default InfrastructureOptimizationModels Emulation Problem Type for unspecified problems -""" -struct GenericEmulationProblem <: DefaultEmulationProblem end - -""" - EmulationModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model}=nothing; - kwargs...) where {M<:EmulationProblem} - -Build the optimization problem of type M with the specific system and template. - -# Arguments - - - `::Type{M} where M<:EmulationProblem`: The abstract Emulation model type - - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, branches, and services. - - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems - - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care - - `name = nothing`: name of model, string or symbol; defaults to the type of template converted to a symbol. - - `optimizer::Union{Nothing,MOI.OptimizerWithAttributes} = nothing` : The optimizer does - not get serialized. Callers should pass whatever they passed to the original problem. - - `warm_start::Bool = true`: True will use the current operation point in the system to initialize variable values. False initializes all variables to zero. Default is true - - `initialize_model::Bool = true`: Option to decide to initialize the model or not. - - `initialization_file::String = ""`: This allows to pass pre-existing initialization values to avoid the solution of an optimization problem to find feasible initial conditions. - - `deserialize_initial_conditions::Bool = false`: Option to deserialize conditions - - `export_pwl_vars::Bool = false`: True to export all the pwl intermediate variables. It can slow down significantly the build and solve time. - - `allow_fails::Bool = false`: True to allow the simulation to continue even if the optimization step fails. Use with care. - - `calculate_conflict::Bool = false`: True to use solver to calculate conflicts for infeasible problems. Only specific solvers are able to calculate conflicts. - - `optimizer_solve_log_print::Bool = false`: Uses JuMP.unset_silent() to print the optimizer's log. By default all solvers are set to MOI.Silent() - - `detailed_optimizer_stats::Bool = false`: True to save detailed optimizer stats log. - - `direct_mode_optimizer::Bool = false`: True to use the solver in direct mode. Creates a [JuMP.direct_model](https://jump.dev/JuMP.jl/dev/reference/models/#JuMP.direct_model). - - `store_variable_names::Bool = false`: True to store variable names in optimization model. - - `rebuild_model::Bool = false`: It will force the rebuild of the underlying JuMP model with each call to update the model. It increases solution times, use only if the model can't be updated in memory. - - `initial_time::Dates.DateTime = UNSET_INI_TIME`: Initial Time for the model solve. - - `time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES`: Size in bytes to cache for each time array. Default is 1 MiB. Set to 0 to disable. - -# Example - -```julia -template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) -OpModel = EmulationModel(MockEmulationProblem, template, system) -``` -""" -mutable struct EmulationModel{M <: EmulationProblem} <: OperationModel - name::Symbol - template::AbstractProblemTemplate - sys::IS.InfrastructureSystemsContainer - internal::ModelInternal - simulation_info::SimulationInfo - store::EmulationModelStore # might be extended to other stores for simulation - ext::Dict{String, Any} - - function EmulationModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - settings::Settings, - jump_model::Union{Nothing, JuMP.Model} = nothing; - name = nothing, - ) where {M <: EmulationProblem} - if name === nothing - name = nameof(M) - elseif name isa String - name = Symbol(name) - end - finalize_template!(template, sys) - internal = ModelInternal( - OptimizationContainer(sys, settings, jump_model, IS.SingleTimeSeries), - ) - new{M}( - name, - template, - sys, - internal, - SimulationInfo(), - EmulationModelStore(), - Dict{String, Any}(), - ) - end -end - -function EmulationModel{M}( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - resolution = UNSET_RESOLUTION, - name = nothing, - optimizer = nothing, - warm_start = true, - initialize_model = true, - initialization_file = "", - deserialize_initial_conditions = false, - export_pwl_vars = false, - allow_fails = false, - calculate_conflict = false, - optimizer_solve_log_print = false, - detailed_optimizer_stats = false, - direct_mode_optimizer = false, - check_numerical_bounds = true, - store_variable_names = false, - rebuild_model = false, - initial_time = UNSET_INI_TIME, - time_series_cache_size::Int = IS.TIME_SERIES_CACHE_SIZE_BYTES, -) where {M <: EmulationProblem} - settings = Settings( - sys; - initial_time = initial_time, - optimizer = optimizer, - time_series_cache_size = time_series_cache_size, - warm_start = warm_start, - initialize_model = initialize_model, - initialization_file = initialization_file, - deserialize_initial_conditions = deserialize_initial_conditions, - export_pwl_vars = export_pwl_vars, - allow_fails = allow_fails, - calculate_conflict = calculate_conflict, - optimizer_solve_log_print = optimizer_solve_log_print, - detailed_optimizer_stats = detailed_optimizer_stats, - direct_mode_optimizer = direct_mode_optimizer, - check_numerical_bounds = check_numerical_bounds, - store_variable_names = store_variable_names, - rebuild_model = rebuild_model, - horizon = resolution, - resolution = resolution, - ) - model = EmulationModel{M}(template, sys, settings, jump_model; name = name) - validate_time_series!(model) - return model -end - -""" -Build the optimization problem of type M with the specific system and template - -# Arguments - - - `::Type{M} where M<:EmulationProblem`: The abstract Emulation model type - - `template::AbstractProblemTemplate`: The model reference made up of transmission, devices, - branches, and services. - - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems - - `jump_model::Union{Nothing, JuMP.Model}`: Enables passing a custom JuMP model. Use with care - -# Example - -```julia -template = ProblemTemplate(CopperPlatePowerModel, devices, branches, services) -problem = EmulationModel(MyEmProblemType, template, system, optimizer) -``` -""" -function EmulationModel( - ::Type{M}, - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) where {M <: EmulationProblem} - return EmulationModel{M}(template, sys, jump_model; kwargs...) -end - -function EmulationModel( - template::AbstractProblemTemplate, - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) - return EmulationModel{GenericEmulationProblem}(template, sys, jump_model; kwargs...) -end - -""" -Builds an empty emulation model. This constructor is used for the implementation of custom -emulation models that do not require a template. - -# Arguments - - - `::Type{M} where M<:EmulationProblem`: The abstract operation model type - - `sys::IS.InfrastructureSystemsContainer`: the system created using Power Systems - - `jump_model::Union{Nothing, JuMP.Model}` = nothing: Enables passing a custom JuMP model. Use with care. - -# Example - -```julia -problem = EmulationModel(system, optimizer) -``` -""" -function EmulationModel{M}( - sys::IS.InfrastructureSystemsContainer, - jump_model::Union{Nothing, JuMP.Model} = nothing; - kwargs..., -) where {M <: EmulationProblem} - return EmulationModel{M}(template, sys, jump_model; kwargs...) -end - -get_problem_type(::EmulationModel{M}) where {M <: EmulationProblem} = M - -function validate_template(::EmulationModel{M}) where {M <: EmulationProblem} - error("validate_template is not implemented for EmulationModel{$M}") -end - -function validate_template(::EmulationModel{<:DefaultEmulationProblem}) - return nothing -end - -function validate_time_series!(model::EmulationModel{<:DefaultEmulationProblem}) - sys = get_system(model) - settings = get_settings(model) - available_resolutions = get_time_series_resolutions(sys) - - if get_resolution(settings) == UNSET_RESOLUTION && length(available_resolutions) != 1 - throw( - IS.ConflictingInputsError( - "Data contains multiple resolutions, the resolution keyword argument must be added to the Model. Time Series Resolutions: $(available_resolutions)", - ), - ) - elseif get_resolution(settings) != UNSET_RESOLUTION && length(available_resolutions) > 1 - if get_resolution(settings) ∉ available_resolutions - throw( - IS.ConflictingInputsError( - "Resolution $(get_resolution(settings)) is not available in the system data. Time Series Resolutions: $(available_resolutions)", - ), - ) - end - else - set_resolution!(settings, first(available_resolutions)) - end - - if get_horizon(settings) == UNSET_HORIZON - # Emulation Models Only solve one "step" so Horizon and Resolution must match - set_horizon!(settings, get_resolution(settings)) - end - - counts = get_time_series_counts(sys) - if counts.static_time_series_count < 1 - error( - "The system does not contain Static Time Series data. A EmulationModel can't be built.", - ) - end - return -end - -function get_current_time(model::EmulationModel) - execution_count = get_execution_count(model) - initial_time = get_initial_time(model) - resolution = get_resolution(model) - return initial_time + resolution * execution_count -end - -function init_model_store_params!(model::EmulationModel) - num_executions = get_executions(model) - system = get_system(model) - settings = get_settings(model) - horizon = interval = resolution = get_resolution(settings) - base_power = get_base_power(system) - sys_uuid = get_system_uuid(system) - set_store_params!( - get_internal(model), - ModelStoreParams( - num_executions, - horizon, - interval, - resolution, - base_power, - sys_uuid, - get_metadata(get_optimization_container(model)), - ), - ) - return -end - -function update_parameters!( - model::EmulationModel, - store::EmulationModelStore{InMemoryDataset}, -) - update_parameters!(model, store.data_container) - return -end - -function update_parameters!(model::EmulationModel, data::DatasetContainer{InMemoryDataset}) - cost_function_unsynch(get_optimization_container(model)) - for key in keys(get_parameters(model)) - update_parameter_values!(model, key, data) - end - if !is_synchronized(model) - update_objective_function!(get_optimization_container(model)) - obj_func = get_objective_expression(get_optimization_container(model)) - set_synchronized_status!(obj_func, true) - end - return -end - -function update_model!( - model::EmulationModel, - source::EmulationModelStore{InMemoryDataset}, - ini_cond_chronology, -) - TimerOutputs.@timeit RUN_SIMULATION_TIMER "Parameter Updates" begin - update_parameters!(model, source) - end - TimerOutputs.@timeit RUN_SIMULATION_TIMER "Ini Cond Updates" begin - update_initial_conditions!(model, source, ini_cond_chronology) - end - return -end - -""" -Standalone update for EmulationModel (non-simulation context). -Updates parameters and initial conditions from the model's own store. -""" -function update_model!(model::EmulationModel) - source = get_store(model) - TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Parameter Updates" begin - update_parameters!(model, source) - end - TimerOutputs.@timeit RUN_OPERATION_MODEL_TIMER "Ini Cond Updates" begin - for key in keys(get_initial_conditions(model)) - update_initial_conditions!(model, key, source) - end - end - return -end - -""" -Update parameter function an OperationModel -""" -function update_parameter_values!( - model::EmulationModel, - key::ParameterKey{T, U}, - input::DatasetContainer{InMemoryDataset}, -) where {T <: ParameterType, U <: IS.InfrastructureSystemsComponent} - # Enable again for detailed debugging - # TimerOutputs.@timeit RUN_SIMULATION_TIMER "$T $U Parameter Update" begin - optimization_container = get_optimization_container(model) - # FIXME: This parameter update logic belongs in POM or PSI, not IOM. - # Move this function (and the surrounding update chain) once EmulationModel - # lifecycle code is fully migrated. - update_container_parameter_values!(optimization_container, model, key, input) - parameter_attributes = get_parameter_attributes(optimization_container, key) - IS.@record :execution ParameterUpdateEvent( - T, - U, - "event", # parameter_attributes, - get_current_timestamp(model), - get_name(model), - ) - #end - return -end diff --git a/src/operation/emulation_model_store.jl b/src/operation/emulation_model_store.jl deleted file mode 100644 index 8dd0bd4c..00000000 --- a/src/operation/emulation_model_store.jl +++ /dev/null @@ -1,200 +0,0 @@ -""" -Stores outputs data for one EmulationModel. -Parameterized by `T <: AbstractDataset` to support different storage backends -(e.g., `InMemoryDataset`, `HDF5Dataset`). -""" -mutable struct EmulationModelStore{T <: AbstractDataset} <: AbstractModelStore - data_container::DatasetContainer{T} - optimizer_stats::OrderedDict{Int, OptimizerStats} -end - -get_data_field(store::EmulationModelStore, ::Val{S}) where {S} = - getfield(store.data_container, S) -@inline Base.@constprop :aggressive get_data_field( - store::EmulationModelStore, - type::Symbol, -) = - get_data_field(store, Val(type)) - -function EmulationModelStore() - return EmulationModelStore( - DatasetContainer{InMemoryDataset}(), - OrderedDict{Int, OptimizerStats}(), - ) -end - -""" - Base.empty!(store::EmulationModelStore) - -Empty the [`EmulationModelStore`](@ref) -""" -function Base.empty!(store::EmulationModelStore) - stype = DatasetContainer - for (name, _) in zip(fieldnames(stype), fieldtypes(stype)) - if name ∉ [:values, :timestamps] - val = get_data_field(store, name) - try - empty!(val) - catch - @error "Base.empty! must be customized for type $stype or skipped" - rethrow() - end - elseif name == :update_timestamp - store.update_timestamp = UNSET_INI_TIME - else - setfield!( - store.data_container, - name, - zero(fieldtype(store.data_container, name)), - ) - end - end - empty!(store.optimizer_stats) - return -end - -function Base.isempty(store::EmulationModelStore) - stype = DatasetContainer - for (name, type) in zip(fieldnames(stype), fieldtypes(stype)) - if name ∉ [:values, :timestamps] - val = get_data_field(store, name) - try - !isempty(val) && return false - catch - @error "Base.isempty must be customized for type $stype or skipped" - rethrow() - end - elseif name == :update_timestamp - store.update_timestamp != UNSET_INI_TIME && return false - else - val = get_data_field(store, name) - iszero(val) && return false - end - end - return isempty(store.optimizer_stats) -end - -function initialize_storage!( - store::EmulationModelStore{InMemoryDataset}, - container::OptimizationContainer, - params::ModelStoreParams, -) - num_of_executions = get_num_executions(params) - for type in STORE_CONTAINERS - field_containers = getfield(container, type) - outputs_container = get_data_field(store, type) - for (key, field_container) in field_containers - @debug "Adding $(encode_key_as_string(key)) to EmulationModelStore" _group = - LOG_GROUP_MODEL_STORE - column_names = get_column_names(container, type, field_container, key) - outputs_container[key] = InMemoryDataset( - fill!( - DenseAxisArray{Float64}(undef, column_names..., 1:num_of_executions), - NaN, - ), - ) - end - end - return -end - -function write_output!( - store::EmulationModelStore, - name::Symbol, - key::OptimizationContainerKey, - index::EmulationModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{Float64, 2}, -) - if size(array, 2) == 1 - write_output!(store, name, key, index, update_timestamp, array[:, 1]) - else - container = get_data_field(store, get_store_container_type(key)) - set_value!( - container[key], - array, - index, - ) - set_last_recorded_row!(container[key], index) - set_update_timestamp!(container[key], update_timestamp) - end - return -end - -function write_output!( - store::EmulationModelStore, - ::Symbol, - key::OptimizationContainerKey, - index::EmulationModelIndexType, - update_timestamp::Dates.DateTime, - array::DenseAxisArray{Float64, 1}, -) - container = get_data_field(store, get_store_container_type(key)) - set_value!( - container[key], - array, - index, - ) - set_last_recorded_row!(container[key], index) - set_update_timestamp!(container[key], update_timestamp) - return -end - -function read_outputs( - store::EmulationModelStore{InMemoryDataset}, - key::OptimizationContainerKey; - index::Union{Int, Nothing} = nothing, - len::Union{Int, Nothing} = nothing, -) - container = get_data_field(store, get_store_container_type(key)) - data = container[key].values - # Return a copy because callers may mutate it. - if isnothing(index) - @assert_op len === nothing - return data[:, :] - elseif isnothing(len) - return data[:, index:end] - else - return data[:, index:(index + len - 1)] - end -end - -function get_column_names( - store::EmulationModelStore{InMemoryDataset}, - key::OptimizationContainerKey, -) - container = get_data_field(store, get_store_container_type(key)) - return get_column_names_from_axis_array(key, container[key].values) -end - -function get_dataset_size(store::EmulationModelStore, key::OptimizationContainerKey) - container = get_data_field(store, get_store_container_type(key)) - return size(container[key].values) -end - -function get_last_updated_timestamp( - store::EmulationModelStore, - key::OptimizationContainerKey, -) - container = get_data_field(store, get_store_container_type(key)) - return get_update_timestamp(container[key]) -end -function write_optimizer_stats!( - store::EmulationModelStore, - stats::OptimizerStats, - index::EmulationModelIndexType, -) - @assert !(index in keys(store.optimizer_stats)) - store.optimizer_stats[index] = stats - return -end - -function read_optimizer_stats(store::EmulationModelStore) - return DataFrames.DataFrame([ - IS.to_namedtuple(x) for x in values(store.optimizer_stats) - ]) -end - -function get_last_recorded_row(x::EmulationModelStore, key::OptimizationContainerKey) - return get_last_recorded_row(x.data_container, key) -end diff --git a/src/operation/initial_conditions_update_in_memory_store.jl b/src/operation/initial_conditions_update_in_memory_store.jl deleted file mode 100644 index 0b00970d..00000000 --- a/src/operation/initial_conditions_update_in_memory_store.jl +++ /dev/null @@ -1,49 +0,0 @@ - -################## ic updates from store for emulation problems simulation ################# - -""" - update_initial_conditions!(model, key, source) - -Update initial conditions for a specific key from the model store. -Dispatches to the per-IC-type `update_initial_conditions!(ics, store, resolution)` method. -""" -function update_initial_conditions!( - model::OperationModel, - key::InitialConditionKey{T, U}, - source, -) where {T <: InitialConditionType, U <: IS.InfrastructureSystemsComponent} - if get_execution_count(model) < 1 - return - end - container = get_optimization_container(model) - model_resolution = get_resolution(get_store_params(model)) - ini_conditions_vector = get_initial_condition(container, key) - update_initial_conditions!(ini_conditions_vector, source, model_resolution) - return -end - -""" - update_initial_conditions!(ics, store, resolution) - -Update initial conditions from the emulation model store. -This is an extension point - concrete implementations for specific initial condition -types should be defined in PowerOperationsModels. - -# Arguments -- `ics`: Vector of InitialCondition objects to update -- `store`: EmulationModelStore containing the recorded values -- `resolution`: Time resolution (Dates.Millisecond) -""" -function update_initial_conditions!( - ics::Vector{<:InitialCondition}, - store::EmulationModelStore{InMemoryDataset}, - resolution::Dates.Millisecond, -) - # This is a stub - concrete implementations for specific initial condition types - # (InitialTimeDurationOn, InitialTimeDurationOff, DevicePower, DeviceStatus, etc.) - # should be defined in PowerOperationsModels. - error( - "update_initial_conditions! not implemented for initial condition type " * - "$(eltype(ics)). Implement this method in PowerOperationsModels.", - ) -end diff --git a/src/operation/model_numerical_analysis_utils.jl b/src/operation/model_numerical_analysis_utils.jl deleted file mode 100644 index b27324a9..00000000 --- a/src/operation/model_numerical_analysis_utils.jl +++ /dev/null @@ -1,152 +0,0 @@ -# The Numerical stability checks code in this file is based on the code from the SDDP.jl package, -# from the below mentioned commit and file. -# commit :8cd305188caffc50a1734913053fc81bba613778 -# link to file :https://github.com/odow/SDDP.jl/blob/d353fe5a2903421e7fed6d609eb9377c35d715a1/src/print.jl#L190 - -mutable struct NumericalBounds - min::Float64 - max::Float64 - min_index::Any - max_index::Any -end - -NumericalBounds() = NumericalBounds(Inf, -Inf, nothing, nothing) - -set_min!(v::NumericalBounds, value::Real) = v.min = value -set_max!(v::NumericalBounds, value::Real) = v.max = value -set_min_index!(v::NumericalBounds, idx) = v.min_index = idx -set_max_index!(v::NumericalBounds, idx) = v.max_index = idx - -mutable struct ConstraintBounds - coefficient::NumericalBounds - rhs::NumericalBounds - function ConstraintBounds() - return new(NumericalBounds(), NumericalBounds()) - end -end - -function update_coefficient_bounds( - v::ConstraintBounds, - constraint::JuMP.ScalarConstraint, - idx, -) - update_numerical_bounds(v.coefficient, constraint.func, idx) - return -end - -function update_rhs_bounds(v::ConstraintBounds, constraint::JuMP.ScalarConstraint, idx) - update_numerical_bounds(v.rhs, constraint.set, idx) - return -end - -mutable struct VariableBounds - bounds::NumericalBounds - function VariableBounds() - return new(NumericalBounds()) - end -end - -function update_variable_bounds(v::VariableBounds, variable::JuMP.VariableRef, idx) - if JuMP.is_binary(variable) - set_min!(v.bounds, 0.0) - update_numerical_bounds(v.bounds, 1.0, idx) - else - if JuMP.has_lower_bound(variable) - update_numerical_bounds(v.bounds, JuMP.lower_bound(variable), idx) - end - if JuMP.has_upper_bound(variable) - update_numerical_bounds(v.bounds, JuMP.upper_bound(variable), idx) - end - end - return -end - -function update_numerical_bounds(v::NumericalBounds, value::Real, idx) - if !isapprox(value, 0.0) - if v.min > abs(value) - set_min!(v, value) - set_min_index!(v, idx) - elseif v.max < abs(value) - set_max!(v, value) - set_max_index!(v, idx) - end - end - return -end - -function update_numerical_bounds(bonuds::NumericalBounds, func::JuMP.GenericAffExpr, idx) - for coefficient in values(func.terms) - update_numerical_bounds(bonuds, coefficient, idx) - end - return -end - -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.LessThan, idx) - return update_numerical_bounds(bonuds, func.upper, idx) -end - -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.GreaterThan, idx) - return update_numerical_bounds(bonuds, func.lower, idx) -end - -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.EqualTo, idx) - return update_numerical_bounds(bonuds, func.value, idx) -end - -function update_numerical_bounds(bonuds::NumericalBounds, func::MOI.Interval, idx) - update_numerical_bounds(bonuds, func.upper, idx) - return update_numerical_bounds(bonuds, func.lower, idx) -end - -# Default fallbacks for unsupported constraints. -update_numerical_bounds(::NumericalBounds, func, idx) = nothing -update_coefficient_bounds(::ConstraintBounds, func, idx) = nothing -update_rhs_bounds(::ConstraintBounds, func, idx) = nothing - -function get_constraint_numerical_bounds(model::OperationModel) - if !is_built(model) - error("Model not built, can't calculate constraint numerical bounds") - end - bounds = ConstraintBounds() - for (const_key, constraint_array) in get_constraints(get_optimization_container(model)) - # TODO: handle this at compile and not at run time - if isa(constraint_array, SparseAxisArray) - for idx in eachindex(constraint_array) - constraint_array[idx] == 0.0 && continue - con_obj = JuMP.constraint_object(constraint_array[idx]) - update_coefficient_bounds(bounds, con_obj, (const_key, idx)) - update_rhs_bounds(bounds, con_obj, (const_key, idx)) - end - else - for idx in Iterators.product(constraint_array.axes...) - !isassigned(constraint_array, idx...) && continue - con_obj = JuMP.constraint_object(constraint_array[idx...]) - update_coefficient_bounds(bounds, con_obj, (const_key, idx)) - update_rhs_bounds(bounds, con_obj, (const_key, idx)) - end - end - end - return bounds -end - -function get_variable_numerical_bounds(model::OperationModel) - if !is_built(model) - error("Model not built, can't calculate variable numerical bounds") - end - bounds = VariableBounds() - for (variable_key, variable_array) in get_variables(get_optimization_container(model)) - if isa(variable_array, SparseAxisArray) - for idx in eachindex(variable_array) - var = variable_array[idx] - var == 0.0 && continue - update_variable_bounds(bounds, var, (variable_key, idx)) - end - else - for idx in Iterators.product(variable_array.axes...) - var = variable_array[idx...] - update_variable_bounds(bounds, var, (variable_key, idx)) - end - end - end - return bounds -end diff --git a/src/operation/operation_model_interface.jl b/src/operation/operation_model_interface.jl index 20f50e5c..017affa3 100644 --- a/src/operation/operation_model_interface.jl +++ b/src/operation/operation_model_interface.jl @@ -1,4 +1,10 @@ # Default implementations of getter/setter functions for OperationModel. +# Methods that compose primitives accessible only via concrete (POM-owned) +# struct fields — _check_numerical_bounds, _pre_solve_model_checks, _read_outputs, +# read_dual/read_variable/etc., list_keys-driven helpers, and +# read_optimizer_stats(::OperationModel) — live in +# PowerOperationsModels/operation/operation_model_glue.jl. + is_built(model::OperationModel) = get_status(get_internal(model)) == ModelBuildStatus.BUILT isempty(model::OperationModel) = @@ -200,83 +206,6 @@ function advance_execution_count!(model::OperationModel) return end -function _check_numerical_bounds(model::OperationModel) - variable_bounds = get_variable_numerical_bounds(model) - if variable_bounds.bounds.max - variable_bounds.bounds.min > 1e9 - @warn "Variable bounds range is $(variable_bounds.bounds.max - variable_bounds.bounds.min) and can result in numerical problems for the solver. \\ - max_bound_variable = $(encode_key_as_string(variable_bounds.bounds.max_index)) \\ - min_bound_variable = $(encode_key_as_string(variable_bounds.bounds.min_index)) \\ - Run get_detailed_variable_numerical_bounds on the model for a deeper analysis" - else - @info "Variable bounds range is [$(variable_bounds.bounds.min) $(variable_bounds.bounds.max)]" - end - - constraint_bounds = get_constraint_numerical_bounds(model) - if constraint_bounds.coefficient.max - constraint_bounds.coefficient.min > 1e9 - @warn "Constraint coefficient bounds range is $(constraint_bounds.coefficient.max - constraint_bounds.coefficient.min) and can result in numerical problems for the solver. \\ - max_bound_constraint = $(encode_key_as_string(constraint_bounds.coefficient.max_index)) \\ - min_bound_constraint = $(encode_key_as_string(constraint_bounds.coefficient.min_index)) \\ - Run get_detailed_constraint_numerical_bounds on the model for a deeper analysis" - else - @info "Constraint coefficient bounds range is [$(constraint_bounds.coefficient.min) $(constraint_bounds.coefficient.max)]" - end - - if constraint_bounds.rhs.max - constraint_bounds.rhs.min > 1e9 - @warn "Constraint right-hand-side bounds range is $(constraint_bounds.rhs.max - constraint_bounds.rhs.min) and can result in numerical problems for the solver. \\ - max_bound_constraint = $(encode_key_as_string(constraint_bounds.rhs.max_index)) \\ - min_bound_constraint = $(encode_key_as_string(constraint_bounds.rhs.min_index)) \\ - Run get_detailed_constraint_numerical_bounds on the model for a deeper analysis" - else - @info "Constraint right-hand-side bounds [$(constraint_bounds.rhs.min) $(constraint_bounds.rhs.max)]" - end - return -end - -function _pre_solve_model_checks(model::OperationModel, optimizer = nothing) - jump_model = get_jump_model(model) - if optimizer !== nothing - JuMP.set_optimizer(jump_model, optimizer) - end - - if JuMP.mode(jump_model) != JuMP.DIRECT - if JuMP.backend(jump_model).state == MOIU.NO_OPTIMIZER - error("No Optimizer has been defined, can't solve the operational problem") - end - else - @assert get_direct_mode_optimizer(get_settings(model)) - end - - optimizer_name = JuMP.solver_name(jump_model) - @info "$(get_name(model)) optimizer set to: $optimizer_name" - settings = get_settings(model) - if get_check_numerical_bounds(settings) - @info "Checking Numerical Bounds" - TimerOutputs.@timeit BUILD_PROBLEMS_TIMER "Numerical Bounds Check" begin - _check_numerical_bounds(model) - end - end - return -end - -function list_names(model::OperationModel, ::Type{T}) where {T <: OptimizationKeyType} - return encode_keys_as_strings( - list_keys(get_store(model), T), - ) -end - -read_dual(model::OperationModel, key::ConstraintKey) = _read_outputs(model, key) -read_parameter(model::OperationModel, key::ParameterKey) = _read_outputs(model, key) -read_aux_variable(model::OperationModel, key::AuxVarKey) = _read_outputs(model, key) -read_variable(model::OperationModel, key::VariableKey) = _read_outputs(model, key) -read_expression(model::OperationModel, key::ExpressionKey) = _read_outputs(model, key) - -function _read_outputs(model::OperationModel, key::OptimizationContainerKey) - array = read_outputs(get_store(model), key) - return to_outputs_dataframe(array, nothing, Val(TableFormat.LONG)) -end - -read_optimizer_stats(model::OperationModel) = read_optimizer_stats(get_store(model)) - function add_recorders!(model::OperationModel, recorders) internal = get_internal(model) for name in recorders @@ -322,23 +251,6 @@ function instantiate_network_model!(model::OperationModel) return end -list_aux_variable_keys(x::OperationModel) = list_keys(get_store(x), AuxVariableType) -list_aux_variable_names(x::OperationModel) = list_names(x, AuxVariableType) -list_variable_keys(x::OperationModel) = list_keys(get_store(x), VariableType) -list_variable_names(x::OperationModel) = list_names(x, VariableType) -list_parameter_keys(x::OperationModel) = list_keys(get_store(x), ParameterType) -list_parameter_names(x::OperationModel) = list_names(x, ParameterType) -list_dual_keys(x::OperationModel) = list_keys(get_store(x), ConstraintType) -list_dual_names(x::OperationModel) = list_names(x, ConstraintType) -list_expression_keys(x::OperationModel) = list_keys(get_store(x), ExpressionType) -list_expression_names(x::OperationModel) = list_names(x, ExpressionType) - -function list_all_keys(x::OperationModel) - return Iterators.flatten( - list_fields(get_store(x), T) for T in STORE_CONTAINER_TYPES - ) -end - function serialize_optimization_model(model::OperationModel, save_path::String) serialize_jump_optimization_model( get_jump_model(get_optimization_container(model)), diff --git a/src/operation/optimization_debugging.jl b/src/operation/optimization_debugging.jl deleted file mode 100644 index b67d7b1e..00000000 --- a/src/operation/optimization_debugging.jl +++ /dev/null @@ -1,110 +0,0 @@ -""" -Each Tuple corresponds to (con_name, internal_index, moi_index) -""" -function get_all_constraint_index(model::OperationModel) - con_index = Vector{Tuple{ConstraintKey, Int, Int}}() - container = get_optimization_container(model) - for (key, value) in get_constraints(container) - for (idx, constraint) in enumerate(value) - moi_index = JuMP.optimizer_index(constraint) - push!(con_index, (key, idx, moi_index.value)) - end - end - return con_index -end - -""" -Each Tuple corresponds to (con_name, internal_index, moi_index) -""" -function get_all_variable_index(model::OperationModel) - var_keys = get_all_variable_keys(model) - return [(encode_key(v[1]), v[2], v[3]) for v in var_keys] -end - -function get_all_variable_keys(model::OperationModel) - var_index = Vector{Tuple{VariableKey, Int, Int}}() - container = get_optimization_container(model) - for (key, value) in get_variables(container) - for (idx, variable) in enumerate(value) - moi_index = JuMP.optimizer_index(variable) - push!(var_index, (key, idx, moi_index.value)) - end - end - return var_index -end - -function get_constraint_index(model::OperationModel, index::Int) - container = get_optimization_container(model) - constraints = get_constraints(container) - for i in get_all_constraint_index(model) - if i[3] == index - return constraints[i[1]].data[i[2]] - end - end - @info "Index not found" - return -end - -function get_variable_index(model::OperationModel, index::Int) - container = get_optimization_container(model) - variables = get_variables(container) - for i in get_all_variable_keys(model) - if i[3] == index - return variables[i[1]].data[i[2]] - end - end - @info "Index not found" - return -end - -function get_detailed_constraint_numerical_bounds(model::OperationModel) - if !is_built(model) - error("Model not built, can't calculate constraint numerical bounds") - end - constraint_bounds = Dict() - for (const_key, constraint_array) in get_constraints(get_optimization_container(model)) - if isa(constraint_array, SparseAxisArray) - bounds = ConstraintBounds() - for idx in eachindex(constraint_array) - constraint_array[idx] == 0.0 && continue - con_obj = JuMP.constraint_object(constraint_array[idx]) - update_coefficient_bounds(bounds, con_obj, idx) - update_rhs_bounds(bounds, con_obj, idx) - end - constraint_bounds[const_key] = bounds - else - bounds = ConstraintBounds() - for idx in Iterators.product(constraint_array.axes...) - con_obj = JuMP.constraint_object(constraint_array[idx...]) - update_coefficient_bounds(bounds, con_obj, idx) - update_rhs_bounds(bounds, con_obj, idx) - end - constraint_bounds[const_key] = bounds - end - end - return constraint_bounds -end - -function get_detailed_variable_numerical_bounds(model::OperationModel) - if !is_built(model) - error("Model not built, can't calculate variable numerical bounds") - end - variable_bounds = Dict() - for (variable_key, variable_array) in get_variables(get_optimization_container(model)) - bounds = VariableBounds() - if isa(variable_array, SparseAxisArray) - for idx in eachindex(variable_array) - var = variable_array[idx] - var == 0.0 && continue - update_variable_bounds(bounds, var, idx) - end - else - for idx in Iterators.product(variable_array.axes...) - var = variable_array[idx...] - update_variable_bounds(bounds, var, idx) - end - end - variable_bounds[variable_key] = bounds - end - return variable_bounds -end diff --git a/src/operation/problem_outputs.jl b/src/operation/problem_outputs.jl deleted file mode 100644 index c4a70d35..00000000 --- a/src/operation/problem_outputs.jl +++ /dev/null @@ -1,88 +0,0 @@ -""" -Construct OptimizationProblemOutputs from a solved DecisionModel. -""" -function OptimizationProblemOutputs(model::DecisionModel) - status = get_run_status(model) - status != RunStatus.SUCCESSFULLY_FINALIZED && - error("problem was not solved successfully: $status") - - model_store = get_store(model) - - if isempty(model_store) - error("Model Solved as part of a Simulation.") - end - - timestamps = get_timestamps(model) - optimizer_stats = to_dataframe(get_optimizer_stats(model)) - - aux_variable_values = - Dict(x => read_aux_variable(model, x) for x in list_aux_variable_keys(model)) - variable_values = Dict(x => read_variable(model, x) for x in list_variable_keys(model)) - dual_values = Dict(x => read_dual(model, x) for x in list_dual_keys(model)) - parameter_values = - Dict(x => read_parameter(model, x) for x in list_parameter_keys(model)) - expression_values = - Dict(x => read_expression(model, x) for x in list_expression_keys(model)) - - sys = get_system(model) - - return OptimizationProblemOutputs( - get_problem_base_power(model), - timestamps, - sys, - get_uuid(sys), - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - get_metadata(get_optimization_container(model)), - IS.strip_module_name(typeof(model)), - get_output_dir(model), - mkpath(joinpath(get_output_dir(model), "outputs")), - ) -end - -""" -Construct OptimizationProblemOutputs from a solved EmulationModel. -""" -function OptimizationProblemOutputs(model::EmulationModel) - status = get_run_status(model) - status != RunStatus.SUCCESSFULLY_FINALIZED && - error("problem was not solved successfully: $status") - - model_store = get_store(model) - - if isempty(model_store) - error("Model Solved as part of a Simulation.") - end - - aux_variables = - Dict(x => read_aux_variable(model, x) for x in list_aux_variable_keys(model)) - variables = Dict(x => read_variable(model, x) for x in list_variable_keys(model)) - duals = Dict(x => read_dual(model, x) for x in list_dual_keys(model)) - parameters = Dict(x => read_parameter(model, x) for x in list_parameter_keys(model)) - expression = Dict(x => read_expression(model, x) for x in list_expression_keys(model)) - optimizer_stats = read_optimizer_stats(model) - initial_time = get_initial_time(model) - container = get_optimization_container(model) - sys = get_system(model) - - return OptimizationProblemOutputs( - get_problem_base_power(model), - StepRange(initial_time, get_resolution(model), initial_time), - sys, - get_uuid(sys), - aux_variables, - variables, - duals, - parameters, - expression, - optimizer_stats, - get_metadata(container), - IS.strip_module_name(typeof(model)), - get_output_dir(model), - mkpath(joinpath(get_output_dir(model), "outputs")), - ) -end diff --git a/src/operation/store_common.jl b/src/operation/store_common.jl deleted file mode 100644 index eb09754f..00000000 --- a/src/operation/store_common.jl +++ /dev/null @@ -1,236 +0,0 @@ -"""Typed container for export configuration parameters used during model output writing.""" -struct ExportParameters{E} - exports::E - exports_path::String - file_type::Type - resolution::Dates.Millisecond - horizon_count::Int -end - -function _export_container_output!( - export_params::ExportParameters, - exports_path, - key, - index, - data, -) - df = to_dataframe(data, key) - time_col = - range(index; length = export_params.horizon_count, step = export_params.resolution) - DataFrames.insertcols!(df, 1, :DateTime => time_col) - ISOPT.export_output(export_params.file_type, exports_path, key, index, df) - return -end - -"""Sanitize a model name for use as a filesystem path component. -Replaces path separators, null bytes, and control characters with underscores.""" -function _sanitize_model_name(name::AbstractString) - _is_path_safe(c::AbstractChar) = - isprint(c) && c ∉ ('/', '\\', ':', '*', '?', '"', '<', '>', '|') - sanitized = map(c -> _is_path_safe(c) ? c : '_', name) - if isempty(sanitized) || sanitized == "." || sanitized == ".." - throw( - IS.InvalidValue("Model name '$name' is not valid for use as a path component"), - ) - end - return sanitized -end - -# Aliases used for clarity in the method dispatches so it is possible to know if writing to -# DecisionModel data or EmulationModel data -# Note: DecisionModelIndexType and EmulationModelIndexType are defined in core/definitions.jl - -function write_outputs!( - store::AbstractModelStore, - model::OperationModel, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime; - exports = nothing, -) - if exports !== nothing - export_params = ExportParameters( - exports, - joinpath(exports.path, _sanitize_model_name(string(get_name(model)))), - get_export_file_type(exports), - get_resolution(model), - get_horizon(get_settings(model)) ÷ get_resolution(model), - ) - else - export_params = nothing - end - - write_model_dual_outputs!(store, model, index, update_timestamp, export_params) - write_model_parameter_outputs!(store, model, index, update_timestamp, export_params) - write_model_variable_outputs!(store, model, index, update_timestamp, export_params) - write_model_aux_variable_outputs!(store, model, index, update_timestamp, export_params) - write_model_expression_outputs!(store, model, index, update_timestamp, export_params) - return -end - -function write_model_dual_outputs!( - store, - model::T, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime, - export_params::Union{ExportParameters, Nothing}, -) where {T <: OperationModel} - container = get_optimization_container(model) - model_name = get_name(model) - if export_params !== nothing - exports_path = joinpath(export_params.exports_path, "duals") - mkpath(exports_path) - end - - for (key, constraint) in get_duals(container) - !should_write_resulting_value(key) && continue - data = jump_value.(constraint) - write_output!(store, model_name, key, index, update_timestamp, data) - - if export_params !== nothing && - should_export_dual(export_params.exports, update_timestamp, model_name, key) - _export_container_output!(export_params, exports_path, key, index, data) - end - end - return -end - -function write_model_parameter_outputs!( - store, - model::T, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime, - export_params::Union{ExportParameters, Nothing}, -) where {T <: OperationModel} - container = get_optimization_container(model) - model_name = get_name(model) - if export_params !== nothing - exports_path = joinpath(export_params.exports_path, "parameters") - mkpath(exports_path) - end - - parameters = get_parameters(container) - for (key, param_container) in parameters - !should_write_resulting_value(key) && continue - data = calculate_parameter_values(param_container) - write_output!(store, model_name, key, index, update_timestamp, data) - - if export_params !== nothing && - should_export_parameter( - export_params.exports, - update_timestamp, - model_name, - key, - ) - _export_container_output!(export_params, exports_path, key, index, data) - end - end - return -end - -function write_model_variable_outputs!( - store, - model::T, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime, - export_params::Union{ExportParameters, Nothing}, -) where {T <: OperationModel} - container = get_optimization_container(model) - model_name = get_name(model) - if export_params !== nothing - exports_path = joinpath(export_params.exports_path, "variables") - mkpath(exports_path) - end - - if !isempty(container.primal_values_cache) - variables = container.primal_values_cache.variables_cache - else - variables = get_variables(container) - end - - for (key, variable) in variables - !should_write_resulting_value(key) && continue - data = jump_value.(variable) - write_output!(store, model_name, key, index, update_timestamp, data) - - if export_params !== nothing && - should_export_variable( - export_params.exports, - update_timestamp, - model_name, - key, - ) - _export_container_output!(export_params, exports_path, key, index, data) - end - end - return -end - -function write_model_aux_variable_outputs!( - store, - model::T, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime, - export_params::Union{ExportParameters, Nothing}, -) where {T <: OperationModel} - container = get_optimization_container(model) - model_name = get_name(model) - if export_params !== nothing - exports_path = joinpath(export_params.exports_path, "aux_variables") - mkpath(exports_path) - end - - for (key, variable) in get_aux_variables(container) - !should_write_resulting_value(key) && continue - data = jump_value.(variable) - write_output!(store, model_name, key, index, update_timestamp, data) - - if export_params !== nothing && - should_export_aux_variable( - export_params.exports, - update_timestamp, - model_name, - key, - ) - _export_container_output!(export_params, exports_path, key, index, data) - end - end - return -end - -function write_model_expression_outputs!( - store, - model::T, - index::Union{DecisionModelIndexType, EmulationModelIndexType}, - update_timestamp::Dates.DateTime, - export_params::Union{ExportParameters, Nothing}, -) where {T <: OperationModel} - container = get_optimization_container(model) - model_name = get_name(model) - if export_params !== nothing - exports_path = joinpath(export_params.exports_path, "expressions") - mkpath(exports_path) - end - - if !isempty(container.primal_values_cache) - expressions = container.primal_values_cache.expressions_cache - else - expressions = get_expressions(container) - end - - for (key, expression) in expressions - !should_write_resulting_value(key) && continue - data = jump_value.(expression) - write_output!(store, model_name, key, index, update_timestamp, data) - - if export_params !== nothing && - should_export_expression( - export_params.exports, - update_timestamp, - model_name, - key, - ) - _export_container_output!(export_params, exports_path, key, index, data) - end - end - return -end diff --git a/src/operation/time_series_interface.jl b/src/operation/time_series_interface.jl deleted file mode 100644 index fbc3b51f..00000000 --- a/src/operation/time_series_interface.jl +++ /dev/null @@ -1,93 +0,0 @@ -function get_time_series_values!( - time_series_type::Type{T}, - model::DecisionModel, - component, - name::String, - initial_time::Dates.DateTime, - horizon::Int; - ignore_scaling_factors = true, - interval::Dates.Millisecond = UNSET_INTERVAL, -) where {T <: IS.Forecast} - is_interval = _to_is_interval(interval) - settings = get_settings(model) - resolution = get_resolution(settings) - if !use_time_series_cache(settings) - return IS.get_time_series_values( - T, - component, - name; - start_time = initial_time, - len = horizon, - ignore_scaling_factors = ignore_scaling_factors, - interval = is_interval, - ) - end - - cache = get_time_series_cache(model) - key = IS.TimeSeriesCacheKey(IS.get_uuid(component), T, name, resolution, is_interval) - if haskey(cache, key) - ts_cache = cache[key] - else - ts_cache = IS.make_time_series_cache( - time_series_type, - component, - name, - initial_time, - horizon; - ignore_scaling_factors = ignore_scaling_factors, - interval = is_interval, - resolution = resolution, - ) - cache[key] = ts_cache - end - - ts = IS.get_time_series_array!(ts_cache, initial_time) - return TimeSeries.values(ts) -end - -function get_time_series_values!( - ::Type{T}, - model::EmulationModel, - component::U, - name::String, - initial_time::Dates.DateTime, - len::Int = 1; - ignore_scaling_factors = true, - resolution::Dates.Millisecond = UNSET_RESOLUTION, -) where {T <: IS.StaticTimeSeries, U <: IS.InfrastructureSystemsComponent} - settings = get_settings(model) - key_resolution = - resolution == UNSET_RESOLUTION ? get_resolution(settings) : resolution - is_resolution = _to_is_resolution(key_resolution) - if !use_time_series_cache(settings) - return IS.get_time_series_values( - T, - component, - name; - start_time = initial_time, - len = len, - ignore_scaling_factors = ignore_scaling_factors, - resolution = is_resolution, - ) - end - - cache = get_time_series_cache(model) - key = IS.TimeSeriesCacheKey(IS.get_uuid(component), T, name, key_resolution, nothing) - if haskey(cache, key) - ts_cache = cache[key] - else - ts_cache = IS.make_time_series_cache( - T, - component, - name, - initial_time, - len; - ignore_scaling_factors = ignore_scaling_factors, - resolution = is_resolution, - ) - cache[key] = ts_cache - end - - ts = IS.get_time_series_array!(ts_cache, initial_time) - return TimeSeries.values(ts) -end diff --git a/src/utils/print_pt_v3.jl b/src/utils/print_pt_v3.jl index 743bfaca..554a17d3 100644 --- a/src/utils/print_pt_v3.jl +++ b/src/utils/print_pt_v3.jl @@ -210,9 +210,6 @@ function Base.show(io::IO, ::MIME"text/html", input::SimulationModels) _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) end -_get_model_type(::DecisionModel{T}) where {T <: DecisionProblem} = T -_get_model_type(::EmulationModel{T}) where {T <: EmulationProblem} = T - function _show_method(io::IO, sim_models::SimulationModels, backend::Symbol; kwargs...) println(io) header = ["Model Name", "Model Type", "Status", "Output Directory"] @@ -220,7 +217,7 @@ function _show_method(io::IO, sim_models::SimulationModels, backend::Symbol; kwa table = Matrix{Any}(undef, length(sim_models.decision_models), length(header)) for (ix, model) in enumerate(sim_models.decision_models) table[ix, 1] = string(get_name(model)) - table[ix, 2] = IS.strip_module_name(string(_get_model_type(model))) + table[ix, 2] = IS.strip_module_name(string(get_problem_type(model))) table[ix, 3] = string(get_status(model)) table[ix, 4] = get_output_dir(model) end @@ -240,7 +237,7 @@ function _show_method(io::IO, sim_models::SimulationModels, backend::Symbol; kwa table = Matrix{Any}(undef, 1, length(header)) table[1, 1] = string(get_name(sim_models.emulation_model)) table[1, 2] = - IS.strip_module_name(string(_get_model_type(sim_models.emulation_model))) + IS.strip_module_name(string(get_problem_type(sim_models.emulation_model))) table[1, 3] = string(get_status(sim_models.emulation_model)) table[1, 4] = get_output_dir(sim_models.emulation_model) @@ -325,186 +322,7 @@ function _show_method(io::IO, sequence::SimulationSequence, backend::Symbol; kwa end end -function Base.show(io::IO, ::MIME"text/plain", input::Simulation) - _show_method(io, input, :auto) -end - -function Base.show(io::IO, ::MIME"text/html", input::Simulation) - # The tf_html_simple format was eliminated from PrettyTables and it was added to PowerSystems - _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) -end - -function _get_initial_time_for_show(sim::Simulation) - ini_time = get_initial_time(sim) - if isnothing(ini_time) - return "Unset Initial Time" - else - return string(ini_time) - end -end - -function _get_build_status_for_show(sim::Simulation) - internal = sim.internal - if isnothing(internal) - return "EMPTY" - else - return string(internal.build_status) - end -end - -function _get_run_status_for_show(sim::Simulation) - internal = sim.internal - if isnothing(internal) - return "NOT_READY" - else - return string(internal.status) - end -end - -function _show_method(io::IO, sim::Simulation, backend::Symbol; kwargs...) - table = [ - "Simulation Name" get_name(sim) - "Build Status" _get_build_status_for_show(sim) - "Run Status" _get_run_status_for_show(sim) - "Initial Time" _get_initial_time_for_show(sim) - "Steps" get_steps(sim) - ] - - PrettyTables.pretty_table( - io, - table; - backend = backend, - show_column_labels = false, - title = "Simulation", - alignment = :l, - kwargs..., - ) - - _show_method(io, sim.models, backend; kwargs...) - _show_method(io, sim.sequence, backend; kwargs...) -end - -function Base.show(io::IO, ::MIME"text/plain", input::SimulationOutputs) - _show_method(io, input, :auto) -end - -function Base.show(io::IO, ::MIME"text/html", input::SimulationOutputs) - # The tf_html_simple format was eliminated from PrettyTables and it was added to PowerSystems - _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) -end - -function _show_method(io::IO, outputs::SimulationOutputs, backend::Symbol; kwargs...) - header = ["Problem Name", "Initial Time", "Resolution", "Last Solution Timestamp"] - - table = Matrix{Any}(undef, length(outputs.decision_problem_outputs), length(header)) - for (ix, (key, output)) in enumerate(outputs.decision_problem_outputs) - table[ix, 1] = key - table[ix, 2] = first(output.timestamps) - table[ix, 3] = Dates.canonicalize(output.resolution) - table[ix, 4] = last(output.timestamps) - end - println(io) - PrettyTables.pretty_table( - io, - table; - column_labels = header, - backend = backend, - title = "Decision Problem Outputs", - alignment = :l, - ) - - println(io) - table = [ - "Name" outputs.emulation_problem_outputs.problem - "Resolution" Dates.Minute(outputs.emulation_problem_outputs.resolution) - "Number of steps" length(outputs.emulation_problem_outputs.timestamps) - ] - PrettyTables.pretty_table( - io, - table; - show_column_labels = false, - backend = backend, - title = "Emulator Outputs", - alignment = :l, - kwargs..., - ) -end - -ProblemOutputsTypes = Union{OptimizationProblemOutputs, SimulationProblemOutputs} -function Base.show(io::IO, ::MIME"text/plain", input::ProblemOutputsTypes) - _show_method(io, input, :auto) -end - -function Base.show(io::IO, ::MIME"text/html", input::ProblemOutputsTypes) - # The tf_html_simple format was eliminated from PrettyTables and it was added to PowerSystems - _show_method(io, input, :html; stand_alone = false, table_format = tf_html_simple) -end - -function _show_method( - io::IO, - outputs::T, - backend::Symbol; - kwargs..., -) where {T <: ProblemOutputsTypes} - timestamps = get_timestamps(outputs) - - if backend == :html - println(io, "

Start: $(first(timestamps))

") - println(io, "

End: $(last(timestamps))

") - println( - io, - "

Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))

", - ) - else - println(io, "Start: $(first(timestamps))") - println(io, "End: $(last(timestamps))") - println(io, "Resolution: $(Dates.Minute(ISOPT.get_resolution(outputs)))") - end - - values = Dict{String, Vector{String}}( - "Variables" => list_variable_names(outputs), - "Auxiliary variables" => list_aux_variable_names(outputs), - "Duals" => list_dual_names(outputs), - "Expressions" => list_expression_names(outputs), - "Parameters" => list_parameter_names(outputs), - ) - - if hasfield(T, :problem) - name = outputs.problem - else - name = "InfrastructureOptimizationModels" - end - - for (k, val) in values - if !isempty(val) - println(io) - PrettyTables.pretty_table( - io, - val; - show_column_labels = false, - backend = backend, - title = "$name Problem $k Outputs", - alignment = :l, - kwargs..., - ) - end - end -end - -function Base.show(io::IO, ::MIME"text/plain", bounds::ConstraintBounds) - println(io, "ConstraintBounds:") - println(io, "Constraint Coefficient") - show(io, MIME"text/plain"(), bounds.coefficient) - println(io, "Constraint RHS") - show(io, MIME"text/plain"(), bounds.rhs) -end - -function Base.show(io::IO, ::MIME"text/plain", bounds::VariableBounds) - println(io, "VariableBounds:") - show(io, MIME"text/plain"(), bounds.bounds) -end - -function Base.show(io::IO, ::MIME"text/plain", bounds::NumericalBounds) - println(io, rpad(" Minimum", 20), "Maximum") - println(io, rpad(" $(bounds.min)", 20), "$(bounds.max)") -end +# show methods for OptimizationProblemOutputs, ConstraintBounds, VariableBounds, +# NumericalBounds live with the concrete types in PowerOperationsModels. +# Simulation/SimulationOutputs/SimulationProblemOutputs show methods live in +# PowerSimulations along with their concrete types. diff --git a/test/InfrastructureOptimizationModelsTests.jl b/test/InfrastructureOptimizationModelsTests.jl index f218b685..87a7be75 100644 --- a/test/InfrastructureOptimizationModelsTests.jl +++ b/test/InfrastructureOptimizationModelsTests.jl @@ -106,8 +106,7 @@ function run_tests() include(joinpath(TEST_DIR, "test_optimization_container_metadata.jl")) # optimization_container_types.jl: no need for tests include(joinpath(TEST_DIR, "test_optimization_container.jl")) - # optimization_problem_outputs_export.jl: low-complexity - include(joinpath(TEST_DIR, "test_optimization_outputs.jl")) + # OptimizationProblemOutputs moved to POM — test lives in POM/test/test_optimization_outputs.jl include(joinpath(TEST_DIR, "test_optimizer_stats.jl")) # parameter_container.jl: low-complexity # TODO outputs_by_time.jl @@ -139,16 +138,8 @@ function run_tests() include(joinpath(TEST_DIR, "test_nmdt_approximations.jl")) end - #= - ============================================================================ - BROKEN/NEEDS-WORK TEST FILES (not included — rewrite with mocks) - ============================================================================ - - test_basic_model_structs.jl, test_model_decision.jl, test_model_emulation.jl: - previously used PSY/PowerModels types; rewrite with mocks before re-enabling. - - test_model_store.jl, test_offer_curve_cost.jl: PSY/PSB-backed integration tests - removed when IOM dropped PSY/PSB dependencies; rewrite with mocks. - ============================================================================ - =# + # NOTE: test_basic_model_structs.jl uses PSY/PowerModels types — rewrite with mocks before re-enabling. + # Decision/Emulation/store tests live in POM now (operation models moved out of IOM). end end diff --git a/test/Project.toml b/test/Project.toml index fed99d0f..0ec6c3d2 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -9,7 +9,6 @@ DataFramesMeta = "1313f7d8-7da2-5740-9ea0-a2ca25f37964" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" HiGHS = "87dc4568-4c63-4d18-b0c0-bb2238e4078b" -InfrastructureOptimizationModels = "bed98974-b02a-5e2f-9ee0-a103f5c45069" InfrastructureSystems = "2cd47ed4-ca9b-11e9-27f2-ab636a7671f1" Ipopt = "b6b21f68-93f8-5de0-b562-5493be1d77c9" JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1" @@ -30,7 +29,6 @@ UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" UnoSolver = "1baa60ac-02f7-4b39-a7a8-2f4f58486b05" [sources] -InfrastructureOptimizationModels = {path = ".."} InfrastructureSystems = {rev = "IS4", url = "https://github.com/Sienna-Platform/InfrastructureSystems.jl"} [compat] diff --git a/test/test_model_emulation.jl b/test/test_model_emulation.jl deleted file mode 100644 index 581162ea..00000000 --- a/test/test_model_emulation.jl +++ /dev/null @@ -1,343 +0,0 @@ -# FIXME not working and not included in the tests. integration of emulation models in -# POM-IOM split is a work in progress. -@testset "Emulation Model Build" begin - template = get_thermal_dispatch_template_network() - c_sys5 = PSB.build_system( - PSITestSystems, - "c_sys5_uc"; - add_single_time_series = true, - force_build = true, - ) - - model = EmulationModel(template, c_sys5; optimizer = HiGHS_optimizer) - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - template = get_thermal_standard_uc_template() - c_sys5_uc_re = PSB.build_system( - PSITestSystems, - "c_sys5_uc_re"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, RenewableDispatch, RenewableFullDispatch) - model = EmulationModel(template, c_sys5_uc_re; optimizer = HiGHS_optimizer) - - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - @test !isempty(collect(readdir(PSI.get_recorder_dir(model)))) -end - -@testset "Emulation Model initial_conditions test for ThermalGen" begin - ######## Test with ThermalStandardUnitCommitment ######## - template = get_thermal_standard_uc_template() - c_sys5_uc_re = PSB.build_system( - PSITestSystems, - "c_sys5_uc_re"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, RenewableDispatch, RenewableFullDispatch) - model = EmulationModel(template, c_sys5_uc_re; optimizer = HiGHS_optimizer) - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - check_duration_on_initial_conditions_values(model, ThermalStandard) - check_duration_off_initial_conditions_values(model, ThermalStandard) - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - ######## Test with ThermalMultiStartUnitCommitment ######## - template = get_thermal_standard_uc_template() - c_sys5_uc = PSB.build_system( - PSITestSystems, - "c_sys5_pglib"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, ThermalMultiStart, ThermalMultiStartUnitCommitment) - model = EmulationModel(template, c_sys5_uc; optimizer = HiGHS_optimizer) - @test build!(model; executions = 1, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - - check_duration_on_initial_conditions_values(model, ThermalStandard) - check_duration_off_initial_conditions_values(model, ThermalStandard) - check_duration_on_initial_conditions_values(model, ThermalMultiStart) - check_duration_off_initial_conditions_values(model, ThermalMultiStart) - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - ######## Test with ThermalStandardUnitCommitment ######## - template = get_thermal_standard_uc_template() - c_sys5_uc = PSB.build_system( - PSITestSystems, - "c_sys5_pglib"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, ThermalMultiStart, ThermalStandardUnitCommitment) - model = EmulationModel(template, c_sys5_uc; optimizer = HiGHS_optimizer) - @test build!(model; executions = 1, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - check_duration_on_initial_conditions_values(model, ThermalStandard) - check_duration_off_initial_conditions_values(model, ThermalStandard) - check_duration_on_initial_conditions_values(model, ThermalMultiStart) - check_duration_off_initial_conditions_values(model, ThermalMultiStart) - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - ######## Test with ThermalStandardDispatch ######## - template = get_thermal_standard_uc_template() - c_sys5_uc = PSB.build_system( - PSITestSystems, - "c_sys5_pglib"; - add_single_time_series = true, - force_build = true, - ) - device_model = DeviceModel(PSY.ThermalStandard, PSI.ThermalStandardDispatch) - set_device_model!(template, device_model) - model = EmulationModel(template, c_sys5_uc; optimizer = HiGHS_optimizer) - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT -end - -@testset "Emulation Model initial_conditions test for Hydro" begin - ######## Test with HydroDispatchRunOfRiver ######## - template = get_thermal_dispatch_template_network() - c_sys5_hyd = PSB.build_system( - PSITestSystems, - "c_sys5_hyd"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, HydroDispatch, HydroDispatchRunOfRiver) - set_device_model!(template, HydroTurbine, HydroTurbineEnergyDispatch) - set_device_model!(template, HydroReservoir, HydroEnergyModelReservoir) - model = EmulationModel(template, c_sys5_hyd; optimizer = HiGHS_optimizer) - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - initial_conditions_data = - PSI.get_initial_conditions_data(PSI.get_optimization_container(model)) - @test !PSI.has_initial_condition_value( - initial_conditions_data, - ActivePowerVariable, - HydroTurbine, - ) - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - ######## Test with HydroCommitmentRunOfRiver ######## - template = get_thermal_dispatch_template_network() - c_sys5_hyd = PSB.build_system( - PSITestSystems, - "c_sys5_hyd"; - add_single_time_series = true, - force_build = true, - ) - set_device_model!(template, HydroDispatch, HydroCommitmentRunOfRiver) - set_device_model!(template, HydroTurbine, HydroTurbineEnergyCommitment) - set_device_model!(template, HydroReservoir, HydroEnergyModelReservoir) - model = EmulationModel(template, c_sys5_hyd; optimizer = HiGHS_optimizer) - - @test build!(model; executions = 10, output_dir = mktempdir(; cleanup = true)) == - PSI.ModelBuildStatus.BUILT - initial_conditions_data = - PSI.get_initial_conditions_data(PSI.get_optimization_container(model)) - @test PSI.has_initial_condition_value( - initial_conditions_data, - OnVariable, - HydroTurbine, - ) - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED -end - -@testset "Emulation Model Outputs" begin - template = get_thermal_dispatch_template_network() - c_sys5 = PSB.build_system( - PSITestSystems, - "c_sys5_uc"; - add_single_time_series = true, - force_build = true, - ) - - model = EmulationModel(template, c_sys5; optimizer = HiGHS_optimizer) - executions = 10 - @test build!( - model; - executions = executions, - output_dir = mktempdir(; cleanup = true), - ) == - PSI.ModelBuildStatus.BUILT - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - outputs = OptimizationProblemOutputs(model) - @test list_aux_variable_names(outputs) == [] - @test list_aux_variable_keys(outputs) == [] - @test list_variable_names(outputs) == ["ActivePowerVariable__ThermalStandard"] - @test list_variable_keys(outputs) == - [PSI.VariableKey(ActivePowerVariable, ThermalStandard)] - @test list_dual_names(outputs) == [] - @test list_dual_keys(outputs) == [] - @test list_parameter_names(outputs) == ["ActivePowerTimeSeriesParameter__PowerLoad"] - @test list_parameter_keys(outputs) == - [PSI.ParameterKey(ActivePowerTimeSeriesParameter, PowerLoad)] - - @test read_variable(outputs, "ActivePowerVariable__ThermalStandard") isa DataFrame - @test read_variable(outputs, ActivePowerVariable, ThermalStandard) isa DataFrame - @test read_variable( - outputs, - PSI.VariableKey(ActivePowerVariable, ThermalStandard), - ) isa - DataFrame - - @test read_parameter(outputs, "ActivePowerTimeSeriesParameter__PowerLoad") isa DataFrame - @test read_parameter(outputs, ActivePowerTimeSeriesParameter, PowerLoad) isa DataFrame - @test read_parameter( - outputs, - PSI.ParameterKey(ActivePowerTimeSeriesParameter, PowerLoad), - ) isa DataFrame - - @test read_optimizer_stats(model) isa DataFrame - for n in names(read_optimizer_stats(model)) - stats_values = read_optimizer_stats(model)[!, n] - if any(ismissing.(stats_values)) - @test ismissing.(stats_values) == - ismissing.(read_optimizer_stats(outputs)[!, n]) - elseif any(isnan.(stats_values)) - @test isnan.(stats_values) == isnan.(read_optimizer_stats(outputs)[!, n]) - else - @test stats_values == read_optimizer_stats(outputs)[!, n] - end - end - - for i in 1:executions - @test get_objective_value(outputs, i) isa Float64 - end -end - -@testset "Run EmulationModel with auto-build" begin - for serialize in (true, false) - template = get_thermal_dispatch_template_network() - c_sys5 = PSB.build_system( - PSITestSystems, - "c_sys5_uc"; - add_single_time_series = true, - force_build = true, - ) - - model = EmulationModel(template, c_sys5; optimizer = HiGHS_optimizer) - @test_throws ErrorException run!(model, executions = 10) - @test run!( - model; - executions = 10, - output_dir = mktempdir(; cleanup = true), - export_optimization_model = serialize, - ) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - end -end - -@testset "Test serialization/deserialization of EmulationModel outputs" begin - path = mktempdir(; cleanup = true) - template = get_thermal_dispatch_template_network() - c_sys5 = PSB.build_system( - PSITestSystems, - "c_sys5_uc"; - add_single_time_series = true, - force_build = true, - ) - - model = EmulationModel(template, c_sys5; optimizer = HiGHS_optimizer) - executions = 10 - @test build!(model; executions = executions, output_dir = path) == - PSI.ModelBuildStatus.BUILT - @test run!(model; export_problem_outputs = true) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - outputs1 = OptimizationProblemOutputs(model) - var1_a = read_variable(outputs1, ActivePowerVariable, ThermalStandard) - # Ensure that we can deserialize strings into keys. - var1_b = read_variable(outputs1, "ActivePowerVariable__ThermalStandard") - @test var1_a == var1_b - - # Outputs were automatically serialized here. - outputs2 = OptimizationProblemOutputs(PSI.get_output_dir(model)) - var2 = read_variable(outputs2, ActivePowerVariable, ThermalStandard) - @test var1_a == var2 - @test get_source_data(outputs2) === nothing - # Commented out for now, as we no longer automatically serialize the system with results, but this should be added back in the future. - # load_system(outputs2) - # @test get_source_data(outputs2) isa PSY.System - - # Serialize to a new directory with the exported function. - outputs_path = joinpath(path, "outputs") - serialize_outputs(outputs1, outputs_path) - @test isfile(joinpath(outputs_path, ISOPT._PROBLEM_OUTPUTS_FILENAME)) - outputs3 = OptimizationProblemOutputs(outputs_path) - var3 = read_variable(outputs3, ActivePowerVariable, ThermalStandard) - @test var1_a == var3 - @test get_source_data(outputs3) === nothing - set_source_data!(outputs3, outputs1.system) - @test get_source_data(outputs3) !== nothing - - exp_file = - joinpath(path, "outputs", "variables", "ActivePowerVariable__ThermalStandard.csv") - var4 = read_dataframe(exp_file) - # Manually Multiply by the base power var1_a has natural units and export writes directly from the solver - @test var1_a.value == var4.value .* 100.0 -end - -@testset "Test serialization of InitialConditionsData" begin - template = get_thermal_standard_uc_template() - sys = PSB.build_system( - PSITestSystems, - "c_sys5_pglib"; - add_single_time_series = true, - force_build = true, - ) - optimizer = HiGHS_optimizer - set_device_model!(template, ThermalMultiStart, ThermalMultiStartUnitCommitment) - model = EmulationModel(template, sys; optimizer = HiGHS_optimizer) - output_dir = mktempdir(; cleanup = true) - - @test build!(model; executions = 1, output_dir = output_dir) == - PSI.ModelBuildStatus.BUILT - ic_file = PSI.get_initial_conditions_file(model) - test_ic_serialization_outputs(model; ic_file_exists = true, message = "make") - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - # Build again, use existing initial conditions. - PSI.reset!(model) - @test build!(model; executions = 1, output_dir = output_dir) == - PSI.ModelBuildStatus.BUILT - test_ic_serialization_outputs(model; ic_file_exists = true, message = "make") - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - # Build again, use existing initial conditions. - model = EmulationModel( - template, - sys; - optimizer = optimizer, - deserialize_initial_conditions = true, - ) - @test build!(model; executions = 1, output_dir = output_dir) == - PSI.ModelBuildStatus.BUILT - test_ic_serialization_outputs(model; ic_file_exists = true, message = "deserialize") - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - # Construct and build again with custom initial conditions file. - initialization_file = joinpath(output_dir, ic_file * ".old") - mv(ic_file, initialization_file) - touch(ic_file) - model = EmulationModel( - template, - sys; - optimizer = optimizer, - initialization_file = initialization_file, - deserialize_initial_conditions = true, - ) - @test build!(model; executions = 1, output_dir = output_dir) == - PSI.ModelBuildStatus.BUILT - test_ic_serialization_outputs(model; ic_file_exists = true, message = "deserialize") - - # Construct and build again while skipping build of initial conditions. - model = EmulationModel(template, sys; optimizer = optimizer, initialize_model = false) - rm(ic_file) - @test build!(model; executions = 1, output_dir = output_dir) == - PSI.ModelBuildStatus.BUILT - test_ic_serialization_outputs(model; ic_file_exists = false, message = "skip") - @test run!(model) == PSI.RunStatus.SUCCESSFULLY_FINALIZED -end diff --git a/test/test_model_store.jl b/test/test_model_store.jl deleted file mode 100644 index 19b9fb03..00000000 --- a/test/test_model_store.jl +++ /dev/null @@ -1 +0,0 @@ -@testset "Test Model Store" begin end diff --git a/test/test_optimization_outputs.jl b/test/test_optimization_outputs.jl deleted file mode 100644 index e690fbcf..00000000 --- a/test/test_optimization_outputs.jl +++ /dev/null @@ -1,264 +0,0 @@ -# Test types defined in test_utils/test_types.jl -import InfrastructureOptimizationModels: - OptimizationContainerMetadata, - OptimizationProblemOutputs, - VariableKey, - ExpressionKey, - read_variable, - read_expression -import Dates: - DateTime, - Millisecond -import InfrastructureSystems as IS - -@testset "Test OptimizationProblemOutputs long format" begin - base_power = 10.0 - # 2 hours timestamp range - timestamp_range = - StepRange( - DateTime("2024-01-01T00:00:00"), - Millisecond(3600000), - DateTime("2024-01-01T03:00:00"), - ) - timestamp_vec = collect(timestamp_range) - data = IS.SystemData() - uuid = IS.make_uuid() - aux_variable_values = Dict() - @test !IOM.convert_output_to_natural_units(MockVariable) - @test IOM.convert_output_to_natural_units(MockVariable2) - var_key1 = VariableKey(MockVariable, IS.TestComponent) - var_key2 = VariableKey(MockVariable2, IS.TestComponent) - vals = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] - variable_values = Dict( - var_key1 => DataFrame( - "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], - "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], - "value" => vals, - ), - var_key2 => DataFrame( - "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], - "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], - "value" => vals, - ), - ) - dual_values = Dict() - parameter_values = Dict() - @test !IOM.convert_output_to_natural_units(MockExpression) - @test IOM.convert_output_to_natural_units(MockExpression2) - exp_key1 = ExpressionKey(MockExpression, IS.TestComponent) - exp_key2 = ExpressionKey(MockExpression2, MockThermalGen) - # Expression only 1 time-step - expression_values = Dict( - exp_key1 => DataFrame( - "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], - "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], - "value" => vals, - ), - exp_key2 => DataFrame( - "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], - "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], - "value" => vals, - ), - ) - optimizer_stats = DataFrames.DataFrame() - metadata = OptimizationContainerMetadata() - # Test with StepRange - opt_res1 = OptimizationProblemOutputs( - base_power, - timestamp_range, - data, - uuid, - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - metadata, - "test_model", - mktempdir(), - mktempdir(), - ) - # Test with Vector{DateTime} - opt_res2 = OptimizationProblemOutputs( - base_power, - timestamp_vec, - data, - uuid, - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - metadata, - "test_model", - mktempdir(), - mktempdir(), - ) - opt_res3 = OptimizationProblemOutputs( - base_power, - [timestamp_vec[1]], - data, - uuid, - aux_variable_values, - variable_values, - dual_values, - parameter_values, - expression_values, - optimizer_stats, - metadata, - "test_model", - mktempdir(), - mktempdir(), - ) - - var_res = read_variable(opt_res1, var_key1) - @test sort!(unique(var_res.DateTime)) == timestamp_vec - @test @rsubset(var_res, :name == "c1")[!, :value] == [1.0, 2.0, 3.0, 4.0] - @test @rsubset(var_res, :name == "c2")[!, :value] == [5.0, 6.0, 7.0, 8.0] - - var_res = read_variable(opt_res1, var_key2) - @test @rsubset(var_res, :name == "c1")[!, :value] == [10.0, 20.0, 30.0, 40.0] - @test @rsubset(var_res, :name == "c2")[!, :value] == [50.0, 60.0, 70.0, 80.0] - - var_res2 = read_variable( - opt_res1, - var_key1; - start_time = DateTime("2024-01-01T01:00:00"), - len = 2, - ) - @test @rsubset(var_res2, :name == "c1")[!, :value] == [2.0, 3.0] - @test @rsubset(var_res2, :name == "c2")[!, :value] == [6.0, 7.0] - - var_res2 = read_variable( - opt_res1, - var_key2; - start_time = DateTime("2024-01-01T01:00:00"), - len = 2, - ) - @test @rsubset(var_res2, :name == "c1")[!, :value] == [20.0, 30.0] - @test @rsubset(var_res2, :name == "c2")[!, :value] == [60.0, 70.0] - - var_res = read_variable(opt_res1, var_key2; table_format = IS.TableFormat.WIDE) - @test var_res[!, :c1] == [10.0, 20.0, 30.0, 40.0] - @test var_res[!, :c2] == [50.0, 60.0, 70.0, 80.0] - - exp_res = read_expression(opt_res2, exp_key1) - @test @rsubset(exp_res, :name == "c1")[!, :value] == [1.0, 2.0, 3.0, 4.0] - @test @rsubset(exp_res, :name == "c2")[!, :value] == [5.0, 6.0, 7.0, 8.0] - exp_res = read_expression(opt_res2, exp_key2) - @test @rsubset(exp_res, :name == "c1")[!, :value] == [10.0, 20.0, 30.0, 40.0] - @test @rsubset(exp_res, :name == "c2")[!, :value] == [50.0, 60.0, 70.0, 80.0] - - @test IOM.get_resolution(opt_res1) == Millisecond(3600000) - @test IOM.get_resolution(opt_res2) == Millisecond(3600000) - @test isnothing(IOM.get_resolution(opt_res3)) -end - -@testset "Test OptimizationProblemOutputs 3d long format" begin - timestamps = StepRange( - DateTime("2024-01-01T00:00:00"), - Millisecond(3600000), - DateTime("2024-01-01T01:00:00"), - ) - data = IS.SystemData() - aux_variable_values = Dict() - var_key = VariableKey(MockVariable, IS.TestComponent) - vals = [1.0, 2.0, 3.0, 4.0] - variable_values = Dict( - var_key => DataFrame( - "time_index" => [1, 2, 1, 2], - "name" => ["c1", "c2", "c1", "c2"], - "name2" => ["c3", "c4", "c3", "c4"], - "value" => vals, - ), - ) - optimizer_stats = DataFrames.DataFrame() - res = OptimizationProblemOutputs( - 100.0, - timestamps, - data, - IS.make_uuid(), - Dict(), - variable_values, - Dict(), - Dict(), - Dict(), - optimizer_stats, - OptimizationContainerMetadata(), - "test_model", - mktempdir(), - mktempdir(), - ) - - var_res = read_variable(res, var_key) - @test @rsubset(var_res, :name == "c1" && :name2 == "c3")[!, :value] == [1.0, 3.0] - @test @rsubset(var_res, :name == "c2" && :name2 == "c4")[!, :value] == [2.0, 4.0] -end - -@testset "Test OptimizationProblemOutputs _process_timestamps" begin - time_ids = [1, 2, 3, 4] - timestamps = [ - DateTime("2024-01-01T00:00:00"), - DateTime("2024-01-01T01:00:00"), - DateTime("2024-01-01T02:00:00"), - DateTime("2024-01-01T03:00:00"), - ] - data = IS.SystemData() - var_key = VariableKey(MockVariable, IS.TestComponent) - vals = [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0] - variable_values = Dict( - var_key => DataFrame( - "time_index" => [1, 2, 3, 4, 1, 2, 3, 4], - "name" => ["c1", "c1", "c1", "c1", "c2", "c2", "c2", "c2"], - "value" => vals, - ), - ) - optimizer_stats = DataFrames.DataFrame() - metadata = OptimizationContainerMetadata() - opt_res = OptimizationProblemOutputs( - 100.0, - timestamps, - data, - IS.make_uuid(), - Dict(), - variable_values, - Dict(), - Dict(), - Dict(), - optimizer_stats, - metadata, - "DecisionModel", - mktempdir(), - mktempdir(), - ) - @test IOM._process_timestamps(opt_res, nothing, nothing) == - (time_ids, timestamps) - @test IOM._process_timestamps(opt_res, timestamps[2], nothing) == - (time_ids[2:end], timestamps[2:end]) - @test IOM._process_timestamps(opt_res, timestamps[4], nothing) == - ([time_ids[4]], [timestamps[4]]) - @test IOM._process_timestamps(opt_res, nothing, 3) == - (time_ids[1:3], timestamps[1:3]) - @test IOM._process_timestamps(opt_res, timestamps[2], 2) == - (time_ids[2:3], timestamps[2:3]) - - @test_throws IS.InvalidValue IOM._process_timestamps( - opt_res, - timestamps[1] - Hour(1), - nothing, - ) - @test_throws IS.InvalidValue IOM._process_timestamps( - opt_res, - timestamps[4] + Hour(1), - nothing, - ) - @test_throws IS.InvalidValue IOM._process_timestamps(opt_res, nothing, -1) - @test_throws IS.InvalidValue IOM._process_timestamps(opt_res, nothing, 10) - @test_throws IS.InvalidValue IOM._process_timestamps( - opt_res, - timestamps[2], - 10, - ) -end diff --git a/test/test_utils/common_operation_model.jl b/test/test_utils/common_operation_model.jl deleted file mode 100644 index 3d465231..00000000 --- a/test/test_utils/common_operation_model.jl +++ /dev/null @@ -1,31 +0,0 @@ -const _DESERIALIZE_MESSAGE = "Deserialized initial_conditions_data" -const _MAKE_IC_MESSAGE = "Make Initial Conditions Model" -const _SKIP_IC_MESSAGE = "Skip build of initial conditions" - -function test_ic_serialization_outputs(model::PSI.OperationModel; ic_file_exists, message) - ic_file = PSI.get_initial_conditions_file(model) - log_file = PSI.get_log_file(model) - - @test isfile(ic_file) == ic_file_exists - if ic_file_exists - @test Serialization.deserialize(ic_file) isa PSI.InitialConditionsData - end - - make = false - deserialize = false - skip = false - if message == "make" - make = true - elseif message == "deserialize" - deserialize = true - elseif message == "skip" - skip = true - else - error("invalid: $message") - end - - text = read(log_file, String) - @test make == occursin(_MAKE_IC_MESSAGE, text) - @test deserialize == occursin(_DESERIALIZE_MESSAGE, text) - @test skip == occursin(_SKIP_IC_MESSAGE, text) -end diff --git a/test/test_utils/run_simulation.jl b/test/test_utils/run_simulation.jl deleted file mode 100644 index 58ff3031..00000000 --- a/test/test_utils/run_simulation.jl +++ /dev/null @@ -1,105 +0,0 @@ -function load_pf_export(root, export_subdir) - raw_path, md_path = get_psse_export_paths(export_subdir) - sys = System(joinpath(root, raw_path), JSON3.read(joinpath(root, md_path), Dict)) - set_units_base_system!(sys, "NATURAL_UNITS") - return sys -end - -function run_simulation( - c_sys5_hy_uc, - c_sys5_hy_ed, - file_path::String, - export_path; - in_memory = false, - uc_network_model = nothing, - ed_network_model = nothing, -) - template_uc = get_template_basic_uc_simulation() - template_ed = get_template_nomin_ed_simulation() - isnothing(uc_network_model) && ( - uc_network_model = - NetworkModel(CopperPlatePowerModel; duals = [CopperPlateBalanceConstraint]) - ) - isnothing(ed_network_model) && ( - ed_network_model = - NetworkModel( - CopperPlatePowerModel; - duals = [CopperPlateBalanceConstraint], - use_slacks = true, - ) - ) - set_device_model!(template_ed, InterruptiblePowerLoad, StaticPowerLoad) - set_network_model!( - template_uc, - uc_network_model, - ) - set_network_model!( - template_ed, - ed_network_model, - ) - models = SimulationModels(; - decision_models = [ - DecisionModel( - template_uc, - c_sys5_hy_uc; - name = "UC", - optimizer = HiGHS_optimizer, - ), - DecisionModel( - template_ed, - c_sys5_hy_ed; - name = "ED", - optimizer = ipopt_optimizer, - ), - ], - ) - - sequence = SimulationSequence(; - models = models, - feedforwards = Dict( - "ED" => [ - SemiContinuousFeedforward(; - component_type = ThermalStandard, - source = OnVariable, - affected_values = [ActivePowerVariable], - ), - ], - ), - ini_cond_chronology = InterProblemChronology(), - ) - sim = Simulation(; - name = "no_cache", - steps = 2, - models = models, - sequence = sequence, - simulation_folder = file_path, - ) - - build_out = build!(sim; console_level = Logging.Error) - @test build_out == PSI.SimulationBuildStatus.BUILT - - exports = Dict( - "models" => [ - Dict( - "name" => "UC", - "store_all_variables" => true, - "store_all_parameters" => true, - "store_all_duals" => true, - "store_all_aux_variables" => true, - ), - Dict( - "name" => "ED", - "store_all_variables" => true, - "store_all_parameters" => true, - "store_all_duals" => true, - "store_all_aux_variables" => true, - ), - ], - "path" => export_path, - "optimizer_stats" => true, - ) - execute_out = execute!(sim; exports = exports, in_memory = in_memory) - @test execute_out == PSI.RunStatus.SUCCESSFULLY_FINALIZED - - return sim -end