From c7677417583a1fe14d6eea6088efd28d608056aa Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:26:25 +1000 Subject: [PATCH 1/5] Translate the new-format network tables to buses and links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit network_geography becomes buses (REZ buses dropped when rezs is attached_to_parent_node), and the path/limit/expansion tables become links. The winter_reference limit is the link's static p_nom — winter is the calendar's default season — with the other timeslices' forward and reverse limits emitted as per-unit values in a link_timeslice_limits table for pypsa_build to expand into p_max_pu/p_min_pu series (per-unit values can exceed 1.0 when a season's limit tops winter's). Asymmetric reverse limits land in p_min_pu rather than separate links. Expansion options become one extendable link per (path, investment period) with annuitised capital costs, gated by transmission_expansion / rez_transmission_expansion according to whether the path connects a REZ. Paths with no capacity data take rez_to_sub_region_transmission_default_limit, replicating the existing REZ behaviour. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/network.py | 424 ++++++++++++++++++++++++++ tests/test_translator/test_network.py | 297 ++++++++++++++++++ 2 files changed, 721 insertions(+) create mode 100644 src/ispypsa/translator/network.py create mode 100644 tests/test_translator/test_network.py diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py new file mode 100644 index 00000000..e3d6d543 --- /dev/null +++ b/src/ispypsa/translator/network.py @@ -0,0 +1,424 @@ +"""Translates the new-format network tables into PyPSA friendly buses and links. + +Consumes ``network_geography``, ``network_transmission_paths``, +``network_transmission_path_limits``, ``network_expansion_options`` and +``network_transmission_path_expansion_costs``, handling flow paths and REZ +connections through one unified pipeline (both are just paths). +""" + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.helpers import _annuitised_investment_costs + +_LINK_COLUMNS = [ + "isp_name", + "name", + "carrier", + "bus0", + "bus1", + "p_nom", + "p_min_pu", + "build_year", + "lifetime", + "capital_cost", + "p_nom_extendable", + "isp_type", +] + +_LINK_TIMESLICE_LIMIT_COLUMNS = ["name", "attribute", "timeslice", "value"] + + +def _translate_network_geography_to_buses( + network_geography: pd.DataFrame, rezs: str +) -> pd.DataFrame: + """Creates one PyPSA bus per geography in the model. + + REZ buses are only created when REZs are modelled as discrete nodes; + with ``rezs="attached_to_parent_node"`` REZ-located components connect + straight to the parent geography's bus. + + I/O Example: + network_geography: + geo_id geo_type region_id subregion_id + NQ subregion QLD NQ + Q1 rez QLD NQ + + rezs = "discrete_nodes" returns: + name + NQ + Q1 + + rezs = "attached_to_parent_node" returns: + name + NQ + """ + buses = network_geography + if rezs != "discrete_nodes": + buses = buses[buses["geo_type"] != "rez"] + buses = buses.loc[:, ["geo_id"]].rename(columns={"geo_id": "name"}) + return buses.reset_index(drop=True) + + +def _translate_network_to_links( + ispypsa_tables: dict[str, pd.DataFrame], + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Translates the network tables into PyPSA links and per-timeslice limits. + + Existing links carry the winter_reference limit as p_nom; limits for the + other demand conditions are returned per unit of p_nom in the + link_timeslice_limits table. Expansion links are built per investment + period from the unified expansion options/costs tables (their total + capacity is capped by the expansion-limit custom constraints built in + ispypsa.translator.constraints). + + Returns: + Tuple of (links, link_timeslice_limits) in PyPSA friendly format. + """ + rez_ids = _rez_geo_ids(ispypsa_tables["network_geography"]) + paths = _drop_rez_paths_if_not_modelled( + ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs + ) + limits = ispypsa_tables["network_transmission_path_limits"] + static_limits = _extract_static_limits(limits) + existing_links = _build_existing_links( + paths, + static_limits, + rez_ids, + config.network.rez_to_sub_region_transmission_default_limit, + config.temporal.range.start_year, + ) + link_timeslice_limits = _translate_timeslice_limits_to_pu(limits, existing_links) + options = _pivot_physical_expansion_options( + ispypsa_tables["network_expansion_options"] + ) + options = _filter_options_to_enabled_expansion( + options, + existing_links, + config.network.transmission_expansion, + config.network.rez_transmission_expansion, + ) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + expansion_links = _build_expansion_links(existing_links, options, costs) + links = pd.concat([existing_links, expansion_links], ignore_index=True) + return links.loc[:, _LINK_COLUMNS], link_timeslice_limits + + +def _rez_geo_ids(network_geography: pd.DataFrame) -> set[str]: + """geo_ids of the REZ entries in the geography. + + I/O Example: + geo_id=NQ/geo_type=subregion, geo_id=Q1/geo_type=rez -> {"Q1"} + """ + return set(network_geography.loc[network_geography["geo_type"] == "rez", "geo_id"]) + + +def _drop_rez_paths_if_not_modelled( + paths: pd.DataFrame, rez_ids: set[str], rezs: str +) -> pd.DataFrame: + """Drops REZ-to-parent paths when REZs are not modelled as discrete nodes. + + I/O Example: + paths (path_ids): CQ-NQ, Q1-NQ (Q1 is a REZ) + + rezs = "attached_to_parent_node" returns paths: CQ-NQ + rezs = "discrete_nodes" returns paths unchanged. + """ + if rezs == "discrete_nodes": + return paths + return paths[~paths["geo_from"].isin(rez_ids)] + + +def _extract_static_limits(limits: pd.DataFrame) -> pd.DataFrame: + """Reduces the per-timeslice limits to one static capacity per path and + direction: the winter_reference limit. + + Winter is used as the link's p_nom because it is the demand condition left + over when no summer timeslice is active — limits for the other conditions + are applied per unit of it. Collapsed all-NaN rows (paths with no limit + data) contribute nothing; paths absent from the result get the default + limit downstream. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_winter_reference 1910 + N1-CNSW , , # collapsed row + + returns: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + """ + winter = limits[limits["timeslice"].str.endswith("_winter_reference", na=False)] + return winter.loc[:, ["path_id", "direction", "capacity"]] + + +def _build_existing_links( + paths: pd.DataFrame, + static_limits: pd.DataFrame, + rez_ids: set[str], + default_limit: float, + start_year: int, +) -> pd.DataFrame: + """Builds one existing (non-extendable) PyPSA link per transmission path. + + I/O Example: + paths: + path_id geo_from geo_to carrier + CQ-NQ CQ NQ AC + N1-CNSW N1 CNSW AC # REZ path, no limit data + + static_limits: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + + returns (abridged): + isp_name name bus0 bus1 p_nom p_min_pu isp_type + CQ-NQ CQ-NQ_existing CQ NQ 1400 -1.364 flow_path + N1-CNSW N1-CNSW_existing N1 CNSW 100000 -1.0 rez # default limit + """ + links = paths.rename( + columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} + ) + links = _add_static_capacities(links, static_limits, default_limit) + links["name"] = links["isp_name"] + "_existing" + links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") + links["build_year"] = start_year - 1 + links["lifetime"] = np.inf + links["capital_cost"] = np.nan + links["p_nom_extendable"] = False + return links + + +def _add_static_capacities( + links: pd.DataFrame, static_limits: pd.DataFrame, default_limit: float +) -> pd.DataFrame: + """Merges the static forward limit as p_nom and the reverse limit as p_min_pu. + + Paths with no static limit in a direction get the default: ``default_limit`` + as p_nom (the path is constraint-modelled with no explicit physical limit) + and a symmetric -1.0 as p_min_pu. Zero-capacity paths (new parallel + corridors) also get a symmetric p_min_pu, avoiding a 0/0 division. + + I/O Example: + links (isp_name): CQ-NQ, N1-CNSW + static_limits: CQ-NQ forward 1400, CQ-NQ reverse 1910 + + returns: + isp_name p_nom p_min_pu + CQ-NQ 1400 -1.364 # -1910/1400 + N1-CNSW 100000 -1.0 # defaults + """ + directions = static_limits.pivot( + index="path_id", columns="direction", values="capacity" + ).reindex(columns=["forward", "reverse"]) + links = links.merge( + directions, left_on="isp_name", right_index=True, how="left" + ).rename(columns={"forward": "p_nom"}) + links["p_min_pu"] = np.where( + links["reverse"].notna() & (links["p_nom"] > 0), + -1.0 * links["reverse"] / links["p_nom"], + -1.0, + ) + links["p_nom"] = links["p_nom"].fillna(default_limit) + return links.drop(columns=["reverse"]) + + +def _translate_timeslice_limits_to_pu( + limits: pd.DataFrame, existing_links: pd.DataFrame +) -> pd.DataFrame: + """Expresses each per-timeslice limit per unit of its link's p_nom. + + Forward limits become p_max_pu values and reverse limits p_min_pu values. + pypsa_build expands them into per-snapshot series via the + timeslice_snapshots mapping; snapshots outside any of a link's tagged + timeslices keep the static (winter_reference) limit. Zero-p_nom links + (new parallel corridors) are skipped — all their limits are zero and the + per-unit form is undefined. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_peak_demand 1440 + + existing_links (abridged): + isp_name name p_nom + CQ-NQ CQ-NQ_existing 1400 + + returns: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu qld_winter_reference 1.0 + CQ-NQ_existing p_min_pu qld_peak_demand -1.029 + """ + rows = limits.dropna(subset=["timeslice", "capacity"]) + rows = rows.merge( + existing_links.loc[:, ["isp_name", "name", "p_nom"]], + left_on="path_id", + right_on="isp_name", + ) + rows = rows[rows["p_nom"] > 0] + rows["attribute"] = rows["direction"].map( + {"forward": "p_max_pu", "reverse": "p_min_pu"} + ) + sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) + rows["value"] = sign * rows["capacity"] / rows["p_nom"] + return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) + + +def _pivot_physical_expansion_options(expansion_options: pd.DataFrame) -> pd.DataFrame: + """Pairs each physical path's forward and reverse expansion rows into one row. + + constraint_relaxation rows are not physical paths — they are translated + into custom-constraint relaxation generators by + ispypsa.translator.constraints. + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 Option 1 + CQ-NQ reverse 900 Option 1 + SWQLD1 constraint_relaxation 500 Option 2 + + returns: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + """ + physical = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "reverse"]) + ] + options = ( + physical.pivot( + index="expansion_id", columns="expansion_type", values="allowed_expansion" + ) + .reindex(columns=["forward", "reverse"]) + .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) + ) + options.columns.name = None + return options.reset_index() + + +def _filter_options_to_enabled_expansion( + options: pd.DataFrame, + existing_links: pd.DataFrame, + transmission_expansion: bool, + rez_transmission_expansion: bool, +) -> pd.DataFrame: + """Keeps the expansion options enabled by the network config flags. + + ``transmission_expansion`` gates paths between (sub)regions; + ``rez_transmission_expansion`` gates REZ connection paths. Options for + paths not in the model (e.g. REZ paths dropped under + attached_to_parent_node) drop out here too, as they match no link. + + I/O Example: + options (expansion_ids): CQ-NQ, Q1-NQ + existing_links: CQ-NQ isp_type=flow_path, Q1-NQ isp_type=rez + + transmission_expansion=True, rez_transmission_expansion=False + returns options: CQ-NQ + """ + rez_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "rez", "isp_name"] + ) + flow_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "flow_path", "isp_name"] + ) + enabled = set() + if transmission_expansion: + enabled |= flow_path_ids + if rez_transmission_expansion: + enabled |= rez_path_ids + return options[options["expansion_id"].isin(enabled)] + + +def _prepare_expansion_costs( + expansion_costs: pd.DataFrame, + investment_periods: list[int], + year_type: str, + wacc: float, + asset_lifetime: int, +) -> pd.DataFrame: + """Filters the long-format expansion costs to the investment periods and + annuitises them. + + The ``year`` column holds financial-year ending years as ints, matching + the investment period labels used with ``year_type="fy"``. + + I/O Example: + expansion_costs: + expansion_id year cost + CQ-NQ 2025 1000 + CQ-NQ 2026 1010 + + investment_periods=[2026], wacc=0.07, asset_lifetime=30 returns: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + """ + if year_type != "fy": + raise NotImplementedError( + f"Network expansion costs are not implemented for year_type: {year_type}" + ) + costs = expansion_costs[expansion_costs["year"].isin(investment_periods)].copy() + costs["capital_cost"] = costs["cost"].apply( + lambda cost: _annuitised_investment_costs(cost, wacc, asset_lifetime) + ) + return costs.loc[:, ["expansion_id", "year", "capital_cost"]] + + +def _build_expansion_links( + existing_links: pd.DataFrame, + options: pd.DataFrame, + costs: pd.DataFrame, +) -> pd.DataFrame: + """Builds one extendable PyPSA link per expandable path and investment period. + + Each expansion link is unbounded on its own — the expansion-limit custom + constraints built in ispypsa.translator.constraints cap the p_nom built + across a path's expansion links at the selected option's forward capacity. + Asymmetric options are modelled with p_min_pu = -reverse/forward, so + however much forward capacity is built, reverse capacity scales in the + option's proportion. + + I/O Example: + options: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + + costs: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + + existing_links (abridged): CQ-NQ bus0=CQ bus1=NQ carrier=AC isp_type=flow_path + + returns (abridged): + isp_name name p_nom p_nom_extendable p_min_pu build_year capital_cost + CQ-NQ CQ-NQ_exp_2026 0.0 True -0.9 2026 81.4 + """ + links = costs.merge(options, on="expansion_id") + links = links.merge( + existing_links.loc[:, ["isp_name", "bus0", "bus1", "carrier", "isp_type"]], + left_on="expansion_id", + right_on="isp_name", + ) + links["name"] = links["expansion_id"] + "_exp_" + links["year"].astype(str) + links["p_nom"] = 0.0 + links["p_nom_extendable"] = True + links["p_min_pu"] = -1.0 * links["reverse_capacity"] / links["forward_capacity"] + links["build_year"] = links["year"] + links["lifetime"] = np.inf + return links diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py new file mode 100644 index 00000000..8c1ae216 --- /dev/null +++ b/tests/test_translator/test_network.py @@ -0,0 +1,297 @@ +import pandas as pd +import pytest + +from ispypsa.translator.helpers import _annuitised_investment_costs +from ispypsa.translator.network import ( + _translate_network_geography_to_buses, + _translate_network_to_links, +) + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25) — expansion cost expectations are multiples of this. +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def test_translate_network_geography_to_buses_discrete_nodes(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses(network_geography, "discrete_nodes") + + expected = csv_str_to_df(""" + name + NQ + CNSW + Q1 + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_translate_network_geography_to_buses_attached_to_parent_node(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses( + network_geography, "attached_to_parent_node" + ) + + expected = csv_str_to_df(""" + name + NQ + """) + pd.testing.assert_frame_equal(result, expected) + + +def _network_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """New-format network tables with one flow path (CQ-NQ, asymmetric winter + limits, expandable), one REZ path with limits (Q1-NQ) and one collapsed + REZ path with no limit data (N1-CNSW).""" + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + N1, rez, NSW + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + N1-CNSW, N1, CNSW, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_summer_typical, 1300 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_peak_demand, 1440 + CQ-NQ, reverse, qld_summer_typical, 1600 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, forward, qld_winter_reference, 750 + Q1-NQ, reverse, qld_winter_reference, 750 + N1-CNSW, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, CQ-NQ Option 1 + CQ-NQ, reverse, 900, CQ-NQ Option 1 + Q1-NQ, forward, 500, Q1 Option 1 + Q1-NQ, reverse, 500, Q1 Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + # 2025 is outside the sample config's investment periods (2026, 2028) and + # is expected to be filtered out. + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2025, 900000 + CQ-NQ, 2026, 1000000 + Q1-NQ, 2026, 500000 + SWQLD1, 2026, 400000 + """) + return tables + + +def test_translate_network_to_links(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, -1.364286, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, -1.0, 2025, inf, , False, rez + N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, -1.0, 2025, inf, , False, rez + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.928571 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -1.142857 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.364286 + Q1-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + Q1-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The flow path's expansion link is gone; the REZ path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + "Q1-NQ_exp_2026", + ] + + +def test_translate_network_to_links_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The REZ path's expansion link is gone; the flow path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "CQ-NQ_exp_2026", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_rezs_attached_to_parent_node( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # REZ paths (and their expansion links and timeslice limits) are dropped. + assert sorted(links["name"]) == ["CQ-NQ_existing", "CQ-NQ_exp_2026"] + assert set(link_timeslice_limits["name"]) == {"CQ-NQ_existing"} + + +def test_translate_network_to_links_zero_capacity_parallel_path( + csv_str_to_df, sample_model_config +): + """New parallel corridors arrive as zero-capacity paths: p_nom 0 with a + symmetric p_min_pu (no 0/0 division), and no per-unit timeslice rows.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CNSW-SNW, CNSW, SNW, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CNSW-SNW, forward, nsw_winter_reference, 0 + CNSW-SNW, reverse, nsw_winter_reference, 0 + """) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, -1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_empty_expansion_tables( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = pd.DataFrame( + columns=["path_id", "geo_from", "geo_to", "carrier"] + ) + ispypsa_tables["network_transmission_path_limits"] = pd.DataFrame( + columns=["path_id", "direction", "timeslice", "capacity"] + ) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) From 0a40ca48fb2f0e9e128e8a8c945298c030469b10 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 2 Jul 2026 09:57:01 +1000 Subject: [PATCH 2/5] Resolve network-table wildcards and carry all link limits per snapshot Pilots the blank-key-as-wildcard convention on the three sparse network tables (limits, expansion options, expansion costs) as a reference implementation for wider adoption: _resolve_wildcards in translator/helpers.py does the resolution until schema validation lands, with runtime guards in network.py standing in for the option-pairing rule. Drops that are designed selection (constraint_relaxation routing, non-investment-period cost years) are no longer logged as data loss. Links now take p_nom as the larger of their two directional capacities so every per-unit limit sits in [-1, 1], and all real limits (including blank-timeslice fallbacks) move into link_timeslice_limits under the coverage contract (Open-ISP/ISPyPSA#123); the static link attributes are inert defaults. rez_to_sub_region_transmission_default_limit becomes transmission_default_limit -- the default now applies to any path without limit data, not just REZ connections. Co-Authored-By: Claude Fable 5 --- docs/config.md | 8 +- .../new_test_run/ispypsa_config.yaml | 6 +- docs/method.md | 2 +- ispypsa_config.yaml | 6 +- ispypsa_config_dev.yaml | 6 +- src/ispypsa/config/validators.py | 2 +- src/ispypsa/translator/helpers.py | 104 ++++ src/ispypsa/translator/network.py | 502 +++++++++++----- .../translator/renewable_energy_zones.py | 8 +- .../schemas/network_expansion_options.yaml | 39 +- ...ork_transmission_path_expansion_costs.yaml | 50 +- .../network_transmission_path_limits.yaml | 68 ++- tests/conftest.py | 2 +- tests/test_cli/cli_test_helpers.py | 4 +- .../test_config/test_pydantic_model_config.py | 4 +- tests/test_translator/ispypsa_config.yaml | 6 +- tests/test_translator/test_network.py | 538 ++++++++++++++++-- .../test_translator_helpers.py | 122 ++++ .../test_limit_on_transmission_expansion.py | 2 +- .../test_time_varying_flow_path_costs.py | 2 +- ...test_vre_build_limit_custom_constraints.py | 2 +- 21 files changed, 1221 insertions(+), 262 deletions(-) diff --git a/docs/config.md b/docs/config.md index 89b5df6a..d9c142b0 100644 --- a/docs/config.md +++ b/docs/config.md @@ -213,10 +213,10 @@ Examples: ```rezs: discrete_nodes``` -### network.rez_to_sub_region_transmission_default_limit +### network.transmission_default_limit -Link capacity limit for rez to node connections that have their limit's modelled -through custom constraint (MW). +Default transmission capacity limit (MW) applied to a path with no explicit limit +in the inputs. The export limits for some REZs are modelled via custom constraints which incorporate multiple transmission line flows and/or generator dispatch levels. For these @@ -226,7 +226,7 @@ export limit. Examples: -```rez_to_sub_region_transmission_default_limit: 1e5``` +```transmission_default_limit: 1e5``` ## Temporal diff --git a/docs/ispypsa_runs/new_test_run/ispypsa_config.yaml b/docs/ispypsa_runs/new_test_run/ispypsa_config.yaml index 4c037e80..0f6986cc 100644 --- a/docs/ispypsa_runs/new_test_run/ispypsa_config.yaml +++ b/docs/ispypsa_runs/new_test_run/ispypsa_config.yaml @@ -68,9 +68,9 @@ network: # "discrete_nodes": REZs are added as network nodes to model REZ transmission limits # "attached_to_parent_node": REZ resources are attached to their parent node (sub-region or NEM region) rezs: discrete_nodes - # Line capacity limit for rez to node connections that have their limit's modelled - # through custom constraint (MW). - rez_to_sub_region_transmission_default_limit: 1e5 + # Default transmission capacity limit (MW) for a path with no explicit limit in + # the inputs (e.g. REZ connections whose limit is modelled via custom constraints). + transmission_default_limit: 1e5 temporal: year_type: fy range: diff --git a/docs/method.md b/docs/method.md index 6893ba5d..5bc167b3 100644 --- a/docs/method.md +++ b/docs/method.md @@ -75,7 +75,7 @@ both directions of flow: `renewable_energy_zones` is used to set power flow limit. - TODO: Implement time varying export limits making use of the peak demand and winter limits provided in the IASR workbook. -- If a NA or blank value is provided then the `rez_to_sub_region_transmission_default_limit` +- If a NA or blank value is provided then the `transmission_default_limit` from the config file is used to set the limit. This is typically set to high value (1e5). Using the default limit is done for REZs where [custom contraints](#custom-constraints) are used to model REZ export limits, such that the static limit will not influence the optimisation. diff --git a/ispypsa_config.yaml b/ispypsa_config.yaml index 0d784d9f..a6a58000 100644 --- a/ispypsa_config.yaml +++ b/ispypsa_config.yaml @@ -83,9 +83,9 @@ network: # Whether Renewable Energy Zones (REZs) are modelled as distinct nodes # Only discrete_nodes implemented rezs: discrete_nodes - # Line capacity limit for rez to node connections that have their limit's modelled - # through custom constraint (MW). - rez_to_sub_region_transmission_default_limit: 1e5 + # Default transmission capacity limit (MW) for a path with no explicit limit in + # the inputs (e.g. REZ connections whose limit is modelled via custom constraints). + transmission_default_limit: 1e5 # ===== Temporal ======================================================================= diff --git a/ispypsa_config_dev.yaml b/ispypsa_config_dev.yaml index 57706d5b..5067b2e2 100644 --- a/ispypsa_config_dev.yaml +++ b/ispypsa_config_dev.yaml @@ -83,9 +83,9 @@ network: # Whether Renewable Energy Zones (REZs) are modelled as distinct nodes # Only discrete_nodes implemented rezs: discrete_nodes - # Line capacity limit for rez to node connections that have their limit's modelled - # through custom constraint (MW). - rez_to_sub_region_transmission_default_limit: 1e5 + # Default transmission capacity limit (MW) for a path with no explicit limit in + # the inputs (e.g. REZ connections whose limit is modelled via custom constraints). + transmission_default_limit: 1e5 # ===== Temporal ======================================================================= diff --git a/src/ispypsa/config/validators.py b/src/ispypsa/config/validators.py index d138419b..c150a8d0 100644 --- a/src/ispypsa/config/validators.py +++ b/src/ispypsa/config/validators.py @@ -44,7 +44,7 @@ class NetworkConfig(BaseModel): annuitisation_lifetime: int transmission_expansion: bool rez_transmission_expansion: bool - rez_to_sub_region_transmission_default_limit: float + transmission_default_limit: float class TemporalAggregationConfig(BaseModel): diff --git a/src/ispypsa/translator/helpers.py b/src/ispypsa/translator/helpers.py index 965c249c..589c15f1 100644 --- a/src/ispypsa/translator/helpers.py +++ b/src/ispypsa/translator/helpers.py @@ -1,3 +1,4 @@ +import logging import re import pandas as pd @@ -154,3 +155,106 @@ def convert_to_numeric_if_possible(df: pd.DataFrame, cols: list[str]) -> pd.Data df = df.drop(columns=["temp_col"]) return df + + +def _resolve_wildcards( + table: pd.DataFrame, + allowed_values: dict[str, list], + value_columns: list[str], + expected_drops: tuple[str, ...] = (), +) -> pd.DataFrame: + """Expand a sparse "wildcard" table into one row per concrete key combination. + + A key column may be left blank (NaN) to act as a wildcard that applies to + every value of that column. ``allowed_values`` lists, for each wildcardable + column, the concrete values it may take — the schema's allowed_values / + allowed_values_from, resolved to actual values. Each blank cell in those + columns is fanned out to every allowed value; a filled cell survives only if + it is itself an allowed value, so a row carrying an out-of-set value drops + out. Drops are logged at INFO unless the column is named in + ``expected_drops`` — for those, dropping is the caller's designed selection + (e.g. keeping only investment-period years), not data loss worth surfacing. + Key columns absent from ``allowed_values`` (e.g. timeslice) ride along + unchanged. + + Once the blanks are filled in, several rows can land on the same key — a + specific row and a wildcard one. The row that used the fewest wildcards (the + most specific) wins; callers rely on the schema's *_resolve_unambiguously + rule to guarantee there is never a tie. + + I/O Example: + table (a blank cell is a wildcard): + path_id direction timeslice capacity + CQ-NQ forward peak 1200 + 500 # all keys blank: a global default + N1-CNSW # blank direction and capacity + + allowed_values = {"path_id": ["CQ-NQ", "N1-CNSW"], + "direction": ["forward", "reverse"]} + value_columns = ["capacity"] + + returns: + path_id direction timeslice capacity + CQ-NQ forward peak 1200 # specific row, untouched + CQ-NQ forward 500 # global default fills the gaps... + CQ-NQ reverse 500 + N1-CNSW forward NaN # ...but N1-CNSW's own blank-capacity + N1-CNSW reverse NaN # row is more specific, so it wins + """ + key_columns = [c for c in table.columns if c not in value_columns] + work = table.copy() + # Count each row's wildcards now, before the blanks are filled in, so that + # after expansion we can keep the row that used the fewest wildcards wherever + # rows land on the same key. + work["_wildcards"] = sum(work[c].isna().astype(int) for c in allowed_values) + # Resolve one wildcardable column per pass; each pass fills that column in. + for column, values in allowed_values.items(): + work = _expand_column(work, column, values, column not in expected_drops) + # Most specific (fewest wildcards) wins where rows now share a key. + most_specific_first = work.sort_values("_wildcards", kind="stable") + resolved = most_specific_first.drop_duplicates(key_columns, keep="first") + return resolved.loc[:, list(table.columns)].reset_index(drop=True) + + +def _expand_column( + table: pd.DataFrame, column: str, allowed_values: list, log_drops: bool +) -> pd.DataFrame: + """Resolve a single wildcardable column of a wildcard table. + + Splits the rows on whether ``column`` is blank, then recombines: a filled + cell is kept only if its value is allowed, while each blank (wildcard) cell is + cross-joined with the allowed values so it fans out to one row per value. + Splitting first is the trick that keeps any blank out of the merge key (a NaN + key would not match itself in a merge). Out-of-set filled cells drop out, + logged only when ``log_drops`` is set (see _resolve_wildcards). + + I/O Example: + table: column = "direction" + path_id direction allowed_values = ["forward", "reverse"] + CQ-NQ forward + N1-CNSW # blank: a wildcard + + returns: + path_id direction + CQ-NQ forward # filled and allowed: kept as-is + N1-CNSW forward # blank: fanned out to every allowed value + N1-CNSW reverse + """ + is_wildcard = table[column].isna() + if log_drops: + _log_dropped_values(table.loc[~is_wildcard, column], allowed_values, column) + # A filled cell survives only if its value is one of the allowed values... + concrete = table[~is_wildcard & table[column].isin(allowed_values)] + # ...and a blank cell fans out to one row per allowed value (a cross join). + allowed_frame = pd.DataFrame({column: allowed_values}) + expanded = table[is_wildcard].drop(columns=column).merge(allowed_frame, how="cross") + return pd.concat([concrete, expanded], ignore_index=True) + + +def _log_dropped_values(values: pd.Series, allowed_values: list, column: str) -> None: + """Log, once, any filled values dropped for falling outside the allowed set.""" + # tolist() converts numpy scalars to native types so the message reads + # "[2025]" rather than "[np.int64(2025)]". + dropped = sorted(set(values.dropna().tolist()) - set(allowed_values)) + if dropped: + logging.info(f"Dropped rows whose {column} is not an allowed value: {dropped}") diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index e3d6d543..f5187fb4 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -1,16 +1,29 @@ -"""Translates the new-format network tables into PyPSA friendly buses and links. - -Consumes ``network_geography``, ``network_transmission_paths``, -``network_transmission_path_limits``, ``network_expansion_options`` and -``network_transmission_path_expansion_costs``, handling flow paths and REZ -connections through one unified pipeline (both are just paths). +"""Translate the new-format network tables into PyPSA buses and links. + +This module sits in the translator stage, between the templater that produces +the ISPyPSA network tables and the PyPSA model build. A flow path is a corridor +between two sub-regions and a REZ connection joins a renewable energy zone to its +parent sub-region; both are just transmission paths, so the module routes them +through one pipeline and tells them apart (via isp_type) only where the model +must treat them differently. + +The three sparse input tables (limits, expansion options, expansion costs) may +use blank key cells as wildcards; each is first put through _resolve_wildcards +(see translator/helpers.py) to expand those wildcards to concrete rows. + +_translate_network_to_links is the pipeline orchestrator and reads as the +step-by-step story; each helper's docstring carries an I/O example, and inline +comments mark the non-obvious modelling choices. """ import numpy as np import pandas as pd from ispypsa.config import ModelConfig -from ispypsa.translator.helpers import _annuitised_investment_costs +from ispypsa.translator.helpers import ( + _annuitised_investment_costs, + _resolve_wildcards, +) _LINK_COLUMNS = [ "isp_name", @@ -20,6 +33,7 @@ "bus1", "p_nom", "p_min_pu", + "p_max_pu", "build_year", "lifetime", "capital_cost", @@ -67,9 +81,9 @@ def _translate_network_to_links( ) -> tuple[pd.DataFrame, pd.DataFrame]: """Translates the network tables into PyPSA links and per-timeslice limits. - Existing links carry the winter_reference limit as p_nom; limits for the - other demand conditions are returned per unit of p_nom in the - link_timeslice_limits table. Expansion links are built per investment + Existing links carry the larger of their forward and reverse capacities as + p_nom; limits for each demand condition are returned per unit of p_nom in + the link_timeslice_limits table. Expansion links are built per investment period from the unified expansion options/costs tables (their total capacity is capped by the expansion-limit custom constraints built in ispypsa.translator.constraints). @@ -77,31 +91,38 @@ def _translate_network_to_links( Returns: Tuple of (links, link_timeslice_limits) in PyPSA friendly format. """ + # Existing rez_ids = _rez_geo_ids(ispypsa_tables["network_geography"]) paths = _drop_rez_paths_if_not_modelled( ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs ) - limits = ispypsa_tables["network_transmission_path_limits"] - static_limits = _extract_static_limits(limits) + limits = _resolve_path_limits( + ispypsa_tables["network_transmission_path_limits"], + paths, + config.network.transmission_default_limit, + ) + static_limits = _extract_max_capacities(limits) existing_links = _build_existing_links( paths, static_limits, rez_ids, - config.network.rez_to_sub_region_transmission_default_limit, config.temporal.range.start_year, ) link_timeslice_limits = _translate_timeslice_limits_to_pu(limits, existing_links) - options = _pivot_physical_expansion_options( - ispypsa_tables["network_expansion_options"] - ) - options = _filter_options_to_enabled_expansion( - options, + + # Expansion + enabled_ids = _enabled_expansion_element_ids( existing_links, config.network.transmission_expansion, config.network.rez_transmission_expansion, ) + options = _resolve_expansion_options( + ispypsa_tables["network_expansion_options"], enabled_ids + ) + options = _pair_forward_and_reverse_options(options) costs = _prepare_expansion_costs( ispypsa_tables["network_transmission_path_expansion_costs"], + enabled_ids, config.temporal.capacity_expansion.investment_periods, config.temporal.year_type, config.wacc, @@ -127,9 +148,17 @@ def _drop_rez_paths_if_not_modelled( """Drops REZ-to-parent paths when REZs are not modelled as discrete nodes. I/O Example: - paths (path_ids): CQ-NQ, Q1-NQ (Q1 is a REZ) + paths: + path_id geo_from geo_to + CQ-NQ CQ NQ + Q1-NQ Q1 NQ # Q1 is a REZ, so geo_from is in rez_ids + + rez_ids = {"Q1"} + + rezs = "attached_to_parent_node" returns: + path_id geo_from geo_to + CQ-NQ CQ NQ - rezs = "attached_to_parent_node" returns paths: CQ-NQ rezs = "discrete_nodes" returns paths unchanged. """ if rezs == "discrete_nodes": @@ -137,38 +166,78 @@ def _drop_rez_paths_if_not_modelled( return paths[~paths["geo_from"].isin(rez_ids)] -def _extract_static_limits(limits: pd.DataFrame) -> pd.DataFrame: - """Reduces the per-timeslice limits to one static capacity per path and - direction: the winter_reference limit. +def _resolve_path_limits( + limits: pd.DataFrame, paths: pd.DataFrame, default_limit: float +) -> pd.DataFrame: + """Resolves the sparse limits table to one row per modelled path, direction + and timeslice, then fills empty capacities with the system default. + + Blank path_id, direction or capacity cells are wildcards (see the + network_transmission_path_limits schema). _resolve_wildcards expands the + path_id and direction wildcards against the modelled paths and the two + directions; timeslice rides along, so a blank-timeslice row stays the + snapshot-level fallback that pypsa_build applies later. A blank capacity then + takes the configured transmission default — the nan_fill the schema declares + for the capacity column — which is how a no-data path (a single all-wildcard + row) becomes a default-limit link in both directions. + + I/O Example (blank cells are wildcards): + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 # specific to CQ-NQ forward + 900 # blank path + direction: a global default + N1-CNSW # no-data row: blank direction and capacity + + paths: path_id in {CQ-NQ, N1-CNSW}; default_limit = 100000 - Winter is used as the link's p_nom because it is the demand condition left - over when no summer timeslice is active — limits for the other conditions - are applied per unit of it. Collapsed all-NaN rows (paths with no limit - data) contribute nothing; paths absent from the result get the default - limit downstream. + returns: + path_id direction timeslice capacity + CQ-NQ forward 1400 # the specific row + CQ-NQ reverse 900 # the global default fills CQ-NQ reverse + N1-CNSW forward 100000 # the no-data row is more specific than the + N1-CNSW reverse 100000 # global default; its blank capacity -> default + """ + allowed_values = { + "path_id": list(paths["path_id"]), + "direction": ["forward", "reverse"], + } + limits = _resolve_wildcards(limits, allowed_values, ["capacity"]) + limits["capacity"] = limits["capacity"].fillna(default_limit) + return limits + + +def _extract_max_capacities(limits: pd.DataFrame) -> pd.DataFrame: + """Reduces the limits to one static capacity per path and direction: the + maximum across every row for that direction. + + Named timeslices and any timeslice = NaN fallback row are pooled — the max + has to include the fallback because at this stage we don't yet know which + snapshots it will cover. The larger of the two directions becomes the link's + p_nom downstream, so every per-unit limit lands in [-1, 1]. No-data paths + arrive already defaulted (see _resolve_path_limits), so they contribute a + forward and reverse max at the default limit like any other path. I/O Example: limits: path_id direction timeslice capacity CQ-NQ forward qld_peak_demand 1200 - CQ-NQ forward qld_winter_reference 1400 + CQ-NQ forward , 1400 # NaN-timeslice fallback CQ-NQ reverse qld_winter_reference 1910 - N1-CNSW , , # collapsed row returns: path_id direction capacity - CQ-NQ forward 1400 + CQ-NQ forward 1400 # fallback exceeds the named row CQ-NQ reverse 1910 """ - winter = limits[limits["timeslice"].str.endswith("_winter_reference", na=False)] - return winter.loc[:, ["path_id", "direction", "capacity"]] + # Max pools the named timeslices with the NaN fallback; the fallback has to be + # in the max because we don't yet know which snapshots it will cover. + return limits.groupby(["path_id", "direction"], as_index=False)["capacity"].max() def _build_existing_links( paths: pd.DataFrame, static_limits: pd.DataFrame, rez_ids: set[str], - default_limit: float, start_year: int, ) -> pd.DataFrame: """Builds one existing (non-extendable) PyPSA link per transmission path. @@ -183,197 +252,328 @@ def _build_existing_links( path_id direction capacity CQ-NQ forward 1400 CQ-NQ reverse 1910 + N1-CNSW forward 100000 # defaulted upstream + N1-CNSW reverse 100000 returns (abridged): isp_name name bus0 bus1 p_nom p_min_pu isp_type - CQ-NQ CQ-NQ_existing CQ NQ 1400 -1.364 flow_path - N1-CNSW N1-CNSW_existing N1 CNSW 100000 -1.0 rez # default limit + CQ-NQ CQ-NQ_existing CQ NQ 1910 0.0 flow_path # p_nom = max(1400, 1910) + N1-CNSW N1-CNSW_existing N1 CNSW 100000 0.0 rez """ links = paths.rename( columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} ) - links = _add_static_capacities(links, static_limits, default_limit) + links = _add_p_nom(links, static_limits) links["name"] = links["isp_name"] + "_existing" links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") links["build_year"] = start_year - 1 links["lifetime"] = np.inf links["capital_cost"] = np.nan links["p_nom_extendable"] = False + # Defaults; the limits come from link_timeslice_limits, which overrides them. + links["p_min_pu"] = 0.0 + links["p_max_pu"] = 1.0 return links -def _add_static_capacities( - links: pd.DataFrame, static_limits: pd.DataFrame, default_limit: float -) -> pd.DataFrame: - """Merges the static forward limit as p_nom and the reverse limit as p_min_pu. +def _add_p_nom(links: pd.DataFrame, static_limits: pd.DataFrame) -> pd.DataFrame: + """Sets p_nom to the larger of the forward and reverse static capacities. - Paths with no static limit in a direction get the default: ``default_limit`` - as p_nom (the path is constraint-modelled with no explicit physical limit) - and a symmetric -1.0 as p_min_pu. Zero-capacity paths (new parallel - corridors) also get a symmetric p_min_pu, avoiding a 0/0 division. + Taking the max of both directions keeps every per-unit limit in [-1, 1]. The + link's reverse limit is not set here — it is carried per snapshot in + link_timeslice_limits (a named timeslice or the timeslice = NaN fallback), + which under the coverage contract (Open-ISP/ISPyPSA#123) sets p_min_pu on + every snapshot, leaving the link's static p_min_pu at its inert default. I/O Example: - links (isp_name): CQ-NQ, N1-CNSW - static_limits: CQ-NQ forward 1400, CQ-NQ reverse 1910 + links: + isp_name + CQ-NQ + PAR-NEW # new parallel corridor, zero capacity + + static_limits: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + PAR-NEW forward 0 + PAR-NEW reverse 0 returns: - isp_name p_nom p_min_pu - CQ-NQ 1400 -1.364 # -1910/1400 - N1-CNSW 100000 -1.0 # defaults + isp_name p_nom + CQ-NQ 1910 # max(1400, 1910) + PAR-NEW 0 """ directions = static_limits.pivot( index="path_id", columns="direction", values="capacity" ).reindex(columns=["forward", "reverse"]) - links = links.merge( - directions, left_on="isp_name", right_index=True, how="left" - ).rename(columns={"forward": "p_nom"}) - links["p_min_pu"] = np.where( - links["reverse"].notna() & (links["p_nom"] > 0), - -1.0 * links["reverse"] / links["p_nom"], - -1.0, - ) - links["p_nom"] = links["p_nom"].fillna(default_limit) - return links.drop(columns=["reverse"]) + links = links.merge(directions, left_on="isp_name", right_index=True, how="left") + # Larger of the two directions; dividing each limit by it keeps every per-unit + # value in [-1, 1]. + links["p_nom"] = links[["forward", "reverse"]].max(axis=1) + return links.drop(columns=["forward", "reverse"]) def _translate_timeslice_limits_to_pu( limits: pd.DataFrame, existing_links: pd.DataFrame ) -> pd.DataFrame: - """Expresses each per-timeslice limit per unit of its link's p_nom. + """Expresses each limit per unit of its link's p_nom. Forward limits become p_max_pu values and reverse limits p_min_pu values. - pypsa_build expands them into per-snapshot series via the - timeslice_snapshots mapping; snapshots outside any of a link's tagged - timeslices keep the static (winter_reference) limit. Zero-p_nom links - (new parallel corridors) are skipped — all their limits are zero and the - per-unit form is undefined. + Named-timeslice rows carry their timeslice; a timeslice = NaN row is the + fallback applied to snapshots no named timeslice covers (the coverage + contract is Open-ISP/ISPyPSA#123). pypsa_build expands these into per-snapshot + series via the timeslice_snapshots mapping. Zero-p_nom links (new parallel + corridors) are skipped — all their limits are zero and the per-unit form is + undefined. I/O Example: limits: path_id direction timeslice capacity CQ-NQ forward qld_peak_demand 1200 CQ-NQ forward qld_winter_reference 1400 - CQ-NQ reverse qld_peak_demand 1440 + CQ-NQ reverse , 1000 # NaN-timeslice fallback existing_links (abridged): - isp_name name p_nom - CQ-NQ CQ-NQ_existing 1400 + isp_name name p_nom + CQ-NQ CQ-NQ_existing 1400 returns: name attribute timeslice value - CQ-NQ_existing p_max_pu qld_peak_demand 0.857 - CQ-NQ_existing p_max_pu qld_winter_reference 1.0 - CQ-NQ_existing p_min_pu qld_peak_demand -1.029 + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 # 1200/1400 + CQ-NQ_existing p_max_pu qld_winter_reference 1.0 # 1400/1400 + CQ-NQ_existing p_min_pu , -0.714 # fallback, -1000/1400 """ - rows = limits.dropna(subset=["timeslice", "capacity"]) - rows = rows.merge( + rows = limits.merge( existing_links.loc[:, ["isp_name", "name", "p_nom"]], left_on="path_id", right_on="isp_name", ) + # Zero-p_nom links (new parallel corridors) have no per-unit form, so they + # contribute no timeslice rows. rows = rows[rows["p_nom"] > 0] rows["attribute"] = rows["direction"].map( {"forward": "p_max_pu", "reverse": "p_min_pu"} ) + # Reverse flow is negative, so reverse limits become negative p_min_pu values. sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) rows["value"] = sign * rows["capacity"] / rows["p_nom"] return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) -def _pivot_physical_expansion_options(expansion_options: pd.DataFrame) -> pd.DataFrame: - """Pairs each physical path's forward and reverse expansion rows into one row. +def _enabled_expansion_element_ids( + existing_links: pd.DataFrame, + transmission_expansion: bool, + rez_transmission_expansion: bool, +) -> list[str]: + """The isp_names of the existing links whose expansion the config enables. - constraint_relaxation rows are not physical paths — they are translated - into custom-constraint relaxation generators by - ispypsa.translator.constraints. + ``transmission_expansion`` gates flow paths between (sub)regions; + ``rez_transmission_expansion`` gates REZ connection paths. The result is the + allowed expansion_id set the options and costs are resolved against, so an + option or cost for a disabled or non-modelled element drops out there. I/O Example: - expansion_options: + existing_links: + isp_name isp_type + CQ-NQ flow_path + Q1-NQ rez + + transmission_expansion=True, rez_transmission_expansion=False + returns ["CQ-NQ"] # rez path Q1-NQ gated out + """ + enabled = set() + if transmission_expansion: + is_flow_path = existing_links["isp_type"] == "flow_path" + enabled |= set(existing_links.loc[is_flow_path, "isp_name"]) + if rez_transmission_expansion: + is_rez = existing_links["isp_type"] == "rez" + enabled |= set(existing_links.loc[is_rez, "isp_name"]) + return sorted(enabled) + + +def _resolve_expansion_options( + options: pd.DataFrame, enabled_ids: list[str] +) -> pd.DataFrame: + """Resolves the expansion-options wildcards to one row per enabled element + and physical direction. + + A blank expansion_id is a system-wide default option, and a blank + expansion_type covers both directions of one element in a single (symmetric) + row. constraint_relaxation rows are not physical paths — they are set aside + first and become relaxation generators in ispypsa.translator.constraints. + _resolve_wildcards then expands the blanks against the enabled elements and + the two physical directions; an option for a disabled or non-modelled element + falls outside the allowed values and drops out (config-driven selection, so + not logged as a drop). + + Each element's forward and reverse must form a coherent pair from one option + (the schema's expansion_options_pair_forward_and_reverse rule); + _check_both_directions_defined and _check_forward_reverse_share_an_option + stand in for that rule once resolved, until schema validation lands. + + I/O Example (blank cells are wildcards; each element's forward and reverse + share one expansion_option): + options: expansion_id expansion_type allowed_expansion expansion_option - CQ-NQ forward 1000 Option 1 - CQ-NQ reverse 900 Option 1 - SWQLD1 constraint_relaxation 500 Option 2 + CQ-NQ forward 1000 BigLine + CQ-NQ reverse 800 BigLine + forward 500 Default # blank id: + reverse 400 Default # a default pair + SWQLD1 constraint_relaxation 400 Relax # standalone, dropped + + enabled_ids = ["CQ-NQ", "Q1-NQ"] + + returns (CQ-NQ keeps its BigLine pair; Q1-NQ inherits the Default pair): + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 BigLine + CQ-NQ reverse 800 BigLine + Q1-NQ forward 500 Default + Q1-NQ reverse 400 Default + """ + # constraint_relaxation options are routed to ispypsa.translator.constraints, + # not dropped, so they are set aside before wildcard resolution (which would + # otherwise log them as dropped rows). + options = options[options["expansion_type"] != "constraint_relaxation"] + allowed_values = { + "expansion_id": enabled_ids, + "expansion_type": ["forward", "reverse"], + } + options = _resolve_wildcards( + options, + allowed_values, + ["allowed_expansion", "expansion_option"], + expected_drops=("expansion_id",), + ) + _check_both_directions_defined(options) + _check_forward_reverse_share_an_option(options) + return options - returns: - expansion_id forward_capacity reverse_capacity - CQ-NQ 1000 900 + +def _check_both_directions_defined(options: pd.DataFrame) -> None: + """Raises if any element's resolved options define only one direction. + + The schema's expansion_options_pair_forward_and_reverse rule requires an + expansion_id that defines one direction to define the other; an element + arriving with only a forward (or only a reverse) row would otherwise produce + an expansion link with a NaN rating in the missing direction. This stands in + until schema validation lands. + + I/O Example: + options with only a CQ-NQ forward row -> raises (no CQ-NQ reverse). """ - physical = expansion_options[ - expansion_options["expansion_type"].isin(["forward", "reverse"]) - ] - options = ( - physical.pivot( - index="expansion_id", columns="expansion_type", values="allowed_expansion" + directions = options.groupby("expansion_id")["expansion_type"].nunique() + one_directional = sorted(directions[directions < 2].index) + if one_directional: + raise ValueError( + f"Expansion options define only one direction for {one_directional}; " + "each element needs both forward and reverse (a blank expansion_type " + "row covers both)." ) - .reindex(columns=["forward", "reverse"]) - .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) - ) - options.columns.name = None - return options.reset_index() -def _filter_options_to_enabled_expansion( - options: pd.DataFrame, - existing_links: pd.DataFrame, - transmission_expansion: bool, - rez_transmission_expansion: bool, -) -> pd.DataFrame: - """Keeps the expansion options enabled by the network config flags. +def _check_forward_reverse_share_an_option(options: pd.DataFrame) -> None: + """Raises if any element's resolved forward and reverse trace to different + options. - ``transmission_expansion`` gates paths between (sub)regions; - ``rez_transmission_expansion`` gates REZ connection paths. Options for - paths not in the model (e.g. REZ paths dropped under - attached_to_parent_node) drop out here too, as they match no link. + Forward and reverse for an expansion_id must come from one physical option, so + the single cost keyed on expansion_id applies to a coherent pair. Wildcard + resolution resolves the two directions independently, so a malformed input + could pair a forward from one option with a reverse from another; this guard + catches that. The schema's expansion_options_pair_forward_and_reverse rule is + the upstream enforcement — this stands in until schema validation lands. I/O Example: - options (expansion_ids): CQ-NQ, Q1-NQ - existing_links: CQ-NQ isp_type=flow_path, Q1-NQ isp_type=rez + options with CQ-NQ forward from "BigLine" and CQ-NQ reverse from "Default" + -> raises (the two directions disagree on the option). + """ + options_per_element = options.groupby("expansion_id")["expansion_option"].nunique() + mismatched = sorted(options_per_element[options_per_element > 1].index) + if mismatched: + raise ValueError( + f"Forward and reverse expansion options disagree for {mismatched}; " + "each element's forward and reverse must come from the same option." + ) - transmission_expansion=True, rez_transmission_expansion=False - returns options: CQ-NQ + +def _pair_forward_and_reverse_options(options: pd.DataFrame) -> pd.DataFrame: + """Pairs each element's forward and reverse expansion rows into one row. + + The options arrive already resolved to physical forward/reverse rows (see + _resolve_expansion_options), so this only reshapes them. + + I/O Example: + options: + expansion_id expansion_type allowed_expansion + CQ-NQ forward 1000 + CQ-NQ reverse 900 + + returns: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 """ - rez_path_ids = set( - existing_links.loc[existing_links["isp_type"] == "rez", "isp_name"] - ) - flow_path_ids = set( - existing_links.loc[existing_links["isp_type"] == "flow_path", "isp_name"] + paired = ( + options.pivot( + index="expansion_id", columns="expansion_type", values="allowed_expansion" + ) + .reindex(columns=["forward", "reverse"]) + .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) ) - enabled = set() - if transmission_expansion: - enabled |= flow_path_ids - if rez_transmission_expansion: - enabled |= rez_path_ids - return options[options["expansion_id"].isin(enabled)] + paired.columns.name = None + return paired.reset_index() def _prepare_expansion_costs( expansion_costs: pd.DataFrame, + enabled_ids: list[str], investment_periods: list[int], year_type: str, wacc: float, asset_lifetime: int, ) -> pd.DataFrame: - """Filters the long-format expansion costs to the investment periods and - annuitises them. - - The ``year`` column holds financial-year ending years as ints, matching - the investment period labels used with ``year_type="fy"``. - - I/O Example: + """Resolves the expansion-cost wildcards to the enabled elements and + investment periods, then annuitises them. + + Blank expansion_id or year cells are wildcards (see the + network_transmission_path_expansion_costs schema): an empty expansion_id is a + table-wide default cost, an empty year a static cost across the investment + periods. _resolve_wildcards expands them against the enabled elements and the + investment periods, which drops any cost for a disabled element, a constraint + group (routed to ispypsa.translator.constraints) or a year outside the + investment periods — all designed selection, so none of it is logged as a + drop. A blank cost then resolves to free — the + nan_fill the schema declares for the cost column. The year column holds + financial-year ending years as ints, matching the investment period labels + used with year_type="fy". + + I/O Example (a blank year is a static cost across the investment periods): expansion_costs: expansion_id year cost - CQ-NQ 2025 1000 - CQ-NQ 2026 1010 + CQ-NQ 1000 # blank year: applies to every investment period + CQ-NQ 2028 1200 # a specific-year override + Q1-NQ 2025 900 # 2025 is not an investment period -> dropped + + enabled_ids=["CQ-NQ", "Q1-NQ"], investment_periods=[2026, 2028], wacc, asset_lifetime - investment_periods=[2026], wacc=0.07, asset_lifetime=30 returns: + returns (cost annuitised; Q1-NQ falls away, its only row was out of period): expansion_id year capital_cost - CQ-NQ 2026 81.4 + CQ-NQ 2026 annuitise(1000) # the static row fills 2026 + CQ-NQ 2028 annuitise(1200) # the 2028 override beats the static row """ if year_type != "fy": raise NotImplementedError( f"Network expansion costs are not implemented for year_type: {year_type}" ) - costs = expansion_costs[expansion_costs["year"].isin(investment_periods)].copy() + allowed_values = {"expansion_id": enabled_ids, "year": investment_periods} + costs = _resolve_wildcards( + expansion_costs, + allowed_values, + ["cost"], + expected_drops=("expansion_id", "year"), + ) + # Every surviving year is now an investment period; cast so expansion link + # names read e.g. CQ-NQ_exp_2026 rather than CQ-NQ_exp_2026.0. + costs["year"] = costs["year"].astype("int64") + costs["cost"] = costs["cost"].fillna(0.0) costs["capital_cost"] = costs["cost"].apply( lambda cost: _annuitised_investment_costs(cost, wacc, asset_lifetime) ) @@ -387,12 +587,14 @@ def _build_expansion_links( ) -> pd.DataFrame: """Builds one extendable PyPSA link per expandable path and investment period. - Each expansion link is unbounded on its own — the expansion-limit custom + p_nom starts at 0 and is unbounded on its own — the expansion-limit custom constraints built in ispypsa.translator.constraints cap the p_nom built - across a path's expansion links at the selected option's forward capacity. - Asymmetric options are modelled with p_min_pu = -reverse/forward, so - however much forward capacity is built, reverse capacity scales in the - option's proportion. + across a path's expansion links at max(forward, reverse) of the selected + option. p_max_pu and p_min_pu are the forward and reverse capacities per unit + of that max, so whatever p_nom is built delivers forward and reverse flow in + the option's proportion, both bounded by [-1, 1]. Using the max as the + denominator means only a both-zero option divides by zero, which falls back + to symmetric defaults. I/O Example: options: @@ -403,11 +605,13 @@ def _build_expansion_links( expansion_id year capital_cost CQ-NQ 2026 81.4 - existing_links (abridged): CQ-NQ bus0=CQ bus1=NQ carrier=AC isp_type=flow_path + existing_links (abridged): + isp_name bus0 bus1 carrier isp_type + CQ-NQ CQ NQ AC flow_path returns (abridged): - isp_name name p_nom p_nom_extendable p_min_pu build_year capital_cost - CQ-NQ CQ-NQ_exp_2026 0.0 True -0.9 2026 81.4 + isp_name name p_nom p_nom_extendable p_max_pu p_min_pu build_year capital_cost + CQ-NQ CQ-NQ_exp_2026 0.0 True 1.0 -0.9 2026 81.4 """ links = costs.merge(options, on="expansion_id") links = links.merge( @@ -416,9 +620,19 @@ def _build_expansion_links( right_on="isp_name", ) links["name"] = links["expansion_id"] + "_exp_" + links["year"].astype(str) + # Starts at 0 and is unbounded on its own; ispypsa.translator.constraints caps the + # p_nom built across a path's expansion links at max(forward, reverse) of the option. links["p_nom"] = 0.0 links["p_nom_extendable"] = True - links["p_min_pu"] = -1.0 * links["reverse_capacity"] / links["forward_capacity"] + # Per unit of max(forward, reverse) keeps both ratings in [-1, 1]; only a both-zero + # option hits the symmetric divide-by-zero fallback. + capacity = links[["forward_capacity", "reverse_capacity"]].max(axis=1) + links["p_max_pu"] = np.where( + capacity > 0, links["forward_capacity"] / capacity, 1.0 + ) + links["p_min_pu"] = np.where( + capacity > 0, -1.0 * links["reverse_capacity"] / capacity, -1.0 + ) links["build_year"] = links["year"] links["lifetime"] = np.inf return links diff --git a/src/ispypsa/translator/renewable_energy_zones.py b/src/ispypsa/translator/renewable_energy_zones.py index c36e346f..6867c97a 100644 --- a/src/ispypsa/translator/renewable_energy_zones.py +++ b/src/ispypsa/translator/renewable_energy_zones.py @@ -27,7 +27,7 @@ def _translate_renewable_energy_zone_build_limits_to_links( # Create existing links from renewable energy zone build limits existing_links = _translate_existing_rez_connections_to_links( renewable_energy_zone_build_limits, - config.network.rez_to_sub_region_transmission_default_limit, + config.network.transmission_default_limit, config.temporal.range.start_year, ) @@ -62,7 +62,7 @@ def _translate_renewable_energy_zone_build_limits_to_links( def _translate_existing_rez_connections_to_links( renewable_energy_zone_build_limits: pd.DataFrame, - rez_to_sub_region_transmission_default_limit: float, + transmission_default_limit: float, start_year: int, ) -> pd.DataFrame: """Process existing REZ connection limits to PyPSA links. @@ -70,7 +70,7 @@ def _translate_existing_rez_connections_to_links( Args: renewable_energy_zone_build_limits: `ISPyPSA` formatted pd.DataFrame detailing Renewable Energy Zone transmission limits. - rez_to_sub_region_transmission_default_limit: float specifying the transmission + transmission_default_limit: float specifying the transmission limit to use for rez to subregion connections when an explicit limit is not given in the inputs. @@ -86,7 +86,7 @@ def _translate_existing_rez_connections_to_links( # custom constraints are given a very large capacity links["isp_type"] = "rez" links.loc[links["p_nom"].isna(), "isp_type"] = "rez_no_limit" - links["p_nom"] = links["p_nom"].fillna(rez_to_sub_region_transmission_default_limit) + links["p_nom"] = links["p_nom"].fillna(transmission_default_limit) links["p_min_pu"] = -1.0 links["build_year"] = start_year - 1 diff --git a/src/ispypsa/validation/schemas/network_expansion_options.yaml b/src/ispypsa/validation/schemas/network_expansion_options.yaml index 02108c82..6b391bbd 100644 --- a/src/ispypsa/validation/schemas/network_expansion_options.yaml +++ b/src/ispypsa/validation/schemas/network_expansion_options.yaml @@ -4,7 +4,16 @@ unique: - [expansion_id, expansion_type] description: > Defines the selected expansion option and allowed capacity increase for - expandable network elements. + expandable network elements, looked up per element and expansion type. + + The key columns — expansion_id and expansion_type — identify the combination an + option applies to. Either may be left empty to act as a wildcard: an empty + expansion_id is a table-wide default across all elements, and an empty + expansion_type applies to forward, reverse and constraint_relaxation alike. + Where several rows match, the most specific (fewest empty keys) wins; the + custom_validation rule requires that resolution to be unambiguous. Unlike the + limits and costs tables there is no coverage requirement — expansion is + optional. Covers both physical transmission paths and group constraints that can be relaxed through expansion. Physical paths emit two rows (forward and @@ -23,10 +32,26 @@ description: > If absent: No network elements can be expanded by the model. +custom_validation: + - name: expansion_options_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows.) + - name: expansion_options_pair_forward_and_reverse + description: > + Forward and reverse for an element must come from one option, so each + expansion_id (blank included) that defines one direction must define the + other with the same expansion_option. A blank expansion_type covers both + directions in one row and satisfies this alone; constraint_relaxation has no + direction and stands alone. This keeps an element's resolved forward and + reverse tied to the same option, so the single cost keyed on expansion_id + applies to a coherent pair. columns: expansion_id: type: string - required: true + required: false allowed_values_from: - network_transmission_paths: path_id - custom_constraints_rhs: constraint_id @@ -34,15 +59,23 @@ columns: Identifier for the expandable network element. Maps to path_id in network_transmission_paths for physical paths, or to constraint_id in custom_constraints_rhs for group constraints. + + If absent (or empty): + Wildcard — applies to every expandable element no more specific row covers + (a system-wide default option). expansion_type: type: string - required: true + required: false allowed_values: [forward, reverse, constraint_relaxation] description: > Which direction or constraint relaxation this row's ``allowed_expansion`` applies to. ``forward`` and ``reverse`` are the two directions of a physical path; ``constraint_relaxation`` is the single relaxation of a group constraint. + + If absent (or empty): + Wildcard — applies to forward, reverse and constraint_relaxation alike for + any combination no more specific row covers. allowed_expansion: type: float required: true diff --git a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml index d9ad83d3..388878a9 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml @@ -3,7 +3,15 @@ required: false unique: - [expansion_id, year] description: > - Time-varying annualised expansion costs for network elements. + Time-varying annualised expansion costs for network elements, looked up per + expandable element and investment period. + + The key columns — expansion_id and year — identify the combination a cost + applies to. Either may be left empty to act as a wildcard: an empty year is a + static cost across all investment periods, and an empty expansion_id is a + table-wide default across all elements. Where several rows match, the most + specific (fewest empty keys) wins; the custom_validation rules require that + resolution to be both unambiguous and complete. Covers both physical transmission paths and group constraints, merging flow path and REZ transmission expansion costs into a single long-format table. @@ -20,26 +28,54 @@ description: > increase used to express costs per MW. If absent: - Network expansion is treated as free. + No network elements are expandable; a populated options table requires matching + costs (see custom_validation). +custom_validation: + - name: costs_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows.) + - name: costs_cover_every_element_and_investment_period + description: > + Coverage spans every expandable element (an expansion_id present in + network_expansion_options once its own wildcards resolve) and every + model-config investment period; a single row with both keys empty covers it + all. columns: expansion_id: type: string - required: true + required: false allowed_values_from: - network_expansion_options: expansion_id - description: Identifier for the expandable network element. + description: > + Identifier for the expandable network element. + + If absent (or empty): + Wildcard — applies to every expandable element no more specific row covers + (a table-wide default). year: type: int - required: true - description: Year the cost applies to. + required: false + description: > + Investment period (financial-year ending year) the cost applies to. + + If absent (or empty): + Wildcard — a static cost applied to every investment period the element has + no explicit year row for. cost: type: float - required: true + required: false units: $/MW gte: 0.0 + nan_fill: 0.0 description: > Annualised expansion cost, expressed per MW of the maximum directional capacity: total option cost divided by max(forward, reverse) of the selected option's allowed_expansion. For symmetric options (forward == reverse) this is unambiguous; for asymmetric options the divisor is whichever direction has the larger capacity increase. + + If absent (or empty): + The expansion is free (equivalent to a cost of 0). diff --git a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml index 977a8bbf..8c41bae4 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml @@ -3,34 +3,59 @@ required: false unique: - [path_id, direction, timeslice] description: > - Transmission capacity limits for each path, by direction and optionally by - demand condition (timeslice). + Transmission capacity limits, looked up per path, flow direction and demand + condition (timeslice). - A limit with a timeslice applies to that demand condition only; a limit with - no timeslice is a fallback for the conditions no per-timeslice limit covers - (see the timeslice column). A path may carry either or both. In the default - IASR data, flow-path limits vary by demand condition while REZ-to-subregion - connection limits are static, but that reflects the source tables rather than - a schema rule. + The three key columns — path_id, direction and timeslice — identify the + combination a limit applies to. Any key may be left empty to act as a wildcard + matching every value of that dimension: an empty timeslice is a fallback across + demand conditions, an empty direction applies to both directions, and an empty + path_id is a table-wide default. Where several rows match a combination the most + specific (fewest empty keys) wins; the custom_validation rules require that + resolution to be both unambiguous and complete. Source tables: - `flow_path_transfer_capability` - `initial_transmission_limits` If absent: - No transmission capacity limits are enforced. + There are no transmission paths to limit; a populated paths table requires + matching limits (see custom_validation). +custom_validation: + - name: limits_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows to all three keys.) + - name: limits_cover_every_path_direction_and_timeslice + description: > + A path that names timeslices must name them all for its region, or add an empty-timeslice + fallback, in both directions; a single all-empty row covers the whole space. + Because the timeslices table's windows partition the year, covering every + region timeslice_id covers every snapshot downstream. columns: path_id: type: string - required: true + required: false allowed_values_from: - network_transmission_paths: path_id - description: Transmission path identifier. + description: > + Transmission path the limit applies to. + + If absent (or empty): + Wildcard — applies to every path no more specific row covers (a table-wide + default). direction: type: string - required: true + required: false allowed_values: [forward, reverse] - description: Power flow direction relative to geo_from and geo_to. + description: > + Power flow direction relative to geo_from and geo_to. + + If absent (or empty): + Wildcard — applies to both directions for any combination no more specific + row covers. timeslice: type: string required: false @@ -38,21 +63,22 @@ columns: - timeslices: timeslice_id description: > Demand condition the limit applies to: the limit binds only at snapshots - inside the timeslice's active windows. + inside the timeslice's active windows. A timeslice_id's region is the prefix + before its first underscore (e.g. qld in qld_peak_demand). If absent (or empty): - The limit is a fallback for this path_id and direction, applying at - snapshots not covered by any non-NaN timeslice row for the same path_id - and direction. A path_id and direction may carry both a no-timeslice row - and per-timeslice rows; where a per-timeslice row covers a snapshot its - limit takes precedence, and the no-timeslice limit fills the remainder. + Wildcard — a fallback across demand conditions, applying at every timeslice + no more specific row covers for the same path and direction. capacity: type: float required: false units: MW gte: 0.0 + nan_fill: {config: network.transmission_default_limit} description: > - Transfer capability limit. + Transfer capability limit for the resolved combination. If absent (or empty): - The path is constraint-modelled with no explicit physical limit. + The combination takes the configured transmission default + (network.transmission_default_limit). A value sentinel, distinct from 0, + which is a genuine zero-capacity corridor. diff --git a/tests/conftest.py b/tests/conftest.py index 53a8c7fb..e21cecd4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -333,7 +333,7 @@ def sample_model_config(): annuitisation_lifetime=25, transmission_expansion=True, rez_transmission_expansion=True, - rez_to_sub_region_transmission_default_limit=1000000.0, + transmission_default_limit=1000000.0, ), unserved_energy=UnservedEnergyConfig(cost=10000, max_per_node=10000), ) diff --git a/tests/test_cli/cli_test_helpers.py b/tests/test_cli/cli_test_helpers.py index 31b0de9b..e08815cc 100644 --- a/tests/test_cli/cli_test_helpers.py +++ b/tests/test_cli/cli_test_helpers.py @@ -146,7 +146,7 @@ def build_mock_config( "nodes": {"regional_granularity": granularity, "rezs": "discrete_nodes"}, "transmission_expansion_limit_override": None, "rez_connection_expansion_limit_override": None, - "rez_to_sub_region_transmission_default_limit": 1e6, + "transmission_default_limit": 1e6, }, "temporal": { "year_type": "fy", @@ -329,7 +329,7 @@ def create_config_with_granularity(tmp_path, mock_workbook_file, granularity): "nodes": {"regional_granularity": granularity, "rezs": "discrete_nodes"}, "transmission_expansion_limit_override": None, "rez_connection_expansion_limit_override": None, - "rez_to_sub_region_transmission_default_limit": 1e6, + "transmission_default_limit": 1e6, }, "temporal": { "year_type": "fy", diff --git a/tests/test_config/test_pydantic_model_config.py b/tests/test_config/test_pydantic_model_config.py index be5c776c..82392582 100644 --- a/tests/test_config/test_pydantic_model_config.py +++ b/tests/test_config/test_pydantic_model_config.py @@ -71,7 +71,7 @@ def get_valid_config(): "regional_granularity": "sub_regions", "rezs": "discrete_nodes", }, - "rez_to_sub_region_transmission_default_limit": 1e6, + "transmission_default_limit": 1e6, }, "temporal": { "year_type": "fy", @@ -167,7 +167,7 @@ def invalid_rez_transmission_expansion(config): def invalid_rez_transmission_limit(config): - config["network"]["rez_to_sub_region_transmission_default_limit"] = "help" + config["network"]["transmission_default_limit"] = "help" return config, ValidationError diff --git a/tests/test_translator/ispypsa_config.yaml b/tests/test_translator/ispypsa_config.yaml index 43654f2b..3e02865c 100644 --- a/tests/test_translator/ispypsa_config.yaml +++ b/tests/test_translator/ispypsa_config.yaml @@ -58,9 +58,9 @@ network: # "discrete_nodes": REZs are added as network nodes to model REZ transmission limits # "attached_to_parent_node": REZ resources are attached to their parent node (sub-region or NEM region) rezs: discrete_nodes - # Line capacity limit for rez to node connections that have their limit's modelled - # through custom constraint (MW). - rez_to_sub_region_transmission_default_limit: 1e5 + # Default transmission capacity limit (MW) for a path with no explicit limit in + # the inputs (e.g. REZ connections whose limit is modelled via custom constraints). + transmission_default_limit: 1e5 temporal: year_type: fy range: diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py index 8c1ae216..36415b24 100644 --- a/tests/test_translator/test_network.py +++ b/tests/test_translator/test_network.py @@ -3,6 +3,7 @@ from ispypsa.translator.helpers import _annuitised_investment_costs from ispypsa.translator.network import ( + _resolve_expansion_options, _translate_network_geography_to_buses, _translate_network_to_links, ) @@ -108,12 +109,12 @@ def test_translate_network_to_links(csv_str_to_df, sample_model_config): ) expected_links = csv_str_to_df(f""" - isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type - CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, -1.364286, 2025, inf, , False, flow_path - Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, -1.0, 2025, inf, , False, rez - N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, -1.0, 2025, inf, , False, rez - CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path - Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1910, 0.0, 1.0, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, 0.0, 1.0, 2025, inf, , False, rez + N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, 0.0, 1.0, 2025, inf, , False, rez + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez """) pd.testing.assert_frame_equal( links.sort_values("name").reset_index(drop=True), @@ -124,12 +125,14 @@ def test_translate_network_to_links(csv_str_to_df, sample_model_config): expected_limits = csv_str_to_df(""" name, attribute, timeslice, value - CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 - CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.928571 - CQ-NQ_existing, p_max_pu, qld_winter_reference, 1.0 - CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 - CQ-NQ_existing, p_min_pu, qld_summer_typical, -1.142857 - CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.364286 + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.628272 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.680628 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.732984 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.753927 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -0.837696 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + N1-CNSW_existing,p_max_pu, , 1.0 + N1-CNSW_existing,p_min_pu, , -1.0 Q1-NQ_existing, p_max_pu, qld_winter_reference, 1.0 Q1-NQ_existing, p_min_pu, qld_winter_reference, -1.0 """) @@ -144,6 +147,238 @@ def test_translate_network_to_links(csv_str_to_df, sample_model_config): ) +def test_translate_network_to_links_p_nom_when_forward_exceeds_reverse( + csv_str_to_df, sample_model_config +): + """p_nom is the larger of the two directions. With forward (peak 1500) above + reverse, p_nom takes it, so a forward limit above winter stays at p_max_pu + 1.0 rather than over-crediting the link above its rating + (Open-ISP/ISPyPSA#123, item 3).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Forward peak (1500) exceeds winter (1400): winter is no longer the max. + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1500 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_winter_reference, 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1500, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 1.0 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.933333 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -0.666667 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_p_nom_when_reverse_exceeds_forward( + csv_str_to_df, sample_model_config +): + """p_nom is the larger of the two directions. With reverse (1400) above + forward (1000), p_nom takes reverse, so the forward limit is p_max_pu < 1 and + the reverse limit reaches p_min_pu -1.0 in link_timeslice_limits — both + per-unit limits within [-1, 1].""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Reverse winter (1400) exceeds forward winter (1000): reverse sets p_nom. + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_winter_reference, 1000 + CQ-NQ, reverse, qld_winter_reference, 1400 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.714286 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_nan_timeslice_fallback( + csv_str_to_df, sample_model_config +): + """A timeslice = NaN row is a fallback limit: it counts towards p_nom (we + don't yet know which snapshots it covers) and flows into link_timeslice_limits + carrying a NaN timeslice (Open-ISP/ISPyPSA#123).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Forward: a named peak row plus a NaN-timeslice fallback (1400, the larger). + # Reverse: only a NaN-timeslice fallback (1000). + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, , 1400 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # p_nom = max(forward 1400, reverse 1000) = 1400; the reverse fallback is in + # link_timeslice_limits, so the static p_min_pu stays at 0.0. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + # The fallback rows keep their NaN timeslice (blank field after the comma). + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.714286 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute", "value"]).reset_index( + drop=True + ), + expected_limits.sort_values(["attribute", "value"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_zero_forward_keeps_reverse( + csv_str_to_df, sample_model_config +): + """A link with a zero forward limit and a non-zero reverse limit takes its + reverse capacity as p_nom, so the reverse capability is preserved rather than + collapsing the link to zero flow in both directions.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_winter_reference, 0 + CQ-NQ, reverse, qld_winter_reference, 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # p_nom = max(0, 1000) = 1000; reverse fully available (via the timeslice + # limit), forward pinned to 0. Static p_min_pu stays at 0.0. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1000, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.0 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_no_data_path_gets_default_fallback( + csv_str_to_df, sample_model_config +): + """A path with no limit data (a collapsed all-NaN row) is normalised to a + symmetric timeslice = NaN fallback at the default limit, so it tiles the year + the same way a data path does (Open-ISP/ISPyPSA#123).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + N1-CNSW, N1, CNSW, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + N1-CNSW, , , + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # No-data path takes the default limit (1000000) in both directions. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + N1-CNSW, N1-CNSW_existing, AC, N1, CNSW, 1000000, 0.0, 1.0, 2025, inf, , False, rez + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + # Both directions get a NaN-timeslice fallback per unit of p_nom. + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + N1-CNSW_existing, p_max_pu, , 1.0 + N1-CNSW_existing, p_min_pu, , -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + def test_translate_network_to_links_expansion_disabled( csv_str_to_df, sample_model_config ): @@ -196,34 +431,33 @@ def test_translate_network_to_links_rezs_attached_to_parent_node( def test_translate_network_to_links_zero_capacity_parallel_path( csv_str_to_df, sample_model_config ): - """New parallel corridors arrive as zero-capacity paths: p_nom 0 with a - symmetric p_min_pu (no 0/0 division), and no per-unit timeslice rows.""" + """A new parallel corridor has zero existing capacity, given as a + timeslice = NaN fallback of 0 in both directions. It becomes an inert + existing link at p_nom 0 (its buildable capacity is modelled by expansion + links), and the zero-p_nom link is skipped when translating per-timeslice + limits, so it yields no link_timeslice_limits rows and no 0/0 division.""" ispypsa_tables = _network_tables(csv_str_to_df) ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" path_id, geo_from, geo_to, carrier CNSW-SNW, CNSW, SNW, AC """) + # Zero existing capacity as a timeslice = NaN fallback (covers the year). ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" - path_id, direction, timeslice, capacity - CNSW-SNW, forward, nsw_winter_reference, 0 - CNSW-SNW, reverse, nsw_winter_reference, 0 - """) - ispypsa_tables["network_expansion_options"] = pd.DataFrame( - columns=[ - "expansion_id", - "expansion_type", - "allowed_expansion", - "expansion_option", - ] - ) + path_id, direction, timeslice, capacity + CNSW-SNW, forward, , 0 + CNSW-SNW, reverse, , 0 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) links, link_timeslice_limits = _translate_network_to_links( ispypsa_tables, sample_model_config ) expected_links = csv_str_to_df(""" - isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type - CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, -1.0, 2025, inf, , False, flow_path + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, 0.0, 1.0, 2025, inf, , False, flow_path """) pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) @@ -239,17 +473,12 @@ def test_translate_network_to_links_empty_expansion_tables( csv_str_to_df, sample_model_config ): ispypsa_tables = _network_tables(csv_str_to_df) - ispypsa_tables["network_expansion_options"] = pd.DataFrame( - columns=[ - "expansion_id", - "expansion_type", - "allowed_expansion", - "expansion_option", - ] - ) - ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( - columns=["expansion_id", "year", "cost"] - ) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) @@ -262,30 +491,25 @@ def test_translate_network_to_links_empty_expansion_tables( def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): ispypsa_tables = _network_tables(csv_str_to_df) - ispypsa_tables["network_transmission_paths"] = pd.DataFrame( - columns=["path_id", "geo_from", "geo_to", "carrier"] - ) - ispypsa_tables["network_transmission_path_limits"] = pd.DataFrame( - columns=["path_id", "direction", "timeslice", "capacity"] - ) - ispypsa_tables["network_expansion_options"] = pd.DataFrame( - columns=[ - "expansion_id", - "expansion_type", - "allowed_expansion", - "expansion_option", - ] - ) - ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( - columns=["expansion_id", "year", "cost"] - ) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) links, link_timeslice_limits = _translate_network_to_links( ispypsa_tables, sample_model_config ) expected_links = csv_str_to_df(""" - isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type """) pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) @@ -295,3 +519,203 @@ def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_conf pd.testing.assert_frame_equal( link_timeslice_limits, expected_limits, check_dtype=False ) + + +def test_translate_network_to_links_path_id_wildcard_default( + csv_str_to_df, sample_model_config +): + """A blank path_id limit row is a table-wide default: a path with no rows of + its own (Q1-NQ) inherits it in both directions, while a path with its own + rows (CQ-NQ) keeps them, being more specific.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1500 + CQ-NQ, reverse, , 1000 + , , , 800 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1500, 0.0, 1.0, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 800, 0.0, 1.0, 2025, inf, , False, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + # CQ-NQ keeps its own per-unit limits; Q1-NQ takes the global default both ways. + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.666667 + Q1-NQ_existing, p_max_pu, , 1.0 + Q1-NQ_existing, p_min_pu, , -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["name", "attribute"]).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_static_cost_across_investment_periods( + csv_str_to_df, sample_model_config +): + """A blank year cost row is a static cost across every investment period, so + an expansion link is built for each (2026 and 2028 in the sample config).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 500, Opt + CQ-NQ, reverse, 500, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, , 1000000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + CQ-NQ, CQ-NQ_exp_2028, AC, CQ, NQ, 0.0, -1.0, 1.0, 2028, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_type_wildcard( + csv_str_to_df, sample_model_config +): + """A blank expansion_type option applies to both directions, so the expansion + link gets symmetric forward and reverse capability.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 700, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal(expansion, expected, check_dtype=False, rtol=1e-5) + + +def test_translate_network_to_links_designed_filtering_is_not_logged( + csv_str_to_df, sample_model_config, caplog +): + """Routing constraint_relaxation rows to the constraints translator and + selecting investment-period cost years are designed filtering, not data + loss, so a normal run emits no drop log lines.""" + ispypsa_tables = _network_tables(csv_str_to_df) + + with caplog.at_level("INFO"): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert "Dropped rows" not in caplog.text + + +def test_resolve_expansion_options_pairs_forward_and_reverse(csv_str_to_df): + """Each option defines both directions; a blank-expansion_id default pair fills + elements with no option of their own, keeping forward and reverse coupled.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + CQ-NQ, reverse, 800, BigLine + , forward, 500, Default + , reverse, 400, Default + SWQLD1, constraint_relaxation, 400, Relax + """) + + result = _resolve_expansion_options(options, ["CQ-NQ", "Q1-NQ"]) + + # CQ-NQ keeps its BigLine pair; Q1-NQ inherits the Default pair; SWQLD1 drops. + expected = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + CQ-NQ, reverse, 800, BigLine + Q1-NQ, forward, 500, Default + Q1-NQ, reverse, 400, Default + """) + pd.testing.assert_frame_equal( + result.sort_values(["expansion_id", "expansion_type"]).reset_index(drop=True), + expected.sort_values(["expansion_id", "expansion_type"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_expansion_options_raises_on_missing_direction(csv_str_to_df): + """An option defining only one direction raises rather than producing an + expansion link with a NaN rating in the missing direction.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + """) + + with pytest.raises( + ValueError, match=r"Expansion options define only one direction" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) + + +def test_resolve_expansion_options_raises_on_mismatched_pair(csv_str_to_df): + """A forward from one option paired with a reverse from another (here a + blank-expansion_id default) is incoherent and raises.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + , reverse, 400, Default + """) + + with pytest.raises( + ValueError, match="Forward and reverse expansion options disagree" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) diff --git a/tests/test_translator/test_translator_helpers.py b/tests/test_translator/test_translator_helpers.py index bba58a68..c00284c2 100644 --- a/tests/test_translator/test_translator_helpers.py +++ b/tests/test_translator/test_translator_helpers.py @@ -6,6 +6,7 @@ from ispypsa.translator.helpers import ( _add_investment_periods_as_build_years, _get_financial_year_int_from_string, + _resolve_wildcards, ) @@ -82,3 +83,124 @@ def test_add_investment_periods_as_build_years_empty(csv_str_to_df): assert result.empty assert "build_year" in result.columns assert list(result.columns) == list(expected.columns) + + +def test_resolve_wildcards_expands_a_blank_key_to_every_allowed_value(csv_str_to_df): + """A blank key cell is a wildcard expanded to every allowed value; a column + not in allowed_values (timeslice) rides along unchanged.""" + table = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, peak, 1200 + N1-CNSW, , , + """) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ", "N1-CNSW"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, peak, 1200 + N1-CNSW, forward, , + N1-CNSW, reverse, , + """) + assert_frame_equal( + result.sort_values(["path_id", "direction"]).reset_index(drop=True), + expected.sort_values(["path_id", "direction"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_wildcards_keeps_the_most_specific_row(csv_str_to_df): + """Where an expanded wildcard row and a specific row land on the same key, + the one that used fewer wildcards wins.""" + table = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + , , , 500 + """) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ", "Q1-NQ"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + # CQ-NQ/forward keeps its specific 1200; every other combination takes the + # global default 500. + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + CQ-NQ, reverse, , 500 + Q1-NQ, forward, , 500 + Q1-NQ, reverse, , 500 + """) + assert_frame_equal( + result.sort_values(["path_id", "direction"]).reset_index(drop=True), + expected.sort_values(["path_id", "direction"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_wildcards_drops_and_logs_out_of_allowed_values(csv_str_to_df, caplog): + """A filled value outside the allowed set drops out, and the drop is logged.""" + table = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 100 + CQ-NQ, 2025, 90 + """) + + with caplog.at_level("INFO"): + result = _resolve_wildcards( + table, {"expansion_id": ["CQ-NQ"], "year": [2026, 2028]}, ["cost"] + ) + + expected = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 100 + """) + assert_frame_equal(result, expected, check_dtype=False) + assert "Dropped rows whose year is not an allowed value: [2025]" in caplog.text + + +def test_resolve_wildcards_expected_drops_are_not_logged(csv_str_to_df, caplog): + """A drop from a column named in expected_drops is the caller's designed + selection, so no log line is emitted.""" + table = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 100 + CQ-NQ, 2025, 90 + """) + + with caplog.at_level("INFO"): + result = _resolve_wildcards( + table, + {"expansion_id": ["CQ-NQ"], "year": [2026, 2028]}, + ["cost"], + expected_drops=("year",), + ) + + expected = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 100 + """) + assert_frame_equal(result, expected, check_dtype=False) + assert "Dropped rows" not in caplog.text + + +def test_resolve_wildcards_empty(csv_str_to_df): + """An empty table resolves to an empty table with the same columns.""" + table = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + assert_frame_equal(result, expected, check_dtype=False) diff --git a/tests/test_translator_and_model/test_limit_on_transmission_expansion.py b/tests/test_translator_and_model/test_limit_on_transmission_expansion.py index d45fdf18..5fb8ad97 100644 --- a/tests/test_translator_and_model/test_limit_on_transmission_expansion.py +++ b/tests/test_translator_and_model/test_limit_on_transmission_expansion.py @@ -62,7 +62,7 @@ def test_flow_path_expansion_limit_respected(csv_str_to_df, tmp_path, monkeypatc "regional_granularity": "sub_regions", "rezs": "attached_to_parent_node", }, - "rez_to_sub_region_transmission_default_limit": 1e5, + "transmission_default_limit": 1e5, }, "temporal": { "year_type": "fy", diff --git a/tests/test_translator_and_model/test_time_varying_flow_path_costs.py b/tests/test_translator_and_model/test_time_varying_flow_path_costs.py index 1216f6dd..85d71f98 100644 --- a/tests/test_translator_and_model/test_time_varying_flow_path_costs.py +++ b/tests/test_translator_and_model/test_time_varying_flow_path_costs.py @@ -58,7 +58,7 @@ def test_link_expansion_economic_timing(csv_str_to_df, tmp_path, monkeypatch): "regional_granularity": "sub_regions", "rezs": "attached_to_parent_node", }, - "rez_to_sub_region_transmission_default_limit": 1e5, + "transmission_default_limit": 1e5, }, "temporal": { "year_type": "fy", diff --git a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py index 5f00e5db..ac637cc3 100644 --- a/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py +++ b/tests/test_translator_and_model/test_vre_build_limit_custom_constraints.py @@ -63,7 +63,7 @@ def test_vre_build_limit_constraint(csv_str_to_df, tmp_path, monkeypatch): "regional_granularity": "sub_regions", "rezs": "discrete_nodes", }, - "rez_to_sub_region_transmission_default_limit": 1e5, + "transmission_default_limit": 1e5, }, "temporal": { "year_type": "fy", From 1441720d3fedfe6b1871089af490b3bd8aca9d74 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 2 Jul 2026 10:19:03 +1000 Subject: [PATCH 3/5] Cover expansion-table combinatorics, cost wildcards and drop logging The options/costs pair now has all its empty/populated combinations pinned -- including options-without-costs, which the schema declares invalid but which silently builds no expansion links until schema validation lands -- plus the no-expansion-study config (both flags off), which resolves wildcards against an empty allowed set. The two cost-wildcard behaviours documented in _prepare_expansion_costs but previously untested are exercised: a concrete-year row overriding a blank-year static row (also the one spot where wildcard-expanded int years concat with CSV-parsed float years before the int cast), and a blank expansion_id acting as a table-wide default cost. The limits drop log gets its firing case (the negative already existed), and the calendar-year NotImplementedError closes the last uncovered line in network.py. Co-Authored-By: Claude Fable 5 --- tests/test_translator/test_network.py | 156 +++++++++++++++++++++++++- 1 file changed, 152 insertions(+), 4 deletions(-) diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py index 36415b24..9f405d73 100644 --- a/tests/test_translator/test_network.py +++ b/tests/test_translator/test_network.py @@ -413,19 +413,45 @@ def test_translate_network_to_links_rez_expansion_disabled( ] -def test_translate_network_to_links_rezs_attached_to_parent_node( +def test_translate_network_to_links_all_expansion_disabled( csv_str_to_df, sample_model_config +): + """With both expansion flags off no element is enabled, so the populated + options and costs tables resolve against an empty allowed set and no + expansion links are built.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_rezs_attached_to_parent_node( + csv_str_to_df, sample_model_config, caplog ): ispypsa_tables = _network_tables(csv_str_to_df) sample_model_config.network.nodes.rezs = "attached_to_parent_node" - links, link_timeslice_limits = _translate_network_to_links( - ispypsa_tables, sample_model_config - ) + with caplog.at_level("INFO"): + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) # REZ paths (and their expansion links and timeslice limits) are dropped. assert sorted(links["name"]) == ["CQ-NQ_existing", "CQ-NQ_exp_2026"] assert set(link_timeslice_limits["name"]) == {"CQ-NQ_existing"} + # The REZ paths' limit rows fall outside the modelled paths and the drop is + # logged — the same log a typo'd path_id would produce. + assert ( + "Dropped rows whose path_id is not an allowed value: ['N1-CNSW', 'Q1-NQ']" + in caplog.text + ) def test_translate_network_to_links_zero_capacity_parallel_path( @@ -489,6 +515,45 @@ def test_translate_network_to_links_empty_expansion_tables( ] +def test_translate_network_to_links_options_populated_costs_empty( + csv_str_to_df, sample_model_config +): + """A populated options table with no matching costs is invalid per the + schema (costs_cover_every_element_and_investment_period), but until schema + validation lands it silently produces no expansion links.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_options_empty_costs_populated( + csv_str_to_df, sample_model_config +): + """Costs without options mean no element is expandable, so no expansion + links are built.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): ispypsa_tables = _network_tables(csv_str_to_df) ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" @@ -615,6 +680,89 @@ def test_translate_network_to_links_static_cost_across_investment_periods( ) +def test_translate_network_to_links_year_override_beats_static_cost( + csv_str_to_df, sample_model_config +): + """A concrete-year cost row overrides the blank-year static row for that + investment period; the static row still fills the other periods.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 500, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, , 1000000 + CQ-NQ, 2028, 1200000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + CQ-NQ, CQ-NQ_exp_2028, AC, CQ, NQ, 0.0, -1.0, 1.0, 2028, inf, {1200000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_id_wildcard_default_cost( + csv_str_to_df, sample_model_config +): + """A blank expansion_id cost row is a table-wide default: an element with no + cost rows of its own (Q1-NQ) inherits it, while CQ-NQ keeps its own cost. It + also fans out to enabled elements with no expansion options (N1-CNSW), which + still build no expansion link.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + , 2026, 500000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_raises_for_calendar_years( + csv_str_to_df, sample_model_config +): + """Expansion-cost years are financial-year ending labels; calendar years are + not implemented.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.temporal.year_type = "calendar" + + with pytest.raises(NotImplementedError, match="year_type: calendar"): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + def test_translate_network_to_links_expansion_type_wildcard( csv_str_to_df, sample_model_config ): From 08a08414611d47a2929c582fb95c869ad39320ee Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 2 Jul 2026 13:23:30 +1000 Subject: [PATCH 4/5] Rename static_limits to max_capacities The name was a leftover from the winter-reference design, where the frame really did hold the link-level static limit; it now holds the per-direction maximum capacities that _extract_max_capacities produces, so the old name misdescribed it. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/network.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index f5187fb4..702abc98 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -101,10 +101,10 @@ def _translate_network_to_links( paths, config.network.transmission_default_limit, ) - static_limits = _extract_max_capacities(limits) + max_capacities = _extract_max_capacities(limits) existing_links = _build_existing_links( paths, - static_limits, + max_capacities, rez_ids, config.temporal.range.start_year, ) @@ -207,8 +207,8 @@ def _resolve_path_limits( def _extract_max_capacities(limits: pd.DataFrame) -> pd.DataFrame: - """Reduces the limits to one static capacity per path and direction: the - maximum across every row for that direction. + """Reduces the limits to one capacity per path and direction: the maximum + across every row for that direction. Named timeslices and any timeslice = NaN fallback row are pooled — the max has to include the fallback because at this stage we don't yet know which @@ -236,7 +236,7 @@ def _extract_max_capacities(limits: pd.DataFrame) -> pd.DataFrame: def _build_existing_links( paths: pd.DataFrame, - static_limits: pd.DataFrame, + max_capacities: pd.DataFrame, rez_ids: set[str], start_year: int, ) -> pd.DataFrame: @@ -248,7 +248,7 @@ def _build_existing_links( CQ-NQ CQ NQ AC N1-CNSW N1 CNSW AC # REZ path, no limit data - static_limits: + max_capacities: path_id direction capacity CQ-NQ forward 1400 CQ-NQ reverse 1910 @@ -263,7 +263,7 @@ def _build_existing_links( links = paths.rename( columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} ) - links = _add_p_nom(links, static_limits) + links = _add_p_nom(links, max_capacities) links["name"] = links["isp_name"] + "_existing" links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") links["build_year"] = start_year - 1 @@ -276,8 +276,8 @@ def _build_existing_links( return links -def _add_p_nom(links: pd.DataFrame, static_limits: pd.DataFrame) -> pd.DataFrame: - """Sets p_nom to the larger of the forward and reverse static capacities. +def _add_p_nom(links: pd.DataFrame, max_capacities: pd.DataFrame) -> pd.DataFrame: + """Sets p_nom to the larger of the forward and reverse maximum capacities. Taking the max of both directions keeps every per-unit limit in [-1, 1]. The link's reverse limit is not set here — it is carried per snapshot in @@ -291,7 +291,7 @@ def _add_p_nom(links: pd.DataFrame, static_limits: pd.DataFrame) -> pd.DataFrame CQ-NQ PAR-NEW # new parallel corridor, zero capacity - static_limits: + max_capacities: path_id direction capacity CQ-NQ forward 1400 CQ-NQ reverse 1910 @@ -303,7 +303,7 @@ def _add_p_nom(links: pd.DataFrame, static_limits: pd.DataFrame) -> pd.DataFrame CQ-NQ 1910 # max(1400, 1910) PAR-NEW 0 """ - directions = static_limits.pivot( + directions = max_capacities.pivot( index="path_id", columns="direction", values="capacity" ).reindex(columns=["forward", "reverse"]) links = links.merge(directions, left_on="isp_name", right_index=True, how="left") From df29f1d15343e3f981ba95d4c3d3130e3baaeb65 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Fri, 3 Jul 2026 15:18:56 +1000 Subject: [PATCH 5/5] Filter designed drops before wildcard resolution and act on review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _resolve_wildcards previously dropped out-of-set values itself, logging or staying silent per an expected_drops escape hatch, which conflated designed config-driven selection with bad input data. Callers now filter their designed selections out first (logged at INFO) and the resolver raises on anything left outside the allowed set — for limits a typo'd path_id halts the run, while an options/costs typo is indistinguishable from a constraint group or disabled element, so the log line is its only trace. Also acts on the PR #126 review: the forward/reverse pairing guard counts a blank expansion_option as its own label (nunique dropna=False) so a blank/named mismatch raises; the expansion_option schema description no longer claims the column is informational-only; p_nom collapses to a single groupby max; annuitisation is vectorised; and name-list test assertions are replaced with full-frame comparisons. Expansion cost years are opaque labels matched against the config's investment periods — year_type only governs how time is chunked — so the calendar-year NotImplementedError guard is gone. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/helpers.py | 59 ++-- src/ispypsa/translator/network.py | 292 +++++++++++------ .../schemas/network_expansion_options.yaml | 9 +- ...ork_transmission_path_expansion_costs.yaml | 4 +- tests/test_translator/test_network.py | 307 ++++++++++++++---- .../test_translator_helpers.py | 47 +-- 6 files changed, 490 insertions(+), 228 deletions(-) diff --git a/src/ispypsa/translator/helpers.py b/src/ispypsa/translator/helpers.py index 589c15f1..90b57c03 100644 --- a/src/ispypsa/translator/helpers.py +++ b/src/ispypsa/translator/helpers.py @@ -1,4 +1,3 @@ -import logging import re import pandas as pd @@ -161,7 +160,6 @@ def _resolve_wildcards( table: pd.DataFrame, allowed_values: dict[str, list], value_columns: list[str], - expected_drops: tuple[str, ...] = (), ) -> pd.DataFrame: """Expand a sparse "wildcard" table into one row per concrete key combination. @@ -169,13 +167,12 @@ def _resolve_wildcards( every value of that column. ``allowed_values`` lists, for each wildcardable column, the concrete values it may take — the schema's allowed_values / allowed_values_from, resolved to actual values. Each blank cell in those - columns is fanned out to every allowed value; a filled cell survives only if - it is itself an allowed value, so a row carrying an out-of-set value drops - out. Drops are logged at INFO unless the column is named in - ``expected_drops`` — for those, dropping is the caller's designed selection - (e.g. keeping only investment-period years), not data loss worth surfacing. - Key columns absent from ``allowed_values`` (e.g. timeslice) ride along - unchanged. + columns is fanned out to every allowed value; a filled cell must itself be + an allowed value — anything else raises. Callers filter their designed + selections (e.g. costs for disabled elements or non-investment-period + years) out before calling, so an out-of-set value reaching this point is + bad input data, not selection. Key columns absent from ``allowed_values`` + (e.g. timeslice) ride along unchanged. Once the blanks are filled in, several rows can land on the same key — a specific row and a wildcard one. The row that used the fewest wildcards (the @@ -209,7 +206,7 @@ def _resolve_wildcards( work["_wildcards"] = sum(work[c].isna().astype(int) for c in allowed_values) # Resolve one wildcardable column per pass; each pass fills that column in. for column, values in allowed_values.items(): - work = _expand_column(work, column, values, column not in expected_drops) + work = _expand_column(work, column, values) # Most specific (fewest wildcards) wins where rows now share a key. most_specific_first = work.sort_values("_wildcards", kind="stable") resolved = most_specific_first.drop_duplicates(key_columns, keep="first") @@ -217,16 +214,15 @@ def _resolve_wildcards( def _expand_column( - table: pd.DataFrame, column: str, allowed_values: list, log_drops: bool + table: pd.DataFrame, column: str, allowed_values: list ) -> pd.DataFrame: """Resolve a single wildcardable column of a wildcard table. Splits the rows on whether ``column`` is blank, then recombines: a filled - cell is kept only if its value is allowed, while each blank (wildcard) cell is - cross-joined with the allowed values so it fans out to one row per value. - Splitting first is the trick that keeps any blank out of the merge key (a NaN - key would not match itself in a merge). Out-of-set filled cells drop out, - logged only when ``log_drops`` is set (see _resolve_wildcards). + cell is kept as-is (after checking its value is allowed), while each blank + (wildcard) cell is cross-joined with the allowed values so it fans out to + one row per value. Splitting first is the trick that keeps any blank out of + the merge key (a NaN key would not match itself in a merge). I/O Example: table: column = "direction" @@ -241,20 +237,33 @@ def _expand_column( N1-CNSW reverse """ is_wildcard = table[column].isna() - if log_drops: - _log_dropped_values(table.loc[~is_wildcard, column], allowed_values, column) - # A filled cell survives only if its value is one of the allowed values... - concrete = table[~is_wildcard & table[column].isin(allowed_values)] + _raise_on_disallowed_values(table.loc[~is_wildcard, column], allowed_values, column) + # Every filled cell is now known to be allowed, so it is kept as-is... + concrete = table[~is_wildcard] # ...and a blank cell fans out to one row per allowed value (a cross join). allowed_frame = pd.DataFrame({column: allowed_values}) expanded = table[is_wildcard].drop(columns=column).merge(allowed_frame, how="cross") return pd.concat([concrete, expanded], ignore_index=True) -def _log_dropped_values(values: pd.Series, allowed_values: list, column: str) -> None: - """Log, once, any filled values dropped for falling outside the allowed set.""" +def _raise_on_disallowed_values( + values: pd.Series, allowed_values: list, column: str +) -> None: + """Raises if any filled value falls outside the allowed set. + + Callers filter every designed drop out before resolving wildcards, so a + disallowed value reaching this point is bad input data or a bug — the run + halts rather than silently dropping the rows. + + I/O Example: + values = [2026, 2025], allowed_values = [2026, 2028], column = "year" + -> raises ValueError naming year and [2025] + """ # tolist() converts numpy scalars to native types so the message reads # "[2025]" rather than "[np.int64(2025)]". - dropped = sorted(set(values.dropna().tolist()) - set(allowed_values)) - if dropped: - logging.info(f"Dropped rows whose {column} is not an allowed value: {dropped}") + disallowed = sorted(set(values.dropna().tolist()) - set(allowed_values)) + if disallowed: + raise ValueError( + f"Cannot resolve wildcards: {column} contains values outside the " + f"allowed set: {disallowed}" + ) diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py index 702abc98..5daa3c91 100644 --- a/src/ispypsa/translator/network.py +++ b/src/ispypsa/translator/network.py @@ -9,13 +9,23 @@ The three sparse input tables (limits, expansion options, expansion costs) may use blank key cells as wildcards; each is first put through _resolve_wildcards -(see translator/helpers.py) to expand those wildcards to concrete rows. +(see translator/helpers.py) to expand those wildcards to concrete rows. Rows +the configuration deliberately excludes (unmodelled REZ paths, disabled +expansions, non-investment-period years) are filtered out before resolution, +logged at INFO. For the limits table a path_id unknown to the paths table +survives that filtering, so _resolve_wildcards raises on it as bad input data. +The options and costs filters cannot make that distinction — an expansion_id +outside the enabled set may be a disabled element, a constraint group or a +typo — so anything outside the enabled elements or investment periods is +filtered and logged rather than raised on. _translate_network_to_links is the pipeline orchestrator and reads as the step-by-step story; each helper's docstring carries an I/O example, and inline comments mark the non-obvious modelling choices. """ +import logging + import numpy as np import pandas as pd @@ -96,15 +106,19 @@ def _translate_network_to_links( paths = _drop_rez_paths_if_not_modelled( ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs ) - limits = _resolve_path_limits( + limits = _drop_limits_for_unmodelled_paths( ispypsa_tables["network_transmission_path_limits"], + ispypsa_tables["network_transmission_paths"], + paths, + ) + limits = _resolve_path_limits( + limits, paths, config.network.transmission_default_limit, ) - max_capacities = _extract_max_capacities(limits) existing_links = _build_existing_links( paths, - max_capacities, + limits, rez_ids, config.temporal.range.start_year, ) @@ -124,7 +138,6 @@ def _translate_network_to_links( ispypsa_tables["network_transmission_path_expansion_costs"], enabled_ids, config.temporal.capacity_expansion.investment_periods, - config.temporal.year_type, config.wacc, config.network.annuitisation_lifetime, ) @@ -166,6 +179,44 @@ def _drop_rez_paths_if_not_modelled( return paths[~paths["geo_from"].isin(rez_ids)] +def _drop_limits_for_unmodelled_paths( + limits: pd.DataFrame, all_paths: pd.DataFrame, modelled_paths: pd.DataFrame +) -> pd.DataFrame: + """Drops limit rows for paths configured out of the model. + + REZ-to-parent paths leave the modelled set when REZs are attached to their + parent nodes (see _drop_rez_paths_if_not_modelled); their limit rows are + config-driven selection, filtered out here (logged at INFO) before wildcard + resolution. Only + rows naming a known-but-unmodelled path drop: blank (wildcard) path_ids ride + through, and a path_id unknown to the paths table survives to + _resolve_wildcards, which raises on it. + + I/O Example: + all_paths: path_id in {CQ-NQ, Q1-NQ} + modelled_paths: path_id in {CQ-NQ} + + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 + Q1-NQ forward 750 # known but unmodelled: dropped + 500 # blank: a wildcard, kept + + returns: + path_id direction timeslice capacity + CQ-NQ forward 1400 + 500 + """ + unmodelled = set(all_paths["path_id"]) - set(modelled_paths["path_id"]) + dropped = limits["path_id"].isin(unmodelled) + if dropped.any(): + logging.info( + f"Filtered limit rows for paths configured out of the model: " + f"{sorted(set(limits.loc[dropped, 'path_id']))}" + ) + return limits[~dropped] + + def _resolve_path_limits( limits: pd.DataFrame, paths: pd.DataFrame, default_limit: float ) -> pd.DataFrame: @@ -173,9 +224,11 @@ def _resolve_path_limits( and timeslice, then fills empty capacities with the system default. Blank path_id, direction or capacity cells are wildcards (see the - network_transmission_path_limits schema). _resolve_wildcards expands the - path_id and direction wildcards against the modelled paths and the two - directions; timeslice rides along, so a blank-timeslice row stays the + network_transmission_path_limits schema). The limits arrive already filtered + to the modelled paths (see _drop_limits_for_unmodelled_paths), so + _resolve_wildcards expands the path_id and direction wildcards against the + modelled paths and the two directions, raising on any unknown path_id or + direction; timeslice rides along, so a blank-timeslice row stays the snapshot-level fallback that pypsa_build applies later. A blank capacity then takes the configured transmission default — the nan_fill the schema declares for the capacity column — which is how a no-data path (a single all-wildcard @@ -206,37 +259,9 @@ def _resolve_path_limits( return limits -def _extract_max_capacities(limits: pd.DataFrame) -> pd.DataFrame: - """Reduces the limits to one capacity per path and direction: the maximum - across every row for that direction. - - Named timeslices and any timeslice = NaN fallback row are pooled — the max - has to include the fallback because at this stage we don't yet know which - snapshots it will cover. The larger of the two directions becomes the link's - p_nom downstream, so every per-unit limit lands in [-1, 1]. No-data paths - arrive already defaulted (see _resolve_path_limits), so they contribute a - forward and reverse max at the default limit like any other path. - - I/O Example: - limits: - path_id direction timeslice capacity - CQ-NQ forward qld_peak_demand 1200 - CQ-NQ forward , 1400 # NaN-timeslice fallback - CQ-NQ reverse qld_winter_reference 1910 - - returns: - path_id direction capacity - CQ-NQ forward 1400 # fallback exceeds the named row - CQ-NQ reverse 1910 - """ - # Max pools the named timeslices with the NaN fallback; the fallback has to be - # in the max because we don't yet know which snapshots it will cover. - return limits.groupby(["path_id", "direction"], as_index=False)["capacity"].max() - - def _build_existing_links( paths: pd.DataFrame, - max_capacities: pd.DataFrame, + limits: pd.DataFrame, rez_ids: set[str], start_year: int, ) -> pd.DataFrame: @@ -248,12 +273,12 @@ def _build_existing_links( CQ-NQ CQ NQ AC N1-CNSW N1 CNSW AC # REZ path, no limit data - max_capacities: - path_id direction capacity - CQ-NQ forward 1400 - CQ-NQ reverse 1910 - N1-CNSW forward 100000 # defaulted upstream - N1-CNSW reverse 100000 + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + N1-CNSW forward 100000 # defaulted upstream + N1-CNSW reverse 100000 returns (abridged): isp_name name bus0 bus1 p_nom p_min_pu isp_type @@ -263,7 +288,7 @@ def _build_existing_links( links = paths.rename( columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} ) - links = _add_p_nom(links, max_capacities) + links = _add_p_nom(links, limits) links["name"] = links["isp_name"] + "_existing" links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") links["build_year"] = start_year - 1 @@ -276,14 +301,19 @@ def _build_existing_links( return links -def _add_p_nom(links: pd.DataFrame, max_capacities: pd.DataFrame) -> pd.DataFrame: - """Sets p_nom to the larger of the forward and reverse maximum capacities. +def _add_p_nom(links: pd.DataFrame, limits: pd.DataFrame) -> pd.DataFrame: + """Sets p_nom to the largest capacity across every limit row of the path. - Taking the max of both directions keeps every per-unit limit in [-1, 1]. The - link's reverse limit is not set here — it is carried per snapshot in - link_timeslice_limits (a named timeslice or the timeslice = NaN fallback), - which under the coverage contract (Open-ISP/ISPyPSA#123) sets p_min_pu on - every snapshot, leaving the link's static p_min_pu at its inert default. + Both directions and all timeslices pool into one max: the timeslice = NaN + fallback has to be included because at this stage we don't yet know which + snapshots it will cover, and taking the larger direction keeps every + per-unit limit in [-1, 1]. The link's reverse limit is not set here — it is + carried per snapshot in link_timeslice_limits (a named timeslice or the + timeslice = NaN fallback), which under the coverage contract + (Open-ISP/ISPyPSA#123) sets p_min_pu on every snapshot, leaving the link's + static p_min_pu at its inert default. No-data paths arrive already + defaulted (see _resolve_path_limits), so they get the default limit like + any other path. I/O Example: links: @@ -291,26 +321,25 @@ def _add_p_nom(links: pd.DataFrame, max_capacities: pd.DataFrame) -> pd.DataFram CQ-NQ PAR-NEW # new parallel corridor, zero capacity - max_capacities: - path_id direction capacity - CQ-NQ forward 1400 - CQ-NQ reverse 1910 - PAR-NEW forward 0 - PAR-NEW reverse 0 + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward 1400 # NaN-timeslice fallback + CQ-NQ reverse 1910 + PAR-NEW forward 0 + PAR-NEW reverse 0 returns: isp_name p_nom - CQ-NQ 1910 # max(1400, 1910) + CQ-NQ 1910 # the reverse row is the largest PAR-NEW 0 """ - directions = max_capacities.pivot( - index="path_id", columns="direction", values="capacity" - ).reindex(columns=["forward", "reverse"]) - links = links.merge(directions, left_on="isp_name", right_index=True, how="left") - # Larger of the two directions; dividing each limit by it keeps every per-unit - # value in [-1, 1]. - links["p_nom"] = links[["forward", "reverse"]].max(axis=1) - return links.drop(columns=["forward", "reverse"]) + max_capacity = ( + limits.groupby("path_id", as_index=False)["capacity"] + .max() + .rename(columns={"path_id": "isp_name", "capacity": "p_nom"}) + ) + return links.merge(max_capacity, on="isp_name", how="left") def _translate_timeslice_limits_to_pu( @@ -369,8 +398,9 @@ def _enabled_expansion_element_ids( ``transmission_expansion`` gates flow paths between (sub)regions; ``rez_transmission_expansion`` gates REZ connection paths. The result is the - allowed expansion_id set the options and costs are resolved against, so an - option or cost for a disabled or non-modelled element drops out there. + enabled expansion_id set the options and costs tables are filtered to (see + _keep_rows_for_enabled_elements) and then resolved against, so an option or + cost for a disabled or non-modelled element drops out before resolution. I/O Example: existing_links: @@ -401,10 +431,9 @@ def _resolve_expansion_options( expansion_type covers both directions of one element in a single (symmetric) row. constraint_relaxation rows are not physical paths — they are set aside first and become relaxation generators in ispypsa.translator.constraints. - _resolve_wildcards then expands the blanks against the enabled elements and - the two physical directions; an option for a disabled or non-modelled element - falls outside the allowed values and drops out (config-driven selection, so - not logged as a drop). + Options for disabled or non-modelled elements are config-driven selection, + filtered out next (logged at INFO). _resolve_wildcards then expands the + blanks against the enabled elements and the two physical directions. Each element's forward and reverse must form a coherent pair from one option (the schema's expansion_options_pair_forward_and_reverse rule); @@ -432,8 +461,9 @@ def _resolve_expansion_options( """ # constraint_relaxation options are routed to ispypsa.translator.constraints, # not dropped, so they are set aside before wildcard resolution (which would - # otherwise log them as dropped rows). + # otherwise raise on them). options = options[options["expansion_type"] != "constraint_relaxation"] + options = _keep_rows_for_enabled_elements(options, enabled_ids) allowed_values = { "expansion_id": enabled_ids, "expansion_type": ["forward", "reverse"], @@ -442,13 +472,47 @@ def _resolve_expansion_options( options, allowed_values, ["allowed_expansion", "expansion_option"], - expected_drops=("expansion_id",), ) _check_both_directions_defined(options) _check_forward_reverse_share_an_option(options) return options +def _keep_rows_for_enabled_elements( + table: pd.DataFrame, enabled_ids: list[str] +) -> pd.DataFrame: + """Keeps rows whose expansion_id is an enabled element or blank (a wildcard). + + Selecting the enabled elements is config-driven, so it happens before + wildcard resolution rather than inside it. Rows for disabled or non-modelled + elements drop out here, as do rows for constraint groups (their expansion_ids + are constraint_ids, routed to ispypsa.translator.constraints instead). The + filtered ids are logged at INFO — a typo'd expansion_id is indistinguishable + from these designed drops, so the log line is its only trace. + + I/O Example: + table: enabled_ids = ["CQ-NQ"] + expansion_id year cost + CQ-NQ 2026 1000000 + Q1-NQ 2026 500000 # disabled element: dropped + SWQLD1 2026 400000 # constraint group: dropped + 2026 900000 # blank: a wildcard, kept + + returns: + expansion_id year cost + CQ-NQ 2026 1000000 + 2026 900000 + """ + ids = table["expansion_id"] + keep = ids.isna() | ids.isin(enabled_ids) + if not keep.all(): + logging.info( + f"Filtered rows whose expansion_id is not an enabled expansion " + f"element: {sorted(set(ids[~keep]))}" + ) + return table[keep] + + def _check_both_directions_defined(options: pd.DataFrame) -> None: """Raises if any element's resolved options define only one direction. @@ -479,14 +543,20 @@ def _check_forward_reverse_share_an_option(options: pd.DataFrame) -> None: the single cost keyed on expansion_id applies to a coherent pair. Wildcard resolution resolves the two directions independently, so a malformed input could pair a forward from one option with a reverse from another; this guard - catches that. The schema's expansion_options_pair_forward_and_reverse rule is - the upstream enforcement — this stands in until schema validation lands. + catches that. A blank expansion_option counts as its own label (nunique with + dropna=False), so an unlabelled direction only pairs with another unlabelled + one — nunique's default NaN-dropping would otherwise let a blank/named + mismatch through. The schema's expansion_options_pair_forward_and_reverse + rule is the upstream enforcement — this stands in until schema validation + lands. I/O Example: options with CQ-NQ forward from "BigLine" and CQ-NQ reverse from "Default" -> raises (the two directions disagree on the option). """ - options_per_element = options.groupby("expansion_id")["expansion_option"].nunique() + options_per_element = options.groupby("expansion_id")["expansion_option"].nunique( + dropna=False + ) mismatched = sorted(options_per_element[options_per_element > 1].index) if mismatched: raise ValueError( @@ -526,7 +596,6 @@ def _prepare_expansion_costs( expansion_costs: pd.DataFrame, enabled_ids: list[str], investment_periods: list[int], - year_type: str, wacc: float, asset_lifetime: int, ) -> pd.DataFrame: @@ -536,14 +605,16 @@ def _prepare_expansion_costs( Blank expansion_id or year cells are wildcards (see the network_transmission_path_expansion_costs schema): an empty expansion_id is a table-wide default cost, an empty year a static cost across the investment - periods. _resolve_wildcards expands them against the enabled elements and the - investment periods, which drops any cost for a disabled element, a constraint - group (routed to ispypsa.translator.constraints) or a year outside the - investment periods — all designed selection, so none of it is logged as a - drop. A blank cost then resolves to free — the - nan_fill the schema declares for the cost column. The year column holds - financial-year ending years as ints, matching the investment period labels - used with year_type="fy". + periods. Costs for disabled elements, constraint groups (routed to + ispypsa.translator.constraints) and years outside the investment periods are + all designed selection, filtered out first (logged at INFO). _resolve_wildcards + then expands the blanks against the enabled elements and the investment + periods. A blank cost then resolves to free — the nan_fill the schema + declares for the cost column. Year values are labels matched against the + config's investment periods as ints — no financial vs calendar year + interpretation happens here; the config's year_type decides what span of + time the labels denote, so the table just has to label years the same way + the config does (templated tables carry financial-year ending years). I/O Example (a blank year is a static cost across the investment periods): expansion_costs: @@ -559,27 +630,52 @@ def _prepare_expansion_costs( CQ-NQ 2026 annuitise(1000) # the static row fills 2026 CQ-NQ 2028 annuitise(1200) # the 2028 override beats the static row """ - if year_type != "fy": - raise NotImplementedError( - f"Network expansion costs are not implemented for year_type: {year_type}" - ) + costs = _keep_rows_for_enabled_elements(expansion_costs, enabled_ids) + costs = _keep_rows_for_investment_period_years(costs, investment_periods) allowed_values = {"expansion_id": enabled_ids, "year": investment_periods} - costs = _resolve_wildcards( - expansion_costs, - allowed_values, - ["cost"], - expected_drops=("expansion_id", "year"), - ) + costs = _resolve_wildcards(costs, allowed_values, ["cost"]) # Every surviving year is now an investment period; cast so expansion link # names read e.g. CQ-NQ_exp_2026 rather than CQ-NQ_exp_2026.0. costs["year"] = costs["year"].astype("int64") costs["cost"] = costs["cost"].fillna(0.0) - costs["capital_cost"] = costs["cost"].apply( - lambda cost: _annuitised_investment_costs(cost, wacc, asset_lifetime) + costs["capital_cost"] = _annuitised_investment_costs( + costs["cost"], wacc, asset_lifetime ) return costs.loc[:, ["expansion_id", "year", "capital_cost"]] +def _keep_rows_for_investment_period_years( + costs: pd.DataFrame, investment_periods: list[int] +) -> pd.DataFrame: + """Keeps cost rows whose year is an investment period or blank (a wildcard). + + The costs table spans the IASR horizon while the model only prices the + configured investment periods, so selecting those years is config-driven and + happens before wildcard resolution rather than inside it. The filtered years + are logged at INFO. + + I/O Example: + costs: investment_periods = [2026, 2028] + expansion_id year cost + CQ-NQ 2025 900000 # not an investment period: dropped + CQ-NQ 2026 1000000 + CQ-NQ 800000 # blank: a wildcard, kept + + returns: + expansion_id year cost + CQ-NQ 2026 1000000 + CQ-NQ 800000 + """ + years = costs["year"] + keep = years.isna() | years.isin(investment_periods) + if not keep.all(): + logging.info( + f"Filtered expansion cost rows for years outside the investment " + f"periods: {sorted(int(year) for year in set(years[~keep]))}" + ) + return costs[keep] + + def _build_expansion_links( existing_links: pd.DataFrame, options: pd.DataFrame, diff --git a/src/ispypsa/validation/schemas/network_expansion_options.yaml b/src/ispypsa/validation/schemas/network_expansion_options.yaml index 6b391bbd..66cfce6d 100644 --- a/src/ispypsa/validation/schemas/network_expansion_options.yaml +++ b/src/ispypsa/validation/schemas/network_expansion_options.yaml @@ -87,7 +87,12 @@ columns: type: string required: false description: > - Name of the selected augmentation option, retained for traceability. + Name of the selected augmentation option. Not read as model data, but it + ties an element's directions together: the + expansion_options_pair_forward_and_reverse rule requires an expansion_id's + forward and reverse rows to carry the same expansion_option, so directions + labelled from different options are rejected. If absent (or empty): - No effect on the model; the column is informational only. + No option name is recorded. A blank still participates in the pairing + rule, pairing only with another blank. diff --git a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml index 388878a9..b6a85053 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml @@ -59,7 +59,9 @@ columns: type: int required: false description: > - Investment period (financial-year ending year) the cost applies to. + Investment period the cost applies to, matching the labels in the model + configuration's investment periods (for templated tables these are + financial-year ending years, e.g. 2025 for FY 2024-25). If absent (or empty): Wildcard — a static cost applied to every investment period the element has diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py index 9f405d73..2e188d6a 100644 --- a/tests/test_translator/test_network.py +++ b/tests/test_translator/test_network.py @@ -3,6 +3,7 @@ from ispypsa.translator.helpers import _annuitised_investment_costs from ispypsa.translator.network import ( + _drop_limits_for_unmodelled_paths, _resolve_expansion_options, _translate_network_geography_to_buses, _translate_network_to_links, @@ -387,13 +388,16 @@ def test_translate_network_to_links_expansion_disabled( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - # The flow path's expansion link is gone; the REZ path's remains. - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "N1-CNSW_existing", - "Q1-NQ_existing", - "Q1-NQ_exp_2026", - ] + # The flow path's expansion link is gone; the REZ path's remains. The + # existing links' content is pinned by test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + expansion, expected_expansion, check_dtype=False, rtol=1e-5 + ) def test_translate_network_to_links_rez_expansion_disabled( @@ -404,13 +408,16 @@ def test_translate_network_to_links_rez_expansion_disabled( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - # The REZ path's expansion link is gone; the flow path's remains. - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "CQ-NQ_exp_2026", - "N1-CNSW_existing", - "Q1-NQ_existing", - ] + # The REZ path's expansion link is gone; the flow path's remains. The + # existing links' content is pinned by test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion, expected_expansion, check_dtype=False, rtol=1e-5 + ) def test_translate_network_to_links_all_expansion_disabled( @@ -425,32 +432,58 @@ def test_translate_network_to_links_all_expansion_disabled( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "N1-CNSW_existing", - "Q1-NQ_existing", - ] + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) def test_translate_network_to_links_rezs_attached_to_parent_node( - csv_str_to_df, sample_model_config, caplog + csv_str_to_df, sample_model_config ): + """With REZs attached to their parent nodes the REZ paths leave the model, + and their limit, option and cost rows are filtered out before wildcard + resolution rather than raising as out-of-set values.""" ispypsa_tables = _network_tables(csv_str_to_df) sample_model_config.network.nodes.rezs = "attached_to_parent_node" - with caplog.at_level("INFO"): - links, link_timeslice_limits = _translate_network_to_links( - ispypsa_tables, sample_model_config - ) + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) # REZ paths (and their expansion links and timeslice limits) are dropped. - assert sorted(links["name"]) == ["CQ-NQ_existing", "CQ-NQ_exp_2026"] - assert set(link_timeslice_limits["name"]) == {"CQ-NQ_existing"} - # The REZ paths' limit rows fall outside the modelled paths and the drop is - # logged — the same log a typo'd path_id would produce. - assert ( - "Dropped rows whose path_id is not an allowed value: ['N1-CNSW', 'Q1-NQ']" - in caplog.text + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1910, 0.0, 1.0, 2025, inf, , False, flow_path + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.628272 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.680628 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.732984 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.753927 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -0.837696 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, ) @@ -508,11 +541,13 @@ def test_translate_network_to_links_empty_expansion_tables( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "N1-CNSW_existing", - "Q1-NQ_existing", - ] + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) def test_translate_network_to_links_options_populated_costs_empty( @@ -528,18 +563,23 @@ def test_translate_network_to_links_options_populated_costs_empty( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "N1-CNSW_existing", - "Q1-NQ_existing", - ] + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) def test_translate_network_to_links_options_empty_costs_populated( csv_str_to_df, sample_model_config ): """Costs without options mean no element is expandable, so no expansion - links are built.""" + links are built. This input is actually invalid per the schema — costs + declare expansion_id allowed_values_from network_expansion_options, and an + empty options table leaves no allowed values — so once the validation layer + lands it will be rejected upstream rather than silently building nothing.""" ispypsa_tables = _network_tables(csv_str_to_df) ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" expansion_id, expansion_type, allowed_expansion, expansion_option @@ -547,11 +587,13 @@ def test_translate_network_to_links_options_empty_costs_populated( links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) - assert sorted(links["name"]) == [ - "CQ-NQ_existing", - "N1-CNSW_existing", - "Q1-NQ_existing", - ] + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): @@ -751,18 +793,6 @@ def test_translate_network_to_links_expansion_id_wildcard_default_cost( ) -def test_translate_network_to_links_raises_for_calendar_years( - csv_str_to_df, sample_model_config -): - """Expansion-cost years are financial-year ending labels; calendar years are - not implemented.""" - ispypsa_tables = _network_tables(csv_str_to_df) - sample_model_config.temporal.year_type = "calendar" - - with pytest.raises(NotImplementedError, match="year_type: calendar"): - _translate_network_to_links(ispypsa_tables, sample_model_config) - - def test_translate_network_to_links_expansion_type_wildcard( csv_str_to_df, sample_model_config ): @@ -797,18 +827,140 @@ def test_translate_network_to_links_expansion_type_wildcard( pd.testing.assert_frame_equal(expansion, expected, check_dtype=False, rtol=1e-5) -def test_translate_network_to_links_designed_filtering_is_not_logged( +def test_translate_network_to_links_raises_on_unknown_limits_path_id( + csv_str_to_df, sample_model_config +): + """A limits path_id absent from the paths table is bad input data — unlike a + config-driven drop, it survives the pre-resolution filtering and the wildcard + resolver halts the run on it. In the long run the validation layer will guard + against this example. + + Run with REZs attached to their parent nodes so the filtering is active: the + raise message lists every disallowed value, so asserting it names only the + typo also proves the known-but-unmodelled Q1-NQ row was filtered out and the + blank wildcard row rode through.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + Q1-NQ, forward, , 750 + , , , 500 + CQ-NQQ, forward, , 1200 + """) + + with pytest.raises( + ValueError, + match=r"path_id contains values outside the allowed set: \['CQ-NQQ'\]", + ): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + +def test_translate_network_to_links_logs_config_filtered_rows( csv_str_to_df, sample_model_config, caplog ): - """Routing constraint_relaxation rows to the constraints translator and - selecting investment-period cost years are designed filtering, not data - loss, so a normal run emits no drop log lines.""" + """Rows filtered out by config-driven selection — limits for unmodelled REZ + paths, options and costs for elements that are not enabled, and cost years + outside the investment periods — are logged at INFO so their absence from + the model can be audited.""" ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" with caplog.at_level("INFO"): _translate_network_to_links(ispypsa_tables, sample_model_config) - assert "Dropped rows" not in caplog.text + assert ( + "Filtered limit rows for paths configured out of the model: " + "['N1-CNSW', 'Q1-NQ']" + ) in caplog.text + # The options table's filtered ids (SWQLD1 was already set aside as a + # constraint_relaxation row)... + assert ( + "Filtered rows whose expansion_id is not an enabled expansion element: " + "['Q1-NQ']" + ) in caplog.text + # ...and the costs table's, where SWQLD1 is filtered rather than set aside. + assert ( + "Filtered rows whose expansion_id is not an enabled expansion element: " + "['Q1-NQ', 'SWQLD1']" + ) in caplog.text + assert ( + "Filtered expansion cost rows for years outside the investment periods: [2025]" + ) in caplog.text + + +def test_translate_network_to_links_no_filter_log_when_all_rows_used( + csv_str_to_df, sample_model_config, caplog +): + """When every input row survives into the model, no filter log lines fire.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 700, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + """) + + with caplog.at_level("INFO"): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert "Filtered" not in caplog.text + + +def test_drop_limits_for_unmodelled_paths_limits_empty(csv_str_to_df): + limits = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + paths = csv_str_to_df(""" + path_id + CQ-NQ + """) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_drop_limits_for_unmodelled_paths_paths_empty(csv_str_to_df): + """With no paths at all nothing is 'known but unmodelled', so every limit row + rides through to _resolve_wildcards, which raises on the out-of-set path_id.""" + limits = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + """) + paths = pd.DataFrame(columns=["path_id"]) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_drop_limits_for_unmodelled_paths_both_empty(csv_str_to_df): + limits = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + paths = pd.DataFrame(columns=["path_id"]) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) def test_resolve_expansion_options_pairs_forward_and_reverse(csv_str_to_df): @@ -842,7 +994,9 @@ def test_resolve_expansion_options_pairs_forward_and_reverse(csv_str_to_df): def test_resolve_expansion_options_raises_on_missing_direction(csv_str_to_df): """An option defining only one direction raises rather than producing an - expansion link with a NaN rating in the missing direction.""" + expansion link with a NaN rating in the missing direction. The check stands + in for the schema's expansion_options_pair_forward_and_reverse rule, which + will reject this input upstream once the validation layer lands.""" options = csv_str_to_df(""" expansion_id, expansion_type, allowed_expansion, expansion_option CQ-NQ, forward, 1000, BigLine @@ -856,7 +1010,9 @@ def test_resolve_expansion_options_raises_on_missing_direction(csv_str_to_df): def test_resolve_expansion_options_raises_on_mismatched_pair(csv_str_to_df): """A forward from one option paired with a reverse from another (here a - blank-expansion_id default) is incoherent and raises.""" + blank-expansion_id default) is incoherent and raises. The check stands in + for the schema's expansion_options_pair_forward_and_reverse rule, which will + reject this input upstream once the validation layer lands.""" options = csv_str_to_df(""" expansion_id, expansion_type, allowed_expansion, expansion_option CQ-NQ, forward, 1000, BigLine @@ -867,3 +1023,22 @@ def test_resolve_expansion_options_raises_on_mismatched_pair(csv_str_to_df): ValueError, match="Forward and reverse expansion options disagree" ): _resolve_expansion_options(options, ["CQ-NQ"]) + + +def test_resolve_expansion_options_raises_on_blank_option_mismatched_pair( + csv_str_to_df, +): + """A blank expansion_option only pairs with another blank: a forward from an + unlabelled option and a reverse from a named one (here a blank-expansion_id + default) is just as incoherent as two differing names, and nunique's default + NaN-dropping would otherwise let it through.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, + , reverse, 400, Default + """) + + with pytest.raises( + ValueError, match="Forward and reverse expansion options disagree" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) diff --git a/tests/test_translator/test_translator_helpers.py b/tests/test_translator/test_translator_helpers.py index c00284c2..dc5c75b0 100644 --- a/tests/test_translator/test_translator_helpers.py +++ b/tests/test_translator/test_translator_helpers.py @@ -144,51 +144,26 @@ def test_resolve_wildcards_keeps_the_most_specific_row(csv_str_to_df): ) -def test_resolve_wildcards_drops_and_logs_out_of_allowed_values(csv_str_to_df, caplog): - """A filled value outside the allowed set drops out, and the drop is logged.""" +def test_resolve_wildcards_raises_on_out_of_allowed_values(csv_str_to_df): + """A filled value outside the allowed set is bad input data — callers filter + designed selections out before resolving — so resolution halts the run. The + validation layer will catch this class of input upstream once it lands (the + allowed values mirror the schemas' allowed_values / allowed_values_from); + the raise stays on as the resolver's backstop invariant.""" table = csv_str_to_df(""" expansion_id, year, cost CQ-NQ, 2026, 100 CQ-NQ, 2025, 90 """) - with caplog.at_level("INFO"): - result = _resolve_wildcards( + with pytest.raises( + ValueError, + match=r"year contains values outside the allowed set: \[2025\]", + ): + _resolve_wildcards( table, {"expansion_id": ["CQ-NQ"], "year": [2026, 2028]}, ["cost"] ) - expected = csv_str_to_df(""" - expansion_id, year, cost - CQ-NQ, 2026, 100 - """) - assert_frame_equal(result, expected, check_dtype=False) - assert "Dropped rows whose year is not an allowed value: [2025]" in caplog.text - - -def test_resolve_wildcards_expected_drops_are_not_logged(csv_str_to_df, caplog): - """A drop from a column named in expected_drops is the caller's designed - selection, so no log line is emitted.""" - table = csv_str_to_df(""" - expansion_id, year, cost - CQ-NQ, 2026, 100 - CQ-NQ, 2025, 90 - """) - - with caplog.at_level("INFO"): - result = _resolve_wildcards( - table, - {"expansion_id": ["CQ-NQ"], "year": [2026, 2028]}, - ["cost"], - expected_drops=("year",), - ) - - expected = csv_str_to_df(""" - expansion_id, year, cost - CQ-NQ, 2026, 100 - """) - assert_frame_equal(result, expected, check_dtype=False) - assert "Dropped rows" not in caplog.text - def test_resolve_wildcards_empty(csv_str_to_df): """An empty table resolves to an empty table with the same columns."""