diff --git a/src/ispypsa/translator/helpers.py b/src/ispypsa/translator/helpers.py index 965c249c..90b57c03 100644 --- a/src/ispypsa/translator/helpers.py +++ b/src/ispypsa/translator/helpers.py @@ -154,3 +154,116 @@ def convert_to_numeric_if_possible(df: pd.DataFrame, cols: list[str]) -> pd.Data df = df.drop(columns=["temp_col"]) return df + + +def _resolve_wildcards( + table: pd.DataFrame, + allowed_values: dict[str, list], + value_columns: list[str], +) -> pd.DataFrame: + """Expand a sparse "wildcard" table into one row per concrete key combination. + + A key column may be left blank (NaN) to act as a wildcard that applies to + every value of that column. ``allowed_values`` lists, for each wildcardable + column, the concrete values it may take — the schema's allowed_values / + allowed_values_from, resolved to actual values. Each blank cell in those + columns is fanned out to every allowed value; a filled cell must itself be + an allowed value — anything else raises. Callers filter their designed + selections (e.g. costs for disabled elements or non-investment-period + years) out before calling, so an out-of-set value reaching this point is + bad input data, not selection. Key columns absent from ``allowed_values`` + (e.g. timeslice) ride along unchanged. + + Once the blanks are filled in, several rows can land on the same key — a + specific row and a wildcard one. The row that used the fewest wildcards (the + most specific) wins; callers rely on the schema's *_resolve_unambiguously + rule to guarantee there is never a tie. + + I/O Example: + table (a blank cell is a wildcard): + path_id direction timeslice capacity + CQ-NQ forward peak 1200 + 500 # all keys blank: a global default + N1-CNSW # blank direction and capacity + + allowed_values = {"path_id": ["CQ-NQ", "N1-CNSW"], + "direction": ["forward", "reverse"]} + value_columns = ["capacity"] + + returns: + path_id direction timeslice capacity + CQ-NQ forward peak 1200 # specific row, untouched + CQ-NQ forward 500 # global default fills the gaps... + CQ-NQ reverse 500 + N1-CNSW forward NaN # ...but N1-CNSW's own blank-capacity + N1-CNSW reverse NaN # row is more specific, so it wins + """ + key_columns = [c for c in table.columns if c not in value_columns] + work = table.copy() + # Count each row's wildcards now, before the blanks are filled in, so that + # after expansion we can keep the row that used the fewest wildcards wherever + # rows land on the same key. + work["_wildcards"] = sum(work[c].isna().astype(int) for c in allowed_values) + # Resolve one wildcardable column per pass; each pass fills that column in. + for column, values in allowed_values.items(): + work = _expand_column(work, column, values) + # Most specific (fewest wildcards) wins where rows now share a key. + most_specific_first = work.sort_values("_wildcards", kind="stable") + resolved = most_specific_first.drop_duplicates(key_columns, keep="first") + return resolved.loc[:, list(table.columns)].reset_index(drop=True) + + +def _expand_column( + table: pd.DataFrame, column: str, allowed_values: list +) -> pd.DataFrame: + """Resolve a single wildcardable column of a wildcard table. + + Splits the rows on whether ``column`` is blank, then recombines: a filled + cell is kept as-is (after checking its value is allowed), while each blank + (wildcard) cell is cross-joined with the allowed values so it fans out to + one row per value. Splitting first is the trick that keeps any blank out of + the merge key (a NaN key would not match itself in a merge). + + I/O Example: + table: column = "direction" + path_id direction allowed_values = ["forward", "reverse"] + CQ-NQ forward + N1-CNSW # blank: a wildcard + + returns: + path_id direction + CQ-NQ forward # filled and allowed: kept as-is + N1-CNSW forward # blank: fanned out to every allowed value + N1-CNSW reverse + """ + is_wildcard = table[column].isna() + _raise_on_disallowed_values(table.loc[~is_wildcard, column], allowed_values, column) + # Every filled cell is now known to be allowed, so it is kept as-is... + concrete = table[~is_wildcard] + # ...and a blank cell fans out to one row per allowed value (a cross join). + allowed_frame = pd.DataFrame({column: allowed_values}) + expanded = table[is_wildcard].drop(columns=column).merge(allowed_frame, how="cross") + return pd.concat([concrete, expanded], ignore_index=True) + + +def _raise_on_disallowed_values( + values: pd.Series, allowed_values: list, column: str +) -> None: + """Raises if any filled value falls outside the allowed set. + + Callers filter every designed drop out before resolving wildcards, so a + disallowed value reaching this point is bad input data or a bug — the run + halts rather than silently dropping the rows. + + I/O Example: + values = [2026, 2025], allowed_values = [2026, 2028], column = "year" + -> raises ValueError naming year and [2025] + """ + # tolist() converts numpy scalars to native types so the message reads + # "[2025]" rather than "[np.int64(2025)]". + disallowed = sorted(set(values.dropna().tolist()) - set(allowed_values)) + if disallowed: + raise ValueError( + f"Cannot resolve wildcards: {column} contains values outside the " + f"allowed set: {disallowed}" + ) diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py new file mode 100644 index 00000000..5daa3c91 --- /dev/null +++ b/src/ispypsa/translator/network.py @@ -0,0 +1,734 @@ +"""Translate the new-format network tables into PyPSA buses and links. + +This module sits in the translator stage, between the templater that produces +the ISPyPSA network tables and the PyPSA model build. A flow path is a corridor +between two sub-regions and a REZ connection joins a renewable energy zone to its +parent sub-region; both are just transmission paths, so the module routes them +through one pipeline and tells them apart (via isp_type) only where the model +must treat them differently. + +The three sparse input tables (limits, expansion options, expansion costs) may +use blank key cells as wildcards; each is first put through _resolve_wildcards +(see translator/helpers.py) to expand those wildcards to concrete rows. Rows +the configuration deliberately excludes (unmodelled REZ paths, disabled +expansions, non-investment-period years) are filtered out before resolution, +logged at INFO. For the limits table a path_id unknown to the paths table +survives that filtering, so _resolve_wildcards raises on it as bad input data. +The options and costs filters cannot make that distinction — an expansion_id +outside the enabled set may be a disabled element, a constraint group or a +typo — so anything outside the enabled elements or investment periods is +filtered and logged rather than raised on. + +_translate_network_to_links is the pipeline orchestrator and reads as the +step-by-step story; each helper's docstring carries an I/O example, and inline +comments mark the non-obvious modelling choices. +""" + +import logging + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.helpers import ( + _annuitised_investment_costs, + _resolve_wildcards, +) + +_LINK_COLUMNS = [ + "isp_name", + "name", + "carrier", + "bus0", + "bus1", + "p_nom", + "p_min_pu", + "p_max_pu", + "build_year", + "lifetime", + "capital_cost", + "p_nom_extendable", + "isp_type", +] + +_LINK_TIMESLICE_LIMIT_COLUMNS = ["name", "attribute", "timeslice", "value"] + + +def _translate_network_geography_to_buses( + network_geography: pd.DataFrame, rezs: str +) -> pd.DataFrame: + """Creates one PyPSA bus per geography in the model. + + REZ buses are only created when REZs are modelled as discrete nodes; + with ``rezs="attached_to_parent_node"`` REZ-located components connect + straight to the parent geography's bus. + + I/O Example: + network_geography: + geo_id geo_type region_id subregion_id + NQ subregion QLD NQ + Q1 rez QLD NQ + + rezs = "discrete_nodes" returns: + name + NQ + Q1 + + rezs = "attached_to_parent_node" returns: + name + NQ + """ + buses = network_geography + if rezs != "discrete_nodes": + buses = buses[buses["geo_type"] != "rez"] + buses = buses.loc[:, ["geo_id"]].rename(columns={"geo_id": "name"}) + return buses.reset_index(drop=True) + + +def _translate_network_to_links( + ispypsa_tables: dict[str, pd.DataFrame], + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Translates the network tables into PyPSA links and per-timeslice limits. + + Existing links carry the larger of their forward and reverse capacities as + p_nom; limits for each demand condition are returned per unit of p_nom in + the link_timeslice_limits table. Expansion links are built per investment + period from the unified expansion options/costs tables (their total + capacity is capped by the expansion-limit custom constraints built in + ispypsa.translator.constraints). + + Returns: + Tuple of (links, link_timeslice_limits) in PyPSA friendly format. + """ + # Existing + rez_ids = _rez_geo_ids(ispypsa_tables["network_geography"]) + paths = _drop_rez_paths_if_not_modelled( + ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs + ) + limits = _drop_limits_for_unmodelled_paths( + ispypsa_tables["network_transmission_path_limits"], + ispypsa_tables["network_transmission_paths"], + paths, + ) + limits = _resolve_path_limits( + limits, + paths, + config.network.transmission_default_limit, + ) + existing_links = _build_existing_links( + paths, + limits, + rez_ids, + config.temporal.range.start_year, + ) + link_timeslice_limits = _translate_timeslice_limits_to_pu(limits, existing_links) + + # Expansion + enabled_ids = _enabled_expansion_element_ids( + existing_links, + config.network.transmission_expansion, + config.network.rez_transmission_expansion, + ) + options = _resolve_expansion_options( + ispypsa_tables["network_expansion_options"], enabled_ids + ) + options = _pair_forward_and_reverse_options(options) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + enabled_ids, + config.temporal.capacity_expansion.investment_periods, + config.wacc, + config.network.annuitisation_lifetime, + ) + expansion_links = _build_expansion_links(existing_links, options, costs) + links = pd.concat([existing_links, expansion_links], ignore_index=True) + return links.loc[:, _LINK_COLUMNS], link_timeslice_limits + + +def _rez_geo_ids(network_geography: pd.DataFrame) -> set[str]: + """geo_ids of the REZ entries in the geography. + + I/O Example: + geo_id=NQ/geo_type=subregion, geo_id=Q1/geo_type=rez -> {"Q1"} + """ + return set(network_geography.loc[network_geography["geo_type"] == "rez", "geo_id"]) + + +def _drop_rez_paths_if_not_modelled( + paths: pd.DataFrame, rez_ids: set[str], rezs: str +) -> pd.DataFrame: + """Drops REZ-to-parent paths when REZs are not modelled as discrete nodes. + + I/O Example: + paths: + path_id geo_from geo_to + CQ-NQ CQ NQ + Q1-NQ Q1 NQ # Q1 is a REZ, so geo_from is in rez_ids + + rez_ids = {"Q1"} + + rezs = "attached_to_parent_node" returns: + path_id geo_from geo_to + CQ-NQ CQ NQ + + rezs = "discrete_nodes" returns paths unchanged. + """ + if rezs == "discrete_nodes": + return paths + return paths[~paths["geo_from"].isin(rez_ids)] + + +def _drop_limits_for_unmodelled_paths( + limits: pd.DataFrame, all_paths: pd.DataFrame, modelled_paths: pd.DataFrame +) -> pd.DataFrame: + """Drops limit rows for paths configured out of the model. + + REZ-to-parent paths leave the modelled set when REZs are attached to their + parent nodes (see _drop_rez_paths_if_not_modelled); their limit rows are + config-driven selection, filtered out here (logged at INFO) before wildcard + resolution. Only + rows naming a known-but-unmodelled path drop: blank (wildcard) path_ids ride + through, and a path_id unknown to the paths table survives to + _resolve_wildcards, which raises on it. + + I/O Example: + all_paths: path_id in {CQ-NQ, Q1-NQ} + modelled_paths: path_id in {CQ-NQ} + + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 + Q1-NQ forward 750 # known but unmodelled: dropped + 500 # blank: a wildcard, kept + + returns: + path_id direction timeslice capacity + CQ-NQ forward 1400 + 500 + """ + unmodelled = set(all_paths["path_id"]) - set(modelled_paths["path_id"]) + dropped = limits["path_id"].isin(unmodelled) + if dropped.any(): + logging.info( + f"Filtered limit rows for paths configured out of the model: " + f"{sorted(set(limits.loc[dropped, 'path_id']))}" + ) + return limits[~dropped] + + +def _resolve_path_limits( + limits: pd.DataFrame, paths: pd.DataFrame, default_limit: float +) -> pd.DataFrame: + """Resolves the sparse limits table to one row per modelled path, direction + and timeslice, then fills empty capacities with the system default. + + Blank path_id, direction or capacity cells are wildcards (see the + network_transmission_path_limits schema). The limits arrive already filtered + to the modelled paths (see _drop_limits_for_unmodelled_paths), so + _resolve_wildcards expands the path_id and direction wildcards against the + modelled paths and the two directions, raising on any unknown path_id or + direction; timeslice rides along, so a blank-timeslice row stays the + snapshot-level fallback that pypsa_build applies later. A blank capacity then + takes the configured transmission default — the nan_fill the schema declares + for the capacity column — which is how a no-data path (a single all-wildcard + row) becomes a default-limit link in both directions. + + I/O Example (blank cells are wildcards): + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 # specific to CQ-NQ forward + 900 # blank path + direction: a global default + N1-CNSW # no-data row: blank direction and capacity + + paths: path_id in {CQ-NQ, N1-CNSW}; default_limit = 100000 + + returns: + path_id direction timeslice capacity + CQ-NQ forward 1400 # the specific row + CQ-NQ reverse 900 # the global default fills CQ-NQ reverse + N1-CNSW forward 100000 # the no-data row is more specific than the + N1-CNSW reverse 100000 # global default; its blank capacity -> default + """ + allowed_values = { + "path_id": list(paths["path_id"]), + "direction": ["forward", "reverse"], + } + limits = _resolve_wildcards(limits, allowed_values, ["capacity"]) + limits["capacity"] = limits["capacity"].fillna(default_limit) + return limits + + +def _build_existing_links( + paths: pd.DataFrame, + limits: pd.DataFrame, + rez_ids: set[str], + start_year: int, +) -> pd.DataFrame: + """Builds one existing (non-extendable) PyPSA link per transmission path. + + I/O Example: + paths: + path_id geo_from geo_to carrier + CQ-NQ CQ NQ AC + N1-CNSW N1 CNSW AC # REZ path, no limit data + + limits: + path_id direction timeslice capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + N1-CNSW forward 100000 # defaulted upstream + N1-CNSW reverse 100000 + + returns (abridged): + isp_name name bus0 bus1 p_nom p_min_pu isp_type + CQ-NQ CQ-NQ_existing CQ NQ 1910 0.0 flow_path # p_nom = max(1400, 1910) + N1-CNSW N1-CNSW_existing N1 CNSW 100000 0.0 rez + """ + links = paths.rename( + columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} + ) + links = _add_p_nom(links, limits) + links["name"] = links["isp_name"] + "_existing" + links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") + links["build_year"] = start_year - 1 + links["lifetime"] = np.inf + links["capital_cost"] = np.nan + links["p_nom_extendable"] = False + # Defaults; the limits come from link_timeslice_limits, which overrides them. + links["p_min_pu"] = 0.0 + links["p_max_pu"] = 1.0 + return links + + +def _add_p_nom(links: pd.DataFrame, limits: pd.DataFrame) -> pd.DataFrame: + """Sets p_nom to the largest capacity across every limit row of the path. + + Both directions and all timeslices pool into one max: the timeslice = NaN + fallback has to be included because at this stage we don't yet know which + snapshots it will cover, and taking the larger direction keeps every + per-unit limit in [-1, 1]. The link's reverse limit is not set here — it is + carried per snapshot in link_timeslice_limits (a named timeslice or the + timeslice = NaN fallback), which under the coverage contract + (Open-ISP/ISPyPSA#123) sets p_min_pu on every snapshot, leaving the link's + static p_min_pu at its inert default. No-data paths arrive already + defaulted (see _resolve_path_limits), so they get the default limit like + any other path. + + I/O Example: + links: + isp_name + CQ-NQ + PAR-NEW # new parallel corridor, zero capacity + + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward 1400 # NaN-timeslice fallback + CQ-NQ reverse 1910 + PAR-NEW forward 0 + PAR-NEW reverse 0 + + returns: + isp_name p_nom + CQ-NQ 1910 # the reverse row is the largest + PAR-NEW 0 + """ + max_capacity = ( + limits.groupby("path_id", as_index=False)["capacity"] + .max() + .rename(columns={"path_id": "isp_name", "capacity": "p_nom"}) + ) + return links.merge(max_capacity, on="isp_name", how="left") + + +def _translate_timeslice_limits_to_pu( + limits: pd.DataFrame, existing_links: pd.DataFrame +) -> pd.DataFrame: + """Expresses each limit per unit of its link's p_nom. + + Forward limits become p_max_pu values and reverse limits p_min_pu values. + Named-timeslice rows carry their timeslice; a timeslice = NaN row is the + fallback applied to snapshots no named timeslice covers (the coverage + contract is Open-ISP/ISPyPSA#123). pypsa_build expands these into per-snapshot + series via the timeslice_snapshots mapping. Zero-p_nom links (new parallel + corridors) are skipped — all their limits are zero and the per-unit form is + undefined. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse , 1000 # NaN-timeslice fallback + + existing_links (abridged): + isp_name name p_nom + CQ-NQ CQ-NQ_existing 1400 + + returns: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 # 1200/1400 + CQ-NQ_existing p_max_pu qld_winter_reference 1.0 # 1400/1400 + CQ-NQ_existing p_min_pu , -0.714 # fallback, -1000/1400 + """ + rows = limits.merge( + existing_links.loc[:, ["isp_name", "name", "p_nom"]], + left_on="path_id", + right_on="isp_name", + ) + # Zero-p_nom links (new parallel corridors) have no per-unit form, so they + # contribute no timeslice rows. + rows = rows[rows["p_nom"] > 0] + rows["attribute"] = rows["direction"].map( + {"forward": "p_max_pu", "reverse": "p_min_pu"} + ) + # Reverse flow is negative, so reverse limits become negative p_min_pu values. + sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) + rows["value"] = sign * rows["capacity"] / rows["p_nom"] + return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) + + +def _enabled_expansion_element_ids( + existing_links: pd.DataFrame, + transmission_expansion: bool, + rez_transmission_expansion: bool, +) -> list[str]: + """The isp_names of the existing links whose expansion the config enables. + + ``transmission_expansion`` gates flow paths between (sub)regions; + ``rez_transmission_expansion`` gates REZ connection paths. The result is the + enabled expansion_id set the options and costs tables are filtered to (see + _keep_rows_for_enabled_elements) and then resolved against, so an option or + cost for a disabled or non-modelled element drops out before resolution. + + I/O Example: + existing_links: + isp_name isp_type + CQ-NQ flow_path + Q1-NQ rez + + transmission_expansion=True, rez_transmission_expansion=False + returns ["CQ-NQ"] # rez path Q1-NQ gated out + """ + enabled = set() + if transmission_expansion: + is_flow_path = existing_links["isp_type"] == "flow_path" + enabled |= set(existing_links.loc[is_flow_path, "isp_name"]) + if rez_transmission_expansion: + is_rez = existing_links["isp_type"] == "rez" + enabled |= set(existing_links.loc[is_rez, "isp_name"]) + return sorted(enabled) + + +def _resolve_expansion_options( + options: pd.DataFrame, enabled_ids: list[str] +) -> pd.DataFrame: + """Resolves the expansion-options wildcards to one row per enabled element + and physical direction. + + A blank expansion_id is a system-wide default option, and a blank + expansion_type covers both directions of one element in a single (symmetric) + row. constraint_relaxation rows are not physical paths — they are set aside + first and become relaxation generators in ispypsa.translator.constraints. + Options for disabled or non-modelled elements are config-driven selection, + filtered out next (logged at INFO). _resolve_wildcards then expands the + blanks against the enabled elements and the two physical directions. + + Each element's forward and reverse must form a coherent pair from one option + (the schema's expansion_options_pair_forward_and_reverse rule); + _check_both_directions_defined and _check_forward_reverse_share_an_option + stand in for that rule once resolved, until schema validation lands. + + I/O Example (blank cells are wildcards; each element's forward and reverse + share one expansion_option): + options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 BigLine + CQ-NQ reverse 800 BigLine + forward 500 Default # blank id: + reverse 400 Default # a default pair + SWQLD1 constraint_relaxation 400 Relax # standalone, dropped + + enabled_ids = ["CQ-NQ", "Q1-NQ"] + + returns (CQ-NQ keeps its BigLine pair; Q1-NQ inherits the Default pair): + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 BigLine + CQ-NQ reverse 800 BigLine + Q1-NQ forward 500 Default + Q1-NQ reverse 400 Default + """ + # constraint_relaxation options are routed to ispypsa.translator.constraints, + # not dropped, so they are set aside before wildcard resolution (which would + # otherwise raise on them). + options = options[options["expansion_type"] != "constraint_relaxation"] + options = _keep_rows_for_enabled_elements(options, enabled_ids) + allowed_values = { + "expansion_id": enabled_ids, + "expansion_type": ["forward", "reverse"], + } + options = _resolve_wildcards( + options, + allowed_values, + ["allowed_expansion", "expansion_option"], + ) + _check_both_directions_defined(options) + _check_forward_reverse_share_an_option(options) + return options + + +def _keep_rows_for_enabled_elements( + table: pd.DataFrame, enabled_ids: list[str] +) -> pd.DataFrame: + """Keeps rows whose expansion_id is an enabled element or blank (a wildcard). + + Selecting the enabled elements is config-driven, so it happens before + wildcard resolution rather than inside it. Rows for disabled or non-modelled + elements drop out here, as do rows for constraint groups (their expansion_ids + are constraint_ids, routed to ispypsa.translator.constraints instead). The + filtered ids are logged at INFO — a typo'd expansion_id is indistinguishable + from these designed drops, so the log line is its only trace. + + I/O Example: + table: enabled_ids = ["CQ-NQ"] + expansion_id year cost + CQ-NQ 2026 1000000 + Q1-NQ 2026 500000 # disabled element: dropped + SWQLD1 2026 400000 # constraint group: dropped + 2026 900000 # blank: a wildcard, kept + + returns: + expansion_id year cost + CQ-NQ 2026 1000000 + 2026 900000 + """ + ids = table["expansion_id"] + keep = ids.isna() | ids.isin(enabled_ids) + if not keep.all(): + logging.info( + f"Filtered rows whose expansion_id is not an enabled expansion " + f"element: {sorted(set(ids[~keep]))}" + ) + return table[keep] + + +def _check_both_directions_defined(options: pd.DataFrame) -> None: + """Raises if any element's resolved options define only one direction. + + The schema's expansion_options_pair_forward_and_reverse rule requires an + expansion_id that defines one direction to define the other; an element + arriving with only a forward (or only a reverse) row would otherwise produce + an expansion link with a NaN rating in the missing direction. This stands in + until schema validation lands. + + I/O Example: + options with only a CQ-NQ forward row -> raises (no CQ-NQ reverse). + """ + directions = options.groupby("expansion_id")["expansion_type"].nunique() + one_directional = sorted(directions[directions < 2].index) + if one_directional: + raise ValueError( + f"Expansion options define only one direction for {one_directional}; " + "each element needs both forward and reverse (a blank expansion_type " + "row covers both)." + ) + + +def _check_forward_reverse_share_an_option(options: pd.DataFrame) -> None: + """Raises if any element's resolved forward and reverse trace to different + options. + + Forward and reverse for an expansion_id must come from one physical option, so + the single cost keyed on expansion_id applies to a coherent pair. Wildcard + resolution resolves the two directions independently, so a malformed input + could pair a forward from one option with a reverse from another; this guard + catches that. A blank expansion_option counts as its own label (nunique with + dropna=False), so an unlabelled direction only pairs with another unlabelled + one — nunique's default NaN-dropping would otherwise let a blank/named + mismatch through. The schema's expansion_options_pair_forward_and_reverse + rule is the upstream enforcement — this stands in until schema validation + lands. + + I/O Example: + options with CQ-NQ forward from "BigLine" and CQ-NQ reverse from "Default" + -> raises (the two directions disagree on the option). + """ + options_per_element = options.groupby("expansion_id")["expansion_option"].nunique( + dropna=False + ) + mismatched = sorted(options_per_element[options_per_element > 1].index) + if mismatched: + raise ValueError( + f"Forward and reverse expansion options disagree for {mismatched}; " + "each element's forward and reverse must come from the same option." + ) + + +def _pair_forward_and_reverse_options(options: pd.DataFrame) -> pd.DataFrame: + """Pairs each element's forward and reverse expansion rows into one row. + + The options arrive already resolved to physical forward/reverse rows (see + _resolve_expansion_options), so this only reshapes them. + + I/O Example: + options: + expansion_id expansion_type allowed_expansion + CQ-NQ forward 1000 + CQ-NQ reverse 900 + + returns: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + """ + paired = ( + options.pivot( + index="expansion_id", columns="expansion_type", values="allowed_expansion" + ) + .reindex(columns=["forward", "reverse"]) + .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) + ) + paired.columns.name = None + return paired.reset_index() + + +def _prepare_expansion_costs( + expansion_costs: pd.DataFrame, + enabled_ids: list[str], + investment_periods: list[int], + wacc: float, + asset_lifetime: int, +) -> pd.DataFrame: + """Resolves the expansion-cost wildcards to the enabled elements and + investment periods, then annuitises them. + + Blank expansion_id or year cells are wildcards (see the + network_transmission_path_expansion_costs schema): an empty expansion_id is a + table-wide default cost, an empty year a static cost across the investment + periods. Costs for disabled elements, constraint groups (routed to + ispypsa.translator.constraints) and years outside the investment periods are + all designed selection, filtered out first (logged at INFO). _resolve_wildcards + then expands the blanks against the enabled elements and the investment + periods. A blank cost then resolves to free — the nan_fill the schema + declares for the cost column. Year values are labels matched against the + config's investment periods as ints — no financial vs calendar year + interpretation happens here; the config's year_type decides what span of + time the labels denote, so the table just has to label years the same way + the config does (templated tables carry financial-year ending years). + + I/O Example (a blank year is a static cost across the investment periods): + expansion_costs: + expansion_id year cost + CQ-NQ 1000 # blank year: applies to every investment period + CQ-NQ 2028 1200 # a specific-year override + Q1-NQ 2025 900 # 2025 is not an investment period -> dropped + + enabled_ids=["CQ-NQ", "Q1-NQ"], investment_periods=[2026, 2028], wacc, asset_lifetime + + returns (cost annuitised; Q1-NQ falls away, its only row was out of period): + expansion_id year capital_cost + CQ-NQ 2026 annuitise(1000) # the static row fills 2026 + CQ-NQ 2028 annuitise(1200) # the 2028 override beats the static row + """ + costs = _keep_rows_for_enabled_elements(expansion_costs, enabled_ids) + costs = _keep_rows_for_investment_period_years(costs, investment_periods) + allowed_values = {"expansion_id": enabled_ids, "year": investment_periods} + costs = _resolve_wildcards(costs, allowed_values, ["cost"]) + # Every surviving year is now an investment period; cast so expansion link + # names read e.g. CQ-NQ_exp_2026 rather than CQ-NQ_exp_2026.0. + costs["year"] = costs["year"].astype("int64") + costs["cost"] = costs["cost"].fillna(0.0) + costs["capital_cost"] = _annuitised_investment_costs( + costs["cost"], wacc, asset_lifetime + ) + return costs.loc[:, ["expansion_id", "year", "capital_cost"]] + + +def _keep_rows_for_investment_period_years( + costs: pd.DataFrame, investment_periods: list[int] +) -> pd.DataFrame: + """Keeps cost rows whose year is an investment period or blank (a wildcard). + + The costs table spans the IASR horizon while the model only prices the + configured investment periods, so selecting those years is config-driven and + happens before wildcard resolution rather than inside it. The filtered years + are logged at INFO. + + I/O Example: + costs: investment_periods = [2026, 2028] + expansion_id year cost + CQ-NQ 2025 900000 # not an investment period: dropped + CQ-NQ 2026 1000000 + CQ-NQ 800000 # blank: a wildcard, kept + + returns: + expansion_id year cost + CQ-NQ 2026 1000000 + CQ-NQ 800000 + """ + years = costs["year"] + keep = years.isna() | years.isin(investment_periods) + if not keep.all(): + logging.info( + f"Filtered expansion cost rows for years outside the investment " + f"periods: {sorted(int(year) for year in set(years[~keep]))}" + ) + return costs[keep] + + +def _build_expansion_links( + existing_links: pd.DataFrame, + options: pd.DataFrame, + costs: pd.DataFrame, +) -> pd.DataFrame: + """Builds one extendable PyPSA link per expandable path and investment period. + + p_nom starts at 0 and is unbounded on its own — the expansion-limit custom + constraints built in ispypsa.translator.constraints cap the p_nom built + across a path's expansion links at max(forward, reverse) of the selected + option. p_max_pu and p_min_pu are the forward and reverse capacities per unit + of that max, so whatever p_nom is built delivers forward and reverse flow in + the option's proportion, both bounded by [-1, 1]. Using the max as the + denominator means only a both-zero option divides by zero, which falls back + to symmetric defaults. + + I/O Example: + options: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + + costs: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + + existing_links (abridged): + isp_name bus0 bus1 carrier isp_type + CQ-NQ CQ NQ AC flow_path + + returns (abridged): + isp_name name p_nom p_nom_extendable p_max_pu p_min_pu build_year capital_cost + CQ-NQ CQ-NQ_exp_2026 0.0 True 1.0 -0.9 2026 81.4 + """ + links = costs.merge(options, on="expansion_id") + links = links.merge( + existing_links.loc[:, ["isp_name", "bus0", "bus1", "carrier", "isp_type"]], + left_on="expansion_id", + right_on="isp_name", + ) + links["name"] = links["expansion_id"] + "_exp_" + links["year"].astype(str) + # Starts at 0 and is unbounded on its own; ispypsa.translator.constraints caps the + # p_nom built across a path's expansion links at max(forward, reverse) of the option. + links["p_nom"] = 0.0 + links["p_nom_extendable"] = True + # Per unit of max(forward, reverse) keeps both ratings in [-1, 1]; only a both-zero + # option hits the symmetric divide-by-zero fallback. + capacity = links[["forward_capacity", "reverse_capacity"]].max(axis=1) + links["p_max_pu"] = np.where( + capacity > 0, links["forward_capacity"] / capacity, 1.0 + ) + links["p_min_pu"] = np.where( + capacity > 0, -1.0 * links["reverse_capacity"] / capacity, -1.0 + ) + links["build_year"] = links["year"] + links["lifetime"] = np.inf + return links diff --git a/src/ispypsa/validation/schemas/network_expansion_options.yaml b/src/ispypsa/validation/schemas/network_expansion_options.yaml index 02108c82..66cfce6d 100644 --- a/src/ispypsa/validation/schemas/network_expansion_options.yaml +++ b/src/ispypsa/validation/schemas/network_expansion_options.yaml @@ -4,7 +4,16 @@ unique: - [expansion_id, expansion_type] description: > Defines the selected expansion option and allowed capacity increase for - expandable network elements. + expandable network elements, looked up per element and expansion type. + + The key columns — expansion_id and expansion_type — identify the combination an + option applies to. Either may be left empty to act as a wildcard: an empty + expansion_id is a table-wide default across all elements, and an empty + expansion_type applies to forward, reverse and constraint_relaxation alike. + Where several rows match, the most specific (fewest empty keys) wins; the + custom_validation rule requires that resolution to be unambiguous. Unlike the + limits and costs tables there is no coverage requirement — expansion is + optional. Covers both physical transmission paths and group constraints that can be relaxed through expansion. Physical paths emit two rows (forward and @@ -23,10 +32,26 @@ description: > If absent: No network elements can be expanded by the model. +custom_validation: + - name: expansion_options_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows.) + - name: expansion_options_pair_forward_and_reverse + description: > + Forward and reverse for an element must come from one option, so each + expansion_id (blank included) that defines one direction must define the + other with the same expansion_option. A blank expansion_type covers both + directions in one row and satisfies this alone; constraint_relaxation has no + direction and stands alone. This keeps an element's resolved forward and + reverse tied to the same option, so the single cost keyed on expansion_id + applies to a coherent pair. columns: expansion_id: type: string - required: true + required: false allowed_values_from: - network_transmission_paths: path_id - custom_constraints_rhs: constraint_id @@ -34,15 +59,23 @@ columns: Identifier for the expandable network element. Maps to path_id in network_transmission_paths for physical paths, or to constraint_id in custom_constraints_rhs for group constraints. + + If absent (or empty): + Wildcard — applies to every expandable element no more specific row covers + (a system-wide default option). expansion_type: type: string - required: true + required: false allowed_values: [forward, reverse, constraint_relaxation] description: > Which direction or constraint relaxation this row's ``allowed_expansion`` applies to. ``forward`` and ``reverse`` are the two directions of a physical path; ``constraint_relaxation`` is the single relaxation of a group constraint. + + If absent (or empty): + Wildcard — applies to forward, reverse and constraint_relaxation alike for + any combination no more specific row covers. allowed_expansion: type: float required: true @@ -54,7 +87,12 @@ columns: type: string required: false description: > - Name of the selected augmentation option, retained for traceability. + Name of the selected augmentation option. Not read as model data, but it + ties an element's directions together: the + expansion_options_pair_forward_and_reverse rule requires an expansion_id's + forward and reverse rows to carry the same expansion_option, so directions + labelled from different options are rejected. If absent (or empty): - No effect on the model; the column is informational only. + No option name is recorded. A blank still participates in the pairing + rule, pairing only with another blank. diff --git a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml index d9ad83d3..b6a85053 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_expansion_costs.yaml @@ -3,7 +3,15 @@ required: false unique: - [expansion_id, year] description: > - Time-varying annualised expansion costs for network elements. + Time-varying annualised expansion costs for network elements, looked up per + expandable element and investment period. + + The key columns — expansion_id and year — identify the combination a cost + applies to. Either may be left empty to act as a wildcard: an empty year is a + static cost across all investment periods, and an empty expansion_id is a + table-wide default across all elements. Where several rows match, the most + specific (fewest empty keys) wins; the custom_validation rules require that + resolution to be both unambiguous and complete. Covers both physical transmission paths and group constraints, merging flow path and REZ transmission expansion costs into a single long-format table. @@ -20,26 +28,56 @@ description: > increase used to express costs per MW. If absent: - Network expansion is treated as free. + No network elements are expandable; a populated options table requires matching + costs (see custom_validation). +custom_validation: + - name: costs_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows.) + - name: costs_cover_every_element_and_investment_period + description: > + Coverage spans every expandable element (an expansion_id present in + network_expansion_options once its own wildcards resolve) and every + model-config investment period; a single row with both keys empty covers it + all. columns: expansion_id: type: string - required: true + required: false allowed_values_from: - network_expansion_options: expansion_id - description: Identifier for the expandable network element. + description: > + Identifier for the expandable network element. + + If absent (or empty): + Wildcard — applies to every expandable element no more specific row covers + (a table-wide default). year: type: int - required: true - description: Year the cost applies to. + required: false + description: > + Investment period the cost applies to, matching the labels in the model + configuration's investment periods (for templated tables these are + financial-year ending years, e.g. 2025 for FY 2024-25). + + If absent (or empty): + Wildcard — a static cost applied to every investment period the element has + no explicit year row for. cost: type: float - required: true + required: false units: $/MW gte: 0.0 + nan_fill: 0.0 description: > Annualised expansion cost, expressed per MW of the maximum directional capacity: total option cost divided by max(forward, reverse) of the selected option's allowed_expansion. For symmetric options (forward == reverse) this is unambiguous; for asymmetric options the divisor is whichever direction has the larger capacity increase. + + If absent (or empty): + The expansion is free (equivalent to a cost of 0). diff --git a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml index 977a8bbf..8c41bae4 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml @@ -3,34 +3,59 @@ required: false unique: - [path_id, direction, timeslice] description: > - Transmission capacity limits for each path, by direction and optionally by - demand condition (timeslice). + Transmission capacity limits, looked up per path, flow direction and demand + condition (timeslice). - A limit with a timeslice applies to that demand condition only; a limit with - no timeslice is a fallback for the conditions no per-timeslice limit covers - (see the timeslice column). A path may carry either or both. In the default - IASR data, flow-path limits vary by demand condition while REZ-to-subregion - connection limits are static, but that reflects the source tables rather than - a schema rule. + The three key columns — path_id, direction and timeslice — identify the + combination a limit applies to. Any key may be left empty to act as a wildcard + matching every value of that dimension: an empty timeslice is a fallback across + demand conditions, an empty direction applies to both directions, and an empty + path_id is a table-wide default. Where several rows match a combination the most + specific (fewest empty keys) wins; the custom_validation rules require that + resolution to be both unambiguous and complete. Source tables: - `flow_path_transfer_capability` - `initial_transmission_limits` If absent: - No transmission capacity limits are enforced. + There are no transmission paths to limit; a populated paths table requires + matching limits (see custom_validation). +custom_validation: + - name: limits_resolve_unambiguously + description: > + No combination may be matched by two rows tied for the fewest wildcards — + resolution must have a single most-specific winner. (The combination and + wildcard model is described in the table description; this generalises + timeslices.no_overlapping_windows to all three keys.) + - name: limits_cover_every_path_direction_and_timeslice + description: > + A path that names timeslices must name them all for its region, or add an empty-timeslice + fallback, in both directions; a single all-empty row covers the whole space. + Because the timeslices table's windows partition the year, covering every + region timeslice_id covers every snapshot downstream. columns: path_id: type: string - required: true + required: false allowed_values_from: - network_transmission_paths: path_id - description: Transmission path identifier. + description: > + Transmission path the limit applies to. + + If absent (or empty): + Wildcard — applies to every path no more specific row covers (a table-wide + default). direction: type: string - required: true + required: false allowed_values: [forward, reverse] - description: Power flow direction relative to geo_from and geo_to. + description: > + Power flow direction relative to geo_from and geo_to. + + If absent (or empty): + Wildcard — applies to both directions for any combination no more specific + row covers. timeslice: type: string required: false @@ -38,21 +63,22 @@ columns: - timeslices: timeslice_id description: > Demand condition the limit applies to: the limit binds only at snapshots - inside the timeslice's active windows. + inside the timeslice's active windows. A timeslice_id's region is the prefix + before its first underscore (e.g. qld in qld_peak_demand). If absent (or empty): - The limit is a fallback for this path_id and direction, applying at - snapshots not covered by any non-NaN timeslice row for the same path_id - and direction. A path_id and direction may carry both a no-timeslice row - and per-timeslice rows; where a per-timeslice row covers a snapshot its - limit takes precedence, and the no-timeslice limit fills the remainder. + Wildcard — a fallback across demand conditions, applying at every timeslice + no more specific row covers for the same path and direction. capacity: type: float required: false units: MW gte: 0.0 + nan_fill: {config: network.transmission_default_limit} description: > - Transfer capability limit. + Transfer capability limit for the resolved combination. If absent (or empty): - The path is constraint-modelled with no explicit physical limit. + The combination takes the configured transmission default + (network.transmission_default_limit). A value sentinel, distinct from 0, + which is a genuine zero-capacity corridor. diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py new file mode 100644 index 00000000..2e188d6a --- /dev/null +++ b/tests/test_translator/test_network.py @@ -0,0 +1,1044 @@ +import pandas as pd +import pytest + +from ispypsa.translator.helpers import _annuitised_investment_costs +from ispypsa.translator.network import ( + _drop_limits_for_unmodelled_paths, + _resolve_expansion_options, + _translate_network_geography_to_buses, + _translate_network_to_links, +) + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25) — expansion cost expectations are multiples of this. +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def test_translate_network_geography_to_buses_discrete_nodes(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses(network_geography, "discrete_nodes") + + expected = csv_str_to_df(""" + name + NQ + CNSW + Q1 + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_translate_network_geography_to_buses_attached_to_parent_node(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses( + network_geography, "attached_to_parent_node" + ) + + expected = csv_str_to_df(""" + name + NQ + """) + pd.testing.assert_frame_equal(result, expected) + + +def _network_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """New-format network tables with one flow path (CQ-NQ, asymmetric winter + limits, expandable), one REZ path with limits (Q1-NQ) and one collapsed + REZ path with no limit data (N1-CNSW).""" + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + N1, rez, NSW + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + N1-CNSW, N1, CNSW, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_summer_typical, 1300 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_peak_demand, 1440 + CQ-NQ, reverse, qld_summer_typical, 1600 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, forward, qld_winter_reference, 750 + Q1-NQ, reverse, qld_winter_reference, 750 + N1-CNSW, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, CQ-NQ Option 1 + CQ-NQ, reverse, 900, CQ-NQ Option 1 + Q1-NQ, forward, 500, Q1 Option 1 + Q1-NQ, reverse, 500, Q1 Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + # 2025 is outside the sample config's investment periods (2026, 2028) and + # is expected to be filtered out. + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2025, 900000 + CQ-NQ, 2026, 1000000 + Q1-NQ, 2026, 500000 + SWQLD1, 2026, 400000 + """) + return tables + + +def test_translate_network_to_links(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1910, 0.0, 1.0, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, 0.0, 1.0, 2025, inf, , False, rez + N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, 0.0, 1.0, 2025, inf, , False, rez + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.628272 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.680628 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.732984 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.753927 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -0.837696 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + N1-CNSW_existing,p_max_pu, , 1.0 + N1-CNSW_existing,p_min_pu, , -1.0 + Q1-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + Q1-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_p_nom_when_forward_exceeds_reverse( + csv_str_to_df, sample_model_config +): + """p_nom is the larger of the two directions. With forward (peak 1500) above + reverse, p_nom takes it, so a forward limit above winter stays at p_max_pu + 1.0 rather than over-crediting the link above its rating + (Open-ISP/ISPyPSA#123, item 3).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Forward peak (1500) exceeds winter (1400): winter is no longer the max. + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1500 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_winter_reference, 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1500, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 1.0 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.933333 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -0.666667 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_p_nom_when_reverse_exceeds_forward( + csv_str_to_df, sample_model_config +): + """p_nom is the larger of the two directions. With reverse (1400) above + forward (1000), p_nom takes reverse, so the forward limit is p_max_pu < 1 and + the reverse limit reaches p_min_pu -1.0 in link_timeslice_limits — both + per-unit limits within [-1, 1].""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Reverse winter (1400) exceeds forward winter (1000): reverse sets p_nom. + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_winter_reference, 1000 + CQ-NQ, reverse, qld_winter_reference, 1400 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.714286 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_nan_timeslice_fallback( + csv_str_to_df, sample_model_config +): + """A timeslice = NaN row is a fallback limit: it counts towards p_nom (we + don't yet know which snapshots it covers) and flows into link_timeslice_limits + carrying a NaN timeslice (Open-ISP/ISPyPSA#123).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + # Forward: a named peak row plus a NaN-timeslice fallback (1400, the larger). + # Reverse: only a NaN-timeslice fallback (1000). + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, , 1400 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # p_nom = max(forward 1400, reverse 1000) = 1400; the reverse fallback is in + # link_timeslice_limits, so the static p_min_pu stays at 0.0. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + # The fallback rows keep their NaN timeslice (blank field after the comma). + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.714286 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute", "value"]).reset_index( + drop=True + ), + expected_limits.sort_values(["attribute", "value"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_zero_forward_keeps_reverse( + csv_str_to_df, sample_model_config +): + """A link with a zero forward limit and a non-zero reverse limit takes its + reverse capacity as p_nom, so the reverse capability is preserved rather than + collapsing the link to zero flow in both directions.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_winter_reference, 0 + CQ-NQ, reverse, qld_winter_reference, 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # p_nom = max(0, 1000) = 1000; reverse fully available (via the timeslice + # limit), forward pinned to 0. Static p_min_pu stays at 0.0. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1000, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.0 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_no_data_path_gets_default_fallback( + csv_str_to_df, sample_model_config +): + """A path with no limit data (a collapsed all-NaN row) is normalised to a + symmetric timeslice = NaN fallback at the default limit, so it tiles the year + the same way a data path does (Open-ISP/ISPyPSA#123).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + N1-CNSW, N1, CNSW, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + N1-CNSW, , , + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # No-data path takes the default limit (1000000) in both directions. + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + N1-CNSW, N1-CNSW_existing, AC, N1, CNSW, 1000000, 0.0, 1.0, 2025, inf, , False, rez + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False, rtol=1e-5) + + # Both directions get a NaN-timeslice fallback per unit of p_nom. + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + N1-CNSW_existing, p_max_pu, , 1.0 + N1-CNSW_existing, p_min_pu, , -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["attribute"]).reset_index(drop=True), + expected_limits.sort_values(["attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The flow path's expansion link is gone; the REZ path's remains. The + # existing links' content is pinned by test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + expansion, expected_expansion, check_dtype=False, rtol=1e-5 + ) + + +def test_translate_network_to_links_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The REZ path's expansion link is gone; the flow path's remains. The + # existing links' content is pinned by test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion, expected_expansion, check_dtype=False, rtol=1e-5 + ) + + +def test_translate_network_to_links_all_expansion_disabled( + csv_str_to_df, sample_model_config +): + """With both expansion flags off no element is enabled, so the populated + options and costs tables resolve against an empty allowed set and no + expansion links are built.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) + + +def test_translate_network_to_links_rezs_attached_to_parent_node( + csv_str_to_df, sample_model_config +): + """With REZs attached to their parent nodes the REZ paths leave the model, + and their limit, option and cost rows are filtered out before wildcard + resolution rather than raising as out-of-set values.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # REZ paths (and their expansion links and timeslice limits) are dropped. + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1910, 0.0, 1.0, 2025, inf, , False, flow_path + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.628272 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.680628 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 0.732984 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -0.753927 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -0.837696 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_zero_capacity_parallel_path( + csv_str_to_df, sample_model_config +): + """A new parallel corridor has zero existing capacity, given as a + timeslice = NaN fallback of 0 in both directions. It becomes an inert + existing link at p_nom 0 (its buildable capacity is modelled by expansion + links), and the zero-p_nom link is skipped when translating per-timeslice + limits, so it yields no link_timeslice_limits rows and no 0/0 division.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CNSW-SNW, CNSW, SNW, AC + """) + # Zero existing capacity as a timeslice = NaN fallback (covers the year). + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CNSW-SNW, forward, , 0 + CNSW-SNW, reverse, , 0 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, 0.0, 1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_empty_expansion_tables( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) + + +def test_translate_network_to_links_options_populated_costs_empty( + csv_str_to_df, sample_model_config +): + """A populated options table with no matching costs is invalid per the + schema (costs_cover_every_element_and_investment_period), but until schema + validation lands it silently produces no expansion links.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) + + +def test_translate_network_to_links_options_empty_costs_populated( + csv_str_to_df, sample_model_config +): + """Costs without options mean no element is expandable, so no expansion + links are built. This input is actually invalid per the schema — costs + declare expansion_id allowed_values_from network_expansion_options, and an + empty options table leaves no allowed values — so once the validation layer + lands it will be rejected upstream rather than silently building nothing.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # No expansion links; the existing links' content is pinned by + # test_translate_network_to_links. + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected_expansion = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(expansion, expected_expansion, check_dtype=False) + + +def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_path_id_wildcard_default( + csv_str_to_df, sample_model_config +): + """A blank path_id limit row is a table-wide default: a path with no rows of + its own (Q1-NQ) inherits it in both directions, while a path with its own + rows (CQ-NQ) keeps them, being more specific.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1500 + CQ-NQ, reverse, , 1000 + , , , 800 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + """) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1500, 0.0, 1.0, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 800, 0.0, 1.0, 2025, inf, , False, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + # CQ-NQ keeps its own per-unit limits; Q1-NQ takes the global default both ways. + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, , 1.0 + CQ-NQ_existing, p_min_pu, , -0.666667 + Q1-NQ_existing, p_max_pu, , 1.0 + Q1-NQ_existing, p_min_pu, , -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values(["name", "attribute"]).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute"]).reset_index(drop=True), + rtol=1e-5, + ) + + +def test_translate_network_to_links_static_cost_across_investment_periods( + csv_str_to_df, sample_model_config +): + """A blank year cost row is a static cost across every investment period, so + an expansion link is built for each (2026 and 2028 in the sample config).""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 500, Opt + CQ-NQ, reverse, 500, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, , 1000000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + CQ-NQ, CQ-NQ_exp_2028, AC, CQ, NQ, 0.0, -1.0, 1.0, 2028, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_year_override_beats_static_cost( + csv_str_to_df, sample_model_config +): + """A concrete-year cost row overrides the blank-year static row for that + investment period; the static row still fills the other periods.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 500, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, , 1000000 + CQ-NQ, 2028, 1200000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + CQ-NQ, CQ-NQ_exp_2028, AC, CQ, NQ, 0.0, -1.0, 1.0, 2028, inf, {1200000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_id_wildcard_default_cost( + csv_str_to_df, sample_model_config +): + """A blank expansion_id cost row is a table-wide default: an element with no + cost rows of its own (Q1-NQ) inherits it, while CQ-NQ keeps its own cost. It + also fans out to enabled elements with no expansion options (N1-CNSW), which + still build no expansion link.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + , 2026, 500000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].sort_values("name") + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + expansion.reset_index(drop=True), + expected.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_type_wildcard( + csv_str_to_df, sample_model_config +): + """A blank expansion_type option applies to both directions, so the expansion + link gets symmetric forward and reverse capability.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 700, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + """) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + expansion = links[links["p_nom_extendable"]].reset_index(drop=True) + expected = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, p_max_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -1.0, 1.0, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + """) + pd.testing.assert_frame_equal(expansion, expected, check_dtype=False, rtol=1e-5) + + +def test_translate_network_to_links_raises_on_unknown_limits_path_id( + csv_str_to_df, sample_model_config +): + """A limits path_id absent from the paths table is bad input data — unlike a + config-driven drop, it survives the pre-resolution filtering and the wildcard + resolver halts the run on it. In the long run the validation layer will guard + against this example. + + Run with REZs attached to their parent nodes so the filtering is active: the + raise message lists every disallowed value, so asserting it names only the + typo also proves the known-but-unmodelled Q1-NQ row was filtered out and the + blank wildcard row rode through.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + Q1-NQ, forward, , 750 + , , , 500 + CQ-NQQ, forward, , 1200 + """) + + with pytest.raises( + ValueError, + match=r"path_id contains values outside the allowed set: \['CQ-NQQ'\]", + ): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + +def test_translate_network_to_links_logs_config_filtered_rows( + csv_str_to_df, sample_model_config, caplog +): + """Rows filtered out by config-driven selection — limits for unmodelled REZ + paths, options and costs for elements that are not enabled, and cost years + outside the investment periods — are logged at INFO so their absence from + the model can be audited.""" + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + + with caplog.at_level("INFO"): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert ( + "Filtered limit rows for paths configured out of the model: " + "['N1-CNSW', 'Q1-NQ']" + ) in caplog.text + # The options table's filtered ids (SWQLD1 was already set aside as a + # constraint_relaxation row)... + assert ( + "Filtered rows whose expansion_id is not an enabled expansion element: " + "['Q1-NQ']" + ) in caplog.text + # ...and the costs table's, where SWQLD1 is filtered rather than set aside. + assert ( + "Filtered rows whose expansion_id is not an enabled expansion element: " + "['Q1-NQ', 'SWQLD1']" + ) in caplog.text + assert ( + "Filtered expansion cost rows for years outside the investment periods: [2025]" + ) in caplog.text + + +def test_translate_network_to_links_no_filter_log_when_all_rows_used( + csv_str_to_df, sample_model_config, caplog +): + """When every input row survives into the model, no filter log lines fire.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1000 + CQ-NQ, reverse, , 1000 + """) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, , 700, Opt + """) + ispypsa_tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + """) + + with caplog.at_level("INFO"): + _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert "Filtered" not in caplog.text + + +def test_drop_limits_for_unmodelled_paths_limits_empty(csv_str_to_df): + limits = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + paths = csv_str_to_df(""" + path_id + CQ-NQ + """) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_drop_limits_for_unmodelled_paths_paths_empty(csv_str_to_df): + """With no paths at all nothing is 'known but unmodelled', so every limit row + rides through to _resolve_wildcards, which raises on the out-of-set path_id.""" + limits = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + """) + paths = pd.DataFrame(columns=["path_id"]) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_drop_limits_for_unmodelled_paths_both_empty(csv_str_to_df): + limits = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + paths = pd.DataFrame(columns=["path_id"]) + + result = _drop_limits_for_unmodelled_paths(limits, paths, paths) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_resolve_expansion_options_pairs_forward_and_reverse(csv_str_to_df): + """Each option defines both directions; a blank-expansion_id default pair fills + elements with no option of their own, keeping forward and reverse coupled.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + CQ-NQ, reverse, 800, BigLine + , forward, 500, Default + , reverse, 400, Default + SWQLD1, constraint_relaxation, 400, Relax + """) + + result = _resolve_expansion_options(options, ["CQ-NQ", "Q1-NQ"]) + + # CQ-NQ keeps its BigLine pair; Q1-NQ inherits the Default pair; SWQLD1 drops. + expected = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + CQ-NQ, reverse, 800, BigLine + Q1-NQ, forward, 500, Default + Q1-NQ, reverse, 400, Default + """) + pd.testing.assert_frame_equal( + result.sort_values(["expansion_id", "expansion_type"]).reset_index(drop=True), + expected.sort_values(["expansion_id", "expansion_type"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_expansion_options_raises_on_missing_direction(csv_str_to_df): + """An option defining only one direction raises rather than producing an + expansion link with a NaN rating in the missing direction. The check stands + in for the schema's expansion_options_pair_forward_and_reverse rule, which + will reject this input upstream once the validation layer lands.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + """) + + with pytest.raises( + ValueError, match=r"Expansion options define only one direction" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) + + +def test_resolve_expansion_options_raises_on_mismatched_pair(csv_str_to_df): + """A forward from one option paired with a reverse from another (here a + blank-expansion_id default) is incoherent and raises. The check stands in + for the schema's expansion_options_pair_forward_and_reverse rule, which will + reject this input upstream once the validation layer lands.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, BigLine + , reverse, 400, Default + """) + + with pytest.raises( + ValueError, match="Forward and reverse expansion options disagree" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) + + +def test_resolve_expansion_options_raises_on_blank_option_mismatched_pair( + csv_str_to_df, +): + """A blank expansion_option only pairs with another blank: a forward from an + unlabelled option and a reverse from a named one (here a blank-expansion_id + default) is just as incoherent as two differing names, and nunique's default + NaN-dropping would otherwise let it through.""" + options = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, + , reverse, 400, Default + """) + + with pytest.raises( + ValueError, match="Forward and reverse expansion options disagree" + ): + _resolve_expansion_options(options, ["CQ-NQ"]) diff --git a/tests/test_translator/test_translator_helpers.py b/tests/test_translator/test_translator_helpers.py index bba58a68..dc5c75b0 100644 --- a/tests/test_translator/test_translator_helpers.py +++ b/tests/test_translator/test_translator_helpers.py @@ -6,6 +6,7 @@ from ispypsa.translator.helpers import ( _add_investment_periods_as_build_years, _get_financial_year_int_from_string, + _resolve_wildcards, ) @@ -82,3 +83,99 @@ def test_add_investment_periods_as_build_years_empty(csv_str_to_df): assert result.empty assert "build_year" in result.columns assert list(result.columns) == list(expected.columns) + + +def test_resolve_wildcards_expands_a_blank_key_to_every_allowed_value(csv_str_to_df): + """A blank key cell is a wildcard expanded to every allowed value; a column + not in allowed_values (timeslice) rides along unchanged.""" + table = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, peak, 1200 + N1-CNSW, , , + """) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ", "N1-CNSW"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, peak, 1200 + N1-CNSW, forward, , + N1-CNSW, reverse, , + """) + assert_frame_equal( + result.sort_values(["path_id", "direction"]).reset_index(drop=True), + expected.sort_values(["path_id", "direction"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_wildcards_keeps_the_most_specific_row(csv_str_to_df): + """Where an expanded wildcard row and a specific row land on the same key, + the one that used fewer wildcards wins.""" + table = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + , , , 500 + """) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ", "Q1-NQ"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + # CQ-NQ/forward keeps its specific 1200; every other combination takes the + # global default 500. + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, , 1200 + CQ-NQ, reverse, , 500 + Q1-NQ, forward, , 500 + Q1-NQ, reverse, , 500 + """) + assert_frame_equal( + result.sort_values(["path_id", "direction"]).reset_index(drop=True), + expected.sort_values(["path_id", "direction"]).reset_index(drop=True), + check_dtype=False, + ) + + +def test_resolve_wildcards_raises_on_out_of_allowed_values(csv_str_to_df): + """A filled value outside the allowed set is bad input data — callers filter + designed selections out before resolving — so resolution halts the run. The + validation layer will catch this class of input upstream once it lands (the + allowed values mirror the schemas' allowed_values / allowed_values_from); + the raise stays on as the resolver's backstop invariant.""" + table = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 100 + CQ-NQ, 2025, 90 + """) + + with pytest.raises( + ValueError, + match=r"year contains values outside the allowed set: \[2025\]", + ): + _resolve_wildcards( + table, {"expansion_id": ["CQ-NQ"], "year": [2026, 2028]}, ["cost"] + ) + + +def test_resolve_wildcards_empty(csv_str_to_df): + """An empty table resolves to an empty table with the same columns.""" + table = pd.DataFrame(columns=["path_id", "direction", "timeslice", "capacity"]) + + result = _resolve_wildcards( + table, + {"path_id": ["CQ-NQ"], "direction": ["forward", "reverse"]}, + ["capacity"], + ) + + expected = csv_str_to_df(""" + path_id, direction, timeslice, capacity + """) + assert_frame_equal(result, expected, check_dtype=False)