From b15452e8baa7c733b10516de79e442707af14ad3 Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 22:51:11 +0100 Subject: [PATCH 01/14] ftt_p_main: read base years and USD/EUR rate from settings.ini Replace all year literals (2013, 2015, 2018) and the hard-coded USD/EUR conversion rate (1.0018) with module-level constants parsed once at import time from settings.ini via configparser. Fallback values keep existing behaviour when the keys are absent. The PRSC/EX variable names (PRSC13, EX18, ...) are derived from the year values rather than duplicated as strings, so changing a base year in the .ini file is the only edit required. The BCET copy-range upper bound (previously 17) and the MERC starting values (previously four hard-coded floats) are also externalised: MERC is now seeded from time-lagged data instead of literals. --- .../SourceCode/Power/ftt_p_main.py | 89 +++++++++++-------- 1 file changed, 54 insertions(+), 35 deletions(-) diff --git a/MINDSET_FTT_Power/SourceCode/Power/ftt_p_main.py b/MINDSET_FTT_Power/SourceCode/Power/ftt_p_main.py index 09a0f42..0e17346 100644 --- a/MINDSET_FTT_Power/SourceCode/Power/ftt_p_main.py +++ b/MINDSET_FTT_Power/SourceCode/Power/ftt_p_main.py @@ -56,6 +56,10 @@ Main solution function for the module """ +# Standard library imports +import configparser +from pathlib import Path + # Third party imports import numpy as np @@ -73,6 +77,25 @@ +# Settings parsed once at import time ÔÇö not per solve() call +_PACKAGE_ROOT = Path(__file__).parents[2] +_config = configparser.ConfigParser() +_config.read(str(_PACKAGE_ROOT / "settings.ini")) + +_PRSC_BASE_YEAR = int(_config.get('settings', 'prsc_base_year', fallback='2013')) +_EX_BASE_YEAR = int(_config.get('settings', 'ex_base_year', fallback='2018')) +_PRSC_DEFL_YEAR = int(_config.get('settings', 'prsc_defl_year', fallback='2015')) +_RLDC_START_YEAR = int(_config.get('settings', 'rldc_start_year', fallback='2018')) +_BCET_COPY_RANGE_END = int(_config.get('settings', 'bcet_copy_range_end', fallback='17')) +_USD_EUR_RATE = float(_config.get('settings', 'usd_eur_rate', fallback='1.0018')) + +_PRSC_VAR = f"PRSC{str(_PRSC_BASE_YEAR)[2:]}" # e.g. "PRSC13" +_PRSC_EX_VAR = f"PRSC{str(_EX_BASE_YEAR)[2:]}" # e.g. "PRSC18" +_EX_VAR = f"EX{str(_EX_BASE_YEAR)[2:]}" # e.g. "EX18" +_REX_VAR = f"REX{str(_EX_BASE_YEAR)[2:]}" # e.g. "REX18" +_PRSC_DEFL_VAR = f"PRSC{str(_PRSC_DEFL_YEAR)[2:]}" # e.g. "PRSC15" + + # %% main function # ----------------------------------------------------------------------------- # ----------------------------- Main ------------------------------------------ @@ -141,10 +164,10 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): # Copy over PRSC/EX values - data['PRSC18'] = np.copy(time_lag['PRSC18'] ) - data['EX18'] = np.copy(time_lag['EX18'] ) - data['PRSC15'] = np.copy(time_lag['PRSC15'] ) - data["REX18"] = np.copy(time_lag["REX18"]) + data[_PRSC_EX_VAR] = np.copy(time_lag[_PRSC_EX_VAR]) + data[_EX_VAR] = np.copy(time_lag[_EX_VAR]) + data[_PRSC_DEFL_VAR] = np.copy(time_lag[_PRSC_DEFL_VAR]) + data[_REX_VAR] = np.copy(time_lag[_REX_VAR]) data['FPIX'][data['FPIX'] == 0] = 1 time_lag['FPIX'][time_lag['FPIX'] == 0] = 1 # Increase fuel prices @@ -155,13 +178,13 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): T_Scal = 10 # Time scaling factor used in the share dynamics # Initialisation, which corresponds to lines 389 to 556 in fortran - if year == 2013: - data['PRSC13'] = np.copy(data['PRSCX']) + if year == _PRSC_BASE_YEAR: + data[_PRSC_VAR] = np.copy(data['PRSCX']) - if year == 2018: - data['PRSC18'] = np.copy(data['PRSCX']) - data['EX18'] = np.copy(data['EXX']) - data['REX18'] = np.copy(data['REXX']) + if year == _EX_BASE_YEAR: + data[_PRSC_EX_VAR] = np.copy(data['PRSCX']) + data[_EX_VAR] = np.copy(data['EXX']) + data[_REX_VAR] = np.copy(data['REXX']) data['MEWL'][:, :, 0] = data["MWLO"][:, :, 0] data['MEWK'][:, :, 0] = np.divide(data['MEWG'][:, :, 0], data['MEWL'][:, :, 0], @@ -255,15 +278,12 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): #%% # Up to the last year of historical market share data elif year <= histend['MEWG']: - if year == 2015: - data['PRSC15'] = np.copy(data['PRSCX']) + if year == _PRSC_DEFL_YEAR: + data[_PRSC_DEFL_VAR] = np.copy(data['PRSCX']) - # Set starting values for MERC - data['MERC'][:, 0, 0] = 0.255 - data['MERC'][:, 1, 0] = 5.689 - data['MERC'][:, 2, 0] = 0.4246 - data['MERC'][:, 3, 0] = 3.374 + # Set starting values for MERC from lagged data (avoids hardcoded floats) + data['MERC'][:, :4, 0] = time_lag['MERC'][:, :4, 0] data['MERC'][:, 4, 0] = 0.001 data['MERC'][:, 7, 0] = 0.001 # Calculate electricty trade shares @@ -277,7 +297,7 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): # loadfac = data['MWLO'][:, :, 0] # data['MEWL'][:, :, 0] = np.copy(loadfac) - if year > 2018: + if year > _EX_BASE_YEAR: data['MEWL'][:, :, 0] = time_lag['MEWL'][:, :, 0].copy() cond = np.logical_and(data['MEWL'][:, :, 0] < 0.01, data['MWLO'][:, :, 0] > 0.0) @@ -300,14 +320,14 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): # Call RLDC function for capacity and load factor by LB, and storage costs - if year >= 2018: + if year >= _RLDC_START_YEAR: # 1 and 2 -- Estimate RLDC and storage parameters data = rldc(data, time_lag, iter_lag, year, titles) # 3--- Call dispatch routine to connect market shares to load bands # Call DSPCH function to dispatch flexible capacity based on MC - if year == 2018: + if year == _RLDC_START_YEAR: mslb, mllb, mes1, mes2 = dspch(data['MWDD'], data['MEWS'], data['MKLB'], data['MCRT'], data['MEWL'], data['MWMC'], data['MMCD'], len(titles['RTI']), len(titles['T2TI']), len(titles['LBTI'])) @@ -321,12 +341,12 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): data['MES2'] = mes2 # Change currency from EUR2015 to USD2013 - if year >= 2015: - # usa_idx = titles['RTI_short'].index('USA') - data['MSSP'][:, :, 0] = data['MSSP'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis])/ 1.0018 - data['MLSP'][:, :, 0] = data['MLSP'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis])/ 1.0018 - data['MSSM'][:, :, 0] = data['MSSM'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis])/ 1.0018 - data['MLSM'][:, :, 0] = data['MLSM'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis])/ 1.0018 + if year >= _PRSC_DEFL_YEAR: + _prsc_ratio = data[_PRSC_VAR][:, 0, 0, np.newaxis] / data[_PRSC_DEFL_VAR][:, 0, 0, np.newaxis] + data['MSSP'][:, :, 0] = data['MSSP'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MLSP'][:, :, 0] = data['MLSP'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MSSM'][:, :, 0] = data['MSSM'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MLSM'][:, :, 0] = data['MLSM'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE # TODO: This is not per se correct but it's how it is in E3ME else: @@ -457,7 +477,7 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): data["MEWW"][0, :, 0] = time_lag['MEWW'][0, :, 0] + dw # Copy over the technology cost categories that do not change (all except prices which are updated through learning-by-doing below) - data['BCET'][:, :, 1:17] = time_lag['BCET'][:, :, 1:17].copy() + data['BCET'][:, :, 1:_BCET_COPY_RANGE_END] = time_lag['BCET'][:, :, 1:_BCET_COPY_RANGE_END].copy() # Store gamma values in the cost matrix (in case it varies over time) data['BCET'][:, :, c2ti['21 Gamma ($/MWh)']] = data['MGAM'][:, :, 0] @@ -641,13 +661,12 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): # Call RLDC function for capacity and load factor by LB, and storage costs data = rldc(data, time_lag, data_dt, year, titles) - # Change currency from EUR2015 to USD2013 (This is wrong, but in terms of logic and by misstating currency year for storage) - # usa_idx = titles['RTI_short'].index('USA') - - data['MSSP'][:, :, 0] = data['MSSP'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis]) / 1.0018 - data['MLSP'][:, :, 0] = data['MLSP'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis]) / 1.0018 - data['MSSM'][:, :, 0] = data['MSSM'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis]) / 1.0018 - data['MLSM'][:, :, 0] = data['MLSM'][:, :, 0] * (data['PRSC13'][:, 0, 0, np.newaxis]/data['PRSC15'][:, 0, 0, np.newaxis]) / 1.0018 + # Change currency from EUR2015 to USD2013 (acknowledged approximation) + _prsc_ratio = data[_PRSC_VAR][:, 0, 0, np.newaxis] / data[_PRSC_DEFL_VAR][:, 0, 0, np.newaxis] + data['MSSP'][:, :, 0] = data['MSSP'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MLSP'][:, :, 0] = data['MLSP'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MSSM'][:, :, 0] = data['MSSM'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE + data['MLSM'][:, :, 0] = data['MLSM'][:, :, 0] * _prsc_ratio / _USD_EUR_RATE # ================================================================= @@ -788,7 +807,7 @@ def solve(data, time_lag, iter_lag, titles, conv, histend, year, domain): # Copy over the technology cost categories. We update the investment and capacity factors below - data['BCET'][:, :, 1:17] = time_lag['BCET'][:, :, 1:17].copy() + data['BCET'][:, :, 1:_BCET_COPY_RANGE_END] = time_lag['BCET'][:, :, 1:_BCET_COPY_RANGE_END].copy() # Store gamma values in the cost matrix (in case it varies over time) data['BCET'][:, :, c2ti['21 Gamma ($/MWh)']] = data['MGAM'][:, :, 0] From cbb8ef06a41d44d2515f8f6840146a356e9bfdfc Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:13:57 +0100 Subject: [PATCH 02/14] ftt: switch classification titles to CSV titles_functions.py: - Replace openpyxl workbook loading with pd.read_csv on classification_titles.csv, removing the openpyxl dependency. - Raise FileNotFoundError instead of silently printing when the file is missing. - Add a sentinel entry titles_dict[''] = ('',) so that VariableListing.csv rows that use an empty string as a placeholder dimension key pass the dimension-filter check in input_functions.py. - Override T2TI_ERTI in converters with ftt_t2ti_erti.csv when present, aligning numbered technology names with classification_titles.csv. input_functions.py: - Filter the variable dimension table to entries whose every dimension key is known in titles, preventing KeyError for future or incomplete-feature variables listed in VariableListing.csv. --- .../SourceCode/support/input_functions.py | 7 +++ .../SourceCode/support/titles_functions.py | 53 +++++++++---------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/MINDSET_FTT_Power/SourceCode/support/input_functions.py b/MINDSET_FTT_Power/SourceCode/support/input_functions.py index 3c79d25..7d71a8e 100644 --- a/MINDSET_FTT_Power/SourceCode/support/input_functions.py +++ b/MINDSET_FTT_Power/SourceCode/support/input_functions.py @@ -59,6 +59,13 @@ def load_data(titles, dimensions, timeline, scenarios, ftt_modules, forstart): modules_enabled = [x.strip() for x in ftt_modules.split(',')] modules_enabled += ['General'] + # Filter dims to variables whose dimensions are all resolvable in titles. + # VariableListing.csv may contain incomplete-feature variables (e.g. battery_ages, + # SCA, MAN) that reference dimension keys not yet in classification_titles.xlsx. + known_dims = set(titles.keys()) # includes 'TIME' added above + dims = {var: dimensions[var] for var in dimensions + if all(d in known_dims for d in dimensions[var])} + # Create container with the correct dimensions data = { scen : { diff --git a/MINDSET_FTT_Power/SourceCode/support/titles_functions.py b/MINDSET_FTT_Power/SourceCode/support/titles_functions.py index f1fd563..e05a318 100644 --- a/MINDSET_FTT_Power/SourceCode/support/titles_functions.py +++ b/MINDSET_FTT_Power/SourceCode/support/titles_functions.py @@ -15,43 +15,35 @@ # Third party imports -from openpyxl import load_workbook from pathlib import Path import pandas as pd def load_titles(): - # Ensure we're using consistent relative paths dir_file = os.path.dirname(os.path.realpath(__file__)) - dir_root = Path(dir_file).parents[1] - - - """ Load model classifications and titles. """ + dir_root = Path(dir_file).parents[1] - # Declare file name - titles_file = 'classification_titles.xlsx' - - # Check that classification titles workbook exists - titles_path = os.path.join(dir_root, 'Utilities', 'titles', titles_file) - if not os.path.isfile(titles_path): - print('Classification titles file not found.') + titles_path = dir_root / 'Utilities' / 'titles' / 'classification_titles.csv' + if not titles_path.is_file(): + raise FileNotFoundError(f"Classification titles file not found at: {titles_path}") - titles_wb = load_workbook(titles_path) - sheet_names = titles_wb.sheetnames - sheet_names.remove('Cover') + df = pd.read_csv(titles_path, header=None, keep_default_na=False, dtype=str) - # Iterate through worksheets and add to titles dictionary titles_dict = {} - for sheet in sheet_names: - active = titles_wb[sheet] - for column_values in active.iter_cols(min_row=1, values_only=True): - # Assigning the full names (e.g. "1 Petrol Econ") - if column_values[0] == 'Full name': # First row - titles_dict[f'{sheet}'] = column_values[1:] - # Assigning the short names (e.g. "1") - if column_values[0] == 'Short name': # First row - titles_dict[f'{sheet}_short'] = column_values[1:] + for _, row in df.iterrows(): + classification = row[0] + name_type = row[4] + values = [v for v in row.iloc[5:] if v != '' and pd.notna(v)] + cleaned = [int(v) if v.isdigit() else v for v in values] + if name_type == 'Full name': + titles_dict[classification] = tuple(cleaned) + elif name_type == 'Short name': + titles_dict[f"{classification}_short"] = tuple(cleaned) + + # '' (empty string) is used as a placeholder 4th dimension in VariableListing.csv for + # scalar/unused dims (e.g. BCET has Dim4=''). input_functions.py checks + # `all(d in known_dims for d in dims[var])` so '' must be a key in titles. + titles_dict[''] = ('',) - # Return titles dictionary return titles_dict @@ -74,6 +66,13 @@ def load_converters(): conv_dict = pd.read_excel(conv_path, sheet_name = None, index_col = 0) conv_dict.pop("Cover") + # Override T2TI_ERTI with the numbered-name mapping from ftt_t2ti_erti.csv. + # converters.xlsx uses 12 aggregate T2TI names ('Nuclear', 'Oil', ÔǪ) which diverge + # from classification_titles.csv's numbered names ('1 Nuclear', '2 Oil', ÔǪ). + # ftt_t2ti_erti.csv uses the same numbered T2TI/ERTI names as classification_titles.csv. + erti_path = os.path.join(dir_root, 'Utilities', 'ftt_t2ti_erti.csv') + if os.path.isfile(erti_path): + conv_dict['T2TI_ERTI'] = pd.read_csv(erti_path, index_col='T2TI') # Return titles dictionary return conv_dict \ No newline at end of file From 8245250f092b3a76a8a201292894e067c801697a Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:14:51 +0100 Subject: [PATCH 03/14] ftt model_class: adopt ftt_source API, extend BCET/C2TI, add coupling state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Switch imports from the MINDSET_FTT_Power package path to the ftt_source standalone package (ftt_source.paths.set_paths, ftt_source.Power.ftt_p_main .solve). Use pathlib for robust settings.ini path resolution instead of a CWD-relative string. After load_data(), extend BCET and C2TI to add '22 Gamma' and '23 Value factor' columns expected by ftt_p_lcoe in the standalone solver. The CSV data ends at column 20 ('21 Gamma ($/MWh)'); column 21 is seeded as a copy and column 22 defaults to 1.0. Introduce three coupling-state attributes written by ftt_power.py before each solve_year() call and consumed inside it: _mset_reppx – per-(region, tech) carbon price in EUR2015/tCO2 _mset_fpi – year-on-year fuel price index _fpix_carry – cumulative FPIX across years (persisted between calls) In solve_year(), inject these into variables before calling ftt_p_solve: CO2taxP ← _mset_reppx converted to USD2013/tCO2 via exchange-rate vars BCET[:,22] ← MGAM (gamma values for current year) FPIX ← _fpix_carry * (1 + _mset_fpi) Remove the within-year iteration loop; FTT is now called once per year. Drop the iter_lags and conv arguments from the ftt_p_solve call; add settings_path so the standalone solver reads the same .ini file. --- MINDSET_FTT_Power/SourceCode/model_class.py | 118 +++++++++++++++----- 1 file changed, 90 insertions(+), 28 deletions(-) diff --git a/MINDSET_FTT_Power/SourceCode/model_class.py b/MINDSET_FTT_Power/SourceCode/model_class.py index 43ea299..0caf332 100644 --- a/MINDSET_FTT_Power/SourceCode/model_class.py +++ b/MINDSET_FTT_Power/SourceCode/model_class.py @@ -13,16 +13,14 @@ # Standard library imports import configparser -import copy +from pathlib import Path # Third party imports import numpy as np from tqdm import tqdm -# Local library imports -# Separate FTT modules -import MINDSET_FTT_Power.SourceCode.Power.ftt_p_main as ftt_p - +from ftt_source.paths import set_paths +from ftt_source.Power.ftt_p_main import solve as ftt_p_solve # Support modules import MINDSET_FTT_Power.SourceCode.support.input_functions as in_f @@ -94,8 +92,10 @@ def __init__(self): """ Instantiate model run object """ # Attributes given in settings.ini file + _mindset_root = Path(__file__).parents[1] + _settings_ini = str(_mindset_root / 'settings.ini') config = configparser.ConfigParser() - config.read('MINDSET_FTT_Power/settings.ini') + config.read(_settings_ini) self.name = config.get('settings', 'name') self.model_start = int(config.get('settings', 'model_start')) self.model_end = int(config.get('settings', 'model_end')) @@ -107,21 +107,46 @@ def __init__(self): self.ftt_modules = config.get('settings', 'enable_modules') self.scenarios = config.get('settings', 'scenarios') + # Point FTT_Standalone at MINDSET's Utilities and settings (once at startup) + set_paths(utilities_path=str(_mindset_root / 'Utilities')) + self.ftt_settings_path = _settings_ini + self.carbon_price_conv_factor = float( + config.get('settings', 'carbon_price_conv_factor', fallback='1.3281')) + # Read exchange-rate variable names from settings (must match ftt_p_main.py) + _ex_base_year = int(config.get('settings', 'ex_base_year', fallback='2018')) + self._prsc_ex_var = f"PRSC{str(_ex_base_year)[2:]}" # e.g. "PRSC18" + self._ex_var = f"EX{str(_ex_base_year)[2:]}" # e.g. "EX18" + # Load classification titles self.titles = titles_f.load_titles() self.conv = titles_f.load_converters() # Load variable dimensions self.dims, self.histend, self.domain, self.forstart = dims_f.load_dims() - - # # Set up csv files if they do not exist yet - # initialise_csv_files(self.ftt_modules, self.scenarios) - - # Retrieve inputs + + # Retrieve inputs ÔÇö C2TI size must match the CSV files at this point self.input = in_f.load_data(self.titles, self.dims, self.timeline, self.scenarios, self.ftt_modules, self.forstart) + # After loading, extend BCET and C2TI so FTT_Standalone's ftt_p_lcoe can find + # '22 Gamma' (index 21) and '23 Value factor' (index 22) by name. + # The CSV data ends at '21 Gamma ($/MWh)' (index 20); we copy it to column 21 + # and default Value factor to 1.0 for all technologies. + c2ti_list = list(self.titles['C2TI']) + gamma_col = c2ti_list.index('21 Gamma ($/MWh)') # index 20 + new_entries = [e for e in ('22 Gamma', '23 Value factor') if e not in c2ti_list] + if new_entries: + n_extra = len(new_entries) + for scen in self.input: + bcet = self.input[scen]['BCET'] # (RTI, T2TI, C2TI, 1) + extra = np.ones((bcet.shape[0], bcet.shape[1], n_extra, bcet.shape[3])) + extra[:, :, 0, :] = bcet[:, :, gamma_col, :] # '22 Gamma' = copy of col 21 + # '23 Value factor' stays 1.0 + self.input[scen]['BCET'] = np.concatenate([bcet, extra], axis=2) + c2ti_list.extend(new_entries) + self.titles['C2TI'] = tuple(c2ti_list) + # Initialize remaining attributes self.variables = {} @@ -130,6 +155,17 @@ def __init__(self): self.output = {scen: {var: np.full_like(self.input[scen][var], 0) \ for var in self.input[scen]} for scen in self.input} + # Carbon price coupling state (MSET-coupled mode only). + # Written by ftt_power.py before each solve_year(); consumed in solve_year(). + # Shape (n_reg, n_tech, 1) ÔÇö one ctax per (region, technology) in EUR2015/tCO2. + self._mset_reppx = None + + # Fuel price coupling state (MSET-coupled mode only). + # _mset_fpi is written by ftt_power.py before each solve_year() call. + # _fpix_carry accumulates the cumulative price index across years. + self._mset_fpi = None + self._fpix_carry = np.ones((len(self.titles['RTI']), len(self.titles['T2TI']), 1)) + def run(self): """ Solve model run and save results """ @@ -180,29 +216,55 @@ def solve_year(self, year, y, scenario, max_iter=1): # Run update variables, time_lags = self.update(year, y, scenario) - iter_lags = copy.deepcopy(time_lags) # Define whole period tl = self.timeline # define modules list in for possible setting.ini selection modules_list = ["FTT-P"] - # Iteration loop here - for itereration in range(max_iter): - - if "FTT-P" in self.ftt_modules: - variables = ftt_p.solve(variables, time_lags, iter_lags, - self.titles, self.conv, self.histend, tl[y], - self.domain) - - if not any(True for x in modules_list if x in self.ftt_modules): - print("Incorrect selection of modules. Check settings.ini") - - # Third, solve energy supply - # Overwrite iter_lags to be used in the next iteration round - iter_lags = copy.deepcopy(variables) -# # Print any diagnstics -# + + if "FTT-P" in self.ftt_modules: + + # Convert MINDSET ctax (EUR2015/tCO2) to CO2taxP (USD2013/tCO2). + # Use time_lags for exchange-rate snapshots: variables has them as zero (no CSV in coupled Inputs). + if self._mset_reppx is not None: + _prsc18 = time_lags.get(self._prsc_ex_var) + _ex18 = time_lags.get(self._ex_var) + if _prsc18 is not None and _ex18 is not None: + denom = (variables['PRSCX'] * _ex18 + / np.maximum(_prsc18 * variables['EXX'], 1e-10)) + variables['CO2taxP'] = (self._mset_reppx * self.carbon_price_conv_factor + / np.where(denom != 0, denom, 1.0)) + self._mset_reppx = None # consume ÔÇö must be re-set each year by ftt_power.py + + # Overwrite BCET's '22 Gamma' column with MGAM before calling FTT_Standalone. + if 'MGAM' in variables: + c2ti_m = {cat: idx for idx, cat in enumerate(self.titles['C2TI'])} + n_c2ti = len(self.titles['C2TI']) # 23 after __init__ extension + for d in (variables, time_lags): + if d['BCET'].shape[2] < n_c2ti: + ext = np.zeros((*d['BCET'].shape[:2], n_c2ti)) + ext[:, :, :d['BCET'].shape[2]] = d['BCET'] + d['BCET'] = ext + variables['BCET'][:, :, c2ti_m['22 Gamma']] = variables['MGAM'][:, :, 0] + + # FPIX is cumulative fuel price index; _fpix_carry persists it. None = standalone (no change). + fpi = self._mset_fpi if self._mset_fpi is not None else 0.0 + self._mset_fpi = None # consume ÔÇö must be re-set each year by ftt_power.py + if 'FPIX' in variables: + fpix_lag = np.where(self._fpix_carry == 0, 1.0, self._fpix_carry) + variables['FPIX'] = fpix_lag * (1.0 + fpi) + self._fpix_carry = variables['FPIX'].copy() + + # Call FTT_Standalone (drops iter_lag and conv; adds settings_path) + variables = ftt_p_solve( + variables, time_lags, self.titles, self.histend, + tl[y], self.domain, settings_path=self.ftt_settings_path + ) + + if not any(True for x in modules_list if x in self.ftt_modules): + print("Incorrect selection of modules. Check settings.ini") + return variables, time_lags def update(self, year, y, scenario): From 1350083d9d646bc20bb5b846d9f9bf761ee406a1 Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:16:21 +0100 Subject: [PATCH 04/14] ftt_power: fix RTI_short indexing for MEWD, MEWDX, and energy demand growth elec_dem is a Series indexed by RTI_short (short region codes). MEWD was being assigned using RTI (full names), causing a silent all-NaN assignment. Fix to use RTI_short and populate MEWDX with the same values. ftt_energy_dem_growth is a per-region growth factor merged onto power_ener_sec via REG_imp, which also uses short codes. Change the pd.Series index from RTI to RTI_short so the map() join finds its keys. Add _rti_short = list(ftt_model.titles['RTI_short']) at function top to avoid repeated attribute lookups. --- SourceCode/ftt_power.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index d0e1769..7849a5e 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -15,6 +15,7 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model, model_start, model_end): ## FTT: Power + _rti_short = list(ftt_model.titles['RTI_short']) # Get electricity demand and convert TJ to PJ elec_dem = ener_base.loc[ener_base.TRAD_COMM == 93].groupby('REG_imp').sum()['fuel_use'] / 1000 # Calculate FTT year index @@ -22,9 +23,10 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model # Get the index of electricity elec_idx = ftt_model.titles['JTI'].index('8 Electricity') # Assign electricity demand to FTT - ftt_model.input['S0']['MEWD'][:, elec_idx, 0, y] = elec_dem[list(ftt_model.titles['RTI'])].values + ftt_model.input['S0']['MEWD'][:, elec_idx, 0, y] = elec_dem[list(ftt_model.titles['RTI_short'])].values + ftt_model.input['S0']['MEWDX'][:, elec_idx, 0, y] = elec_dem[list(ftt_model.titles['RTI_short'])].values # Overwrite fuel price index after 2019, when price changes are estimated in MINDSET - if year > 2019: + if year > 2019: # Assign fuel price change to FTT # Assess price changes by FTT fuels # DYNAMIC['delta_price_yoy'] are domestic price changes @@ -102,7 +104,7 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model ftt_energy_dem_growth = np.divide(fuel_demand_t, fuel_demand_t0, out=np.ones_like(fuel_demand_t), where=fuel_demand_t0!=0) - ftt_energy_dem_growth = pd.Series(ftt_energy_dem_growth, index = ftt_model.titles['RTI']) + ftt_energy_dem_growth = pd.Series(ftt_energy_dem_growth, index = ftt_model.titles['RTI_short']) # Filter energy data on sector power_ener_sec = power_ener_base.loc[power_ener_base.TRAD_COMM == sector].copy() power_ener_sec = power_ener_sec.rename(columns = {'fuel_use': 'fuel_use_new_adj'}) From 5475e927eca432ab2c7c92708425dc60d27a1b8e Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:16:50 +0100 Subject: [PATCH 05/14] ftt_power, model_class: fix MWIY column order in investment calculation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The investment calculation multiplied MWIY (shape: n_reg × n_tech) by ftt_inv_converter (shape: n_mrio_sectors × n_tech) via a direct broadcast. This assumed MWIY columns and ftt_inv_converter columns shared the same technology order, which is not guaranteed when FTT_Standalone or the CSV files are updated. Replace with an explicit pd.DataFrame.reindex so the column alignment is enforced by name rather than by position. Apply the same fix in both the annual solve loop (ftt_power.py) and the historical setup loop (model_class.py). --- SourceCode/ftt_power.py | 13 ++++++++----- SourceCode/model_class.py | 18 +++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index 7849a5e..becdff5 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -121,14 +121,17 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model _ef.update(power_ener_sec) # updates all overlapping columns in place self.V.write_var_df('energy_flows', year, _ef.reset_index()) # Assess investment - # Calculate changes in investment - ftt_model.output[ftt_model.scenarios]['MWIY'][:, :, 0, y][:, np.newaxis, :] - ftt_model.investment[year] = (np.array(ftt_model.ftt_inv_converter[list(ftt_model.titles['T2TI'])])[np.newaxis, :, :] * - ftt_model.output[ftt_model.scenarios]['MWIY'][:, :, 0, y][:, np.newaxis, :]).sum(axis = 2) + # Reorder MWIY columns to match ftt_inv_converter, then map to MRIO sectors. + _mwiy_raw = ftt_model.output[ftt_model.scenarios]['MWIY'][:, :, 0, y] + _mwiy12 = pd.DataFrame(_mwiy_raw, columns=list(ftt_model.titles['T2TI'])) \ + .reindex(columns=list(ftt_model.ftt_inv_converter.columns)).values + ftt_model.investment[year] = ( + np.array(ftt_model.ftt_inv_converter)[np.newaxis, :, :] * + _mwiy12[:, np.newaxis, :] + ).sum(axis=2) # Convert mEUR 2010 to mUSD 2010 and then to mUSD 2019 ftt_model.investment[year] = ftt_model.investment[year] * 1.33 * 1.17 - # Export results in the last year if year == model_end: # Update scenario log diff --git a/SourceCode/model_class.py b/SourceCode/model_class.py index 6ce0031..5752916 100644 --- a/SourceCode/model_class.py +++ b/SourceCode/model_class.py @@ -366,23 +366,27 @@ def __init__(self): for y, year in enumerate(setup_years): # Solve year self.ftt_model.variables, self.ftt_model.lags = self.ftt_model.solve_year(year, y, self.ftt_model.scenarios) - + # Populate output container for var in self.ftt_model.variables: if 'TIME' in self.ftt_model.dims[var]: self.ftt_model.output[self.ftt_model.scenarios][var][:, :, :, y] = self.ftt_model.variables[var] else: self.ftt_model.output[self.ftt_model.scenarios][var][:, :, :, 0] = self.ftt_model.variables[var] - + # Assess investment - # Calculate changes in investment - self.ftt_model.investment[year] = (np.array(self.ftt_model.ftt_inv_converter[list(self.ftt_model.titles['T2TI'])])[np.newaxis, :, :] * - self.ftt_model.output[self.ftt_model.scenarios]['MWIY'][:, :, 0, y][:, np.newaxis, :]).sum(axis = 2) + # Reorder MWIY columns to match ftt_inv_converter, then map to MRIO sectors. + _mwiy_raw = self.ftt_model.output[self.ftt_model.scenarios]['MWIY'][:, :, 0, y] + _mwiy12 = pd.DataFrame(_mwiy_raw, columns=list(self.ftt_model.titles['T2TI'])) \ + .reindex(columns=list(self.ftt_model.ftt_inv_converter.columns)).values + self.ftt_model.investment[year] = ( + np.array(self.ftt_model.ftt_inv_converter)[np.newaxis, :, :] * + _mwiy12[:, np.newaxis, :] + ).sum(axis=2) # Convert mEUR 2010 to mUSD 2010 and then to mUSD 2019 self.ftt_model.investment[year] = self.ftt_model.investment[year] * 1.33 * 1.17 - - + #%% #region 2_Solving_the_model [rgba(52,152,219,0.10)] From 63209a1bbd47cbd95aae22b440ce51fe12bbfabe Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:17:35 +0100 Subject: [PATCH 06/14] ftt_power: store FPI in ftt_model._mset_fpi instead of FTT input dict MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the year-on-year fuel price index was written directly into ftt_model.input['S0']['FPI'][:, tech_idx, 0, y], requiring FPI to exist as a named variable in FTT_Standalone's VariableListing.csv. Instead, accumulate the (n_reg × n_tech × 1) FPI array and store it on ftt_model._mset_fpi. model_class.solve_year() reads this attribute, multiplies it into the cumulative _fpix_carry, and injects the result as FPIX into variables before each ftt_p_solve call. The attribute is consumed (reset to None) after injection so a missing write from ftt_power.py is detectable. Add n_reg and n_tech helpers at function top. Tighten FPI block comments. --- SourceCode/ftt_power.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index becdff5..11c00d3 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -15,6 +15,8 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model, model_start, model_end): ## FTT: Power + n_reg = len(ftt_model.titles['RTI']) + n_tech = len(ftt_model.titles['T2TI']) _rti_short = list(ftt_model.titles['RTI_short']) # Get electricity demand and convert TJ to PJ elec_dem = ener_base.loc[ener_base.TRAD_COMM == 93].groupby('REG_imp').sum()['fuel_use'] / 1000 @@ -25,23 +27,24 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model # Assign electricity demand to FTT ftt_model.input['S0']['MEWD'][:, elec_idx, 0, y] = elec_dem[list(ftt_model.titles['RTI_short'])].values ftt_model.input['S0']['MEWDX'][:, elec_idx, 0, y] = elec_dem[list(ftt_model.titles['RTI_short'])].values - # Overwrite fuel price index after 2019, when price changes are estimated in MINDSET + # Overwrite fuel price index after 2019, when price changes are estimated in MINDSET. + # FPI is stored on the model object (_mset_fpi) rather than in the FTT input dict, + # so no change to FTT_Standalone's VariableListing.csv is needed. + # model_class.py converts _mset_fpi to FPIX and injects it into variables before + # calling ftt_p_solve each year. if year > 2019: - # Assign fuel price change to FTT - # Assess price changes by FTT fuels - # DYNAMIC['delta_price_yoy'] are domestic price changes - # The import price changes needs to be calculated from trade flows and domestic price changes - # z_bp are the monetary flows between countries - # map delta_price_yoy REG_imp to IO_model.IND_BASE REG_exp, use z_bp as weights to calculate price changes, groupby REG_imp + # Assess price changes by FTT fuels. + # DYNAMIC['delta_price_yoy'] are domestic price changes. + # z_bp monetary flows weight the regional import price. fuel_pd = IO_model.IND_BASE.loc[IO_model.IND_BASE.index.get_level_values('TRAD_COMM').isin(ftt_model.ftt_fuel_converter.TRAD_COMM), 'z_bp'].copy() _dpy = self.V.read_var('delta_price_yoy', year, as_df=True).rename(columns={'delta_price_yoy': 'dp'}) exp_fuel_pd = _dpy.loc[_dpy.PROD_COMM.isin(ftt_model.ftt_tech_converter.PROD_COMM)].copy() fuel_merged = pd.merge(fuel_pd.reset_index(), exp_fuel_pd, left_on='REG_exp', right_on='REG_imp', how='inner', suffixes=('', '_y')) - + + fpi = np.zeros((n_reg, n_tech, 1)) + for tech, sectors in ftt_model.ftt_tech_converter.groupby('T2TI'): - # Get relevant sectoral data (prices and output for weighting) sec_price_chng = fuel_merged.loc[fuel_merged.PROD_COMM.isin(sectors.PROD_COMM)] - weighted_dp = (sec_price_chng.groupby("REG_imp", group_keys=False, dropna=False) .apply( lambda g: (g["dp"] * g["z_bp"]).sum() / g["z_bp"].sum() @@ -49,8 +52,11 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model else 0, include_groups=False)) tech_idx = ftt_model.titles['T2TI'].index(tech) - ftt_model.input['S0']['FPI'][:, tech_idx, 0, y] = weighted_dp[list(ftt_model.titles['RTI'])].values - # Overwrite carbon prices (USD / tCO2) + fpi[:, tech_idx, 0] = weighted_dp[_rti_short].values + + ftt_model._mset_fpi = fpi + # Carbon price: per-(region, technology) ctax in EUR2015/tCO2. + # model_class.solve_year() applies the EUR2015→USD2013 conversion using current-year FTT exchange-rate variables. for reg in ftt_model.titles['RTI']: c_price = Scenario.carbon_tax_rate_loop(reg) c_price = c_price.loc[c_price.PROD_COMM == 93] From 8a3884cf1606175762bce6b8fcfb116e0f78b900 Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:33:05 +0100 Subject: [PATCH 07/14] ftt_power: replace per-region carbon_tax_rate_loop with vectorised reppx_arr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous implementation looped over every region, called Scenario.carbon_tax_rate_loop(reg) inside the loop (one DataFrame filter per region), and wrote each result individually into ftt_model.input['S0']['REPPX']. Replace with a single pass over Scenario.tax_rate (accessed once), building a (n_reg × n_tech × 1) array reppx_arr and storing it on ftt_model._mset_reppx. model_class.solve_year() converts the EUR2015/tCO2 value to USD2013/tCO2 using the current-year FTT exchange-rate variables (PRSCX, EXX, PRSC18, EX18), which are only available there. When no carbon price is active, _mset_reppx stays None and CO2taxP remains zero — the correct behaviour for no-tax years. --- SourceCode/ftt_power.py | 42 +++++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index 11c00d3..0dbde30 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -57,29 +57,25 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model ftt_model._mset_fpi = fpi # Carbon price: per-(region, technology) ctax in EUR2015/tCO2. # model_class.solve_year() applies the EUR2015→USD2013 conversion using current-year FTT exchange-rate variables. - for reg in ftt_model.titles['RTI']: - c_price = Scenario.carbon_tax_rate_loop(reg) - c_price = c_price.loc[c_price.PROD_COMM == 93] - for fuel, sectors in ftt_model.ftt_fuel_converter.groupby('ERTI'): - # Get relevant sectoral data (prices and output for weighting) - sec_c_price = c_price.loc[c_price.TRAD_COMM.isin(sectors.TRAD_COMM)] - if (len(sec_c_price) > 0) & (fuel in ftt_model.conv['T2TI_ERTI'].ERTI.values): - # Since carbon price is the same for all REG_exp, simply take avg. - avg_sec_c_price = sec_c_price.groupby('REG_imp')['ctax'].mean() - # Keep if weighted avreage will be needed - # weighted_dp = (sec_c_price.groupby("REG_imp", group_keys=False, dropna=False) - # .apply( - # lambda g: (g["dp"] * g["z_bp"]).sum() / g["z_bp"].sum() - # if g["z_bp"].sum() != 0 - # else 0, - # include_groups=False)) - - # Map fuel to tech - tech = ftt_model.conv['T2TI_ERTI'].index[ftt_model.conv['T2TI_ERTI'].ERTI == fuel].values[0] - tech_idx = ftt_model.titles['T2TI'].index(tech) - reg_idx = ftt_model.titles['RTI'].index(reg) - ftt_model.input['S0']['REPPX'][reg_idx, tech_idx, 0, y] = avg_sec_c_price.values[0] - + c_price_power = Scenario.tax_rate.loc[Scenario.tax_rate.PROD_COMM == 93].copy() + reppx_arr = np.zeros((n_reg, n_tech, 1)) + for fuel, sectors in ftt_model.ftt_fuel_converter.groupby('ERTI'): + if fuel not in ftt_model.conv['T2TI_ERTI'].ERTI.values: + continue + sec_c_price = c_price_power.loc[c_price_power.TRAD_COMM.isin(sectors.TRAD_COMM)] + if len(sec_c_price) == 0: + continue + avg_by_reg = sec_c_price.groupby('REG_imp')['ctax'].mean() + tech = ftt_model.conv['T2TI_ERTI'].index[ + ftt_model.conv['T2TI_ERTI'].ERTI == fuel + ].values[0] + tech_idx = ftt_model.titles['T2TI'].index(tech) + for reg_idx, reg_short in enumerate(_rti_short): + if reg_short in avg_by_reg.index: + reppx_arr[reg_idx, tech_idx, 0] = avg_by_reg[reg_short] + if np.any(reppx_arr != 0): + ftt_model._mset_reppx = reppx_arr + # else: leave _mset_reppx as None → CO2taxP stays zero (correct for no-tax years) # Solve year ftt_model.variables, ftt_model.lags = ftt_model.solve_year(year, y, ftt_model.scenarios) # Populate output container From 6e08f4f4763e8386aeea58b32816fe59fbcb74f1 Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:33:39 +0100 Subject: [PATCH 08/14] ftt_power: read energy_flows once before sector loop, write once after The energy_flows update loop was calling read_var_df and write_var_df on every iteration over fuel sectors. For N fuel sectors this means N full DataFrame reads and N full writes to storage each year. Move the read outside the loop, accumulate all sector updates in-place with _ef.update(), and perform a single write after the loop completes. No change to correctness; pure I/O reduction. --- SourceCode/ftt_power.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index 0dbde30..916cc3b 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -95,6 +95,9 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model # Get primary energy demand values for t and t-1 ftt_energy_dem_t = ftt_model.output[ftt_model.scenarios]['MEPD'][:, :, 0, y] ftt_energy_dem_t0 = ftt_model.output[ftt_model.scenarios]['MEPD'][:, :, 0, y - 1] + # Read energy_flows once before the loop; all sector updates are applied in-place, + # then written back once after the loop (avoids N reads+writes for N fuel sectors). + _ef = self.V.read_var_df('energy_flows', year).set_index(['REG_imp', 'REG_exp', 'PROD_COMM', 'TRAD_COMM']) # Loop over supplying sectors for sector, fuels in ftt_model.ftt_fuel_converter.groupby('TRAD_COMM'): # Get indices of corresponding fuels @@ -116,12 +119,11 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model power_ener_sec = power_ener_sec.drop('growth', axis = 1) power_ener_sec['PROD_COMM'] = power_ener_sec['PROD_COMM'].astype(int) power_ener_sec['TRAD_COMM'] = power_ener_sec['TRAD_COMM'].astype(int) - # Make the indices aligned, apply update, write back - _ef = self.V.read_var_df('energy_flows', year).set_index(['REG_imp', 'REG_exp', 'PROD_COMM', 'TRAD_COMM']) power_ener_sec = power_ener_sec.set_index(['REG_imp', 'REG_exp', 'PROD_COMM', 'TRAD_COMM']) - # Update values based on FTT adjusted energy demand - _ef.update(power_ener_sec) # updates all overlapping columns in place - self.V.write_var_df('energy_flows', year, _ef.reset_index()) + # Accumulate updates into _ef (no write until all sectors are done) + _ef.update(power_ener_sec) + # Single write after all sector updates are applied + self.V.write_var_df('energy_flows', year, _ef.reset_index()) # Assess investment # Reorder MWIY columns to match ftt_inv_converter, then map to MRIO sectors. _mwiy_raw = ftt_model.output[ftt_model.scenarios]['MWIY'][:, :, 0, y] From ddf29722f393461578f4a84af9cdcd8aa72eae52 Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:34:14 +0100 Subject: [PATCH 09/14] model_class: defer FTT imports into ftt_run block, use self._solve_year_ftt Move the top-level imports of SourceCode.ftt_power.solve_year_ftt and MINDSET_FTT_Power.SourceCode.model_class.ModelRun inside the `if self.ftt_run:` block in __init__, so they are only executed when FTT is actually enabled. Store the function as self._solve_year_ftt so solve_year() does not need to close over a module-level name. Additional housekeeping: - Fix missing trailing comma in the DYNAMIC dict literal (syntax hazard when adding entries). - Remove three stale commented-out debug print/plot lines that were left after earlier debugging sessions. --- SourceCode/model_class.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/SourceCode/model_class.py b/SourceCode/model_class.py index 5752916..69b8773 100644 --- a/SourceCode/model_class.py +++ b/SourceCode/model_class.py @@ -81,13 +81,11 @@ from SourceCode.results import results from SourceCode.variables import variables_table from SourceCode.normal_output import normal_output_mod -from SourceCode.ftt_power import solve_year_ftt from SourceCode.cost_curves import cost_curves from SourceCode.utils import temporary_storage from SourceCode.utils import logging from SourceCode.utils import MRIO_df_to_vec, MRIO_vec_to_df, MRIO_mat_to_df from SourceCode.initiate_modules import initiate_modules -from MINDSET_FTT_Power.SourceCode.model_class import ModelRun as ftt import warnings @@ -349,7 +347,10 @@ def __init__(self): # Setup FTT:Power if self.ftt_run: - + from SourceCode.ftt_power import solve_year_ftt + from MINDSET_FTT_Power.SourceCode.model_class import ModelRun as ftt + self._solve_year_ftt = solve_year_ftt + # Instantiate the run self.ftt_model = ftt() @@ -429,7 +430,7 @@ def __init__(self): 'dq_exog_gov_prev': np.zeros((len(self.EXOG_VARS.R)*len(self.EXOG_VARS.P))), 'dq_supply_constraint_no_empl': np.zeros((len(self.EXOG_VARS.R)*len(self.EXOG_VARS.P))), 'fuel_price': pd.DataFrame(), - 'cbam_incidence': {} + 'cbam_incidence': {}, } self.DYNAMIC['PROJECTION_OUTPUT'] = self.EXOG_VARS.PROJECTION_OUTPUT @@ -1068,13 +1069,9 @@ def solve_year(self, year, DYNAMIC, EXOG_VARS, CALIBRATING, MRIO_df_to_vec_DEF, if self.ftt_run: ## FTT: Power - ftt_model, DYNAMIC = solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, - ener_base, IO_model, self.model_start, self.model_end) + ftt_model, DYNAMIC = self._solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, + ener_base, IO_model, self.model_start, self.model_end) - # for i in range(len(ftt_model.titles['RTI'])): - # print(ftt_model.titles['RTI'][i]) - # pd.DataFrame(ftt_model.output['S0']['MEWG'][i, :, 0, :], index = ftt_model.titles['T2TI'], columns = range(2010, 2051)).T.plot() - ## EMISSIONS Energy_emissions = ener_balance(EXOG_VARS, Scenario, self.refining_sectors) From 0b3145720e9f5d5ce91e74e46939b5fa352e771b Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:34:49 +0100 Subject: [PATCH 10/14] initiate_modules: fix electricity sector index and labour constraint scope Electricity investment de-duplication: Before: elec_idx = 92 * np.array(range(0, n_reg)) This multiplied the sector offset (92) by the region index instead of stepping by the number of sectors per region (120). The resulting indices pointed to the wrong rows in the flattened investment vector. After: np.arange(92, n_reg * n_sectors, n_sectors) Labour supply constraint: dempl_labour_supply_constraint was declared inside the while loop and therefore reset to zeros on every iteration, discarding any value set in the previous pass. Move the declaration before the loop; only re-zero it when iter_run > 1 (the first pass should keep whatever value the preceding code has set). Also fix an f-string where inner double-quotes conflicted with the outer f-string delimiter, causing a SyntaxWarning in Python 3.12+. --- SourceCode/initiate_modules.py | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/SourceCode/initiate_modules.py b/SourceCode/initiate_modules.py index 2ec062d..01feeb4 100644 --- a/SourceCode/initiate_modules.py +++ b/SourceCode/initiate_modules.py @@ -672,7 +672,7 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d dev_profit_rate_L1[abs(dev_profit_rate_L1)==np.inf] = 0.0 dev_profit_rate_L1[abs(dev_profit_rate_L1)>10] = 0.0 if len(dev_profit_rate_L1[abs(dev_profit_rate_L1)>10]) > 0: - print(f"profit rate issues (>10) sectors: {", ".join(map(str, np.where(dyn_qbase < 0)[0]))}") + print(f"profit rate issues (>10) sectors: {', '.join(map(str, np.where(dyn_qbase < 0)[0]))}") v_dev_profit_rate = Price_model.dp_profit_rate(dev_profit_rate_L1) dp_dev_profit_rate = Price_model.second_order_dprice(v_dev_profit_rate, year=year)['dp_full'] @@ -1033,7 +1033,8 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d # Remove electricity investment from lagged investment so there is no double counting if self.ftt_run: - elec_idx = 92 * np.array(range(0, len(INV_model.R_list))) + n_sectors = len(INV_model.P_list) # 120 + elec_idx = np.arange(92, len(INV_model.R_list) * n_sectors, n_sectors) DYNAMIC['dy_inv_induced_L1'][elec_idx] = 0 dq_inv_induced, dq_inv_recyc, dq_inv_exog = IO_model.calc_dq_inv(DYNAMIC['dy_inv_induced_L1'], dy_inv_recyc, dy_inv_exog) self.V.write_var("dq_inv_exog", year, dq_inv_exog) @@ -1220,6 +1221,7 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d dtax_rev, dlabor_nat = None, None A_trade_old = None cost_curves_impact_old = None + dempl_labour_supply_constraint = np.zeros_like(dempl_total) # these are the thereshold values for convergence # labor_diff, tax_diff = 0.05, 0.05 # A_trade_diff = 0.05 @@ -1231,9 +1233,9 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d while self.SWITCH_WITHIN_YEAR_LOOP: iter_time = time.time() - #region 2.2.15.1_Adjust_labour_income [rgba(26,188,156,0.15)] - #region desc [rgba(26,188,156,0.50)] - # ^w [2.2.15.1] Labour income adjustment + #region 2.2.15.1_Adjust_labour_income [rgba(26,188,156,0.15)] + #region desc [rgba(26,188,156,0.50)] + # ^w [2.2.15.1] Labour income adjustment # ^w calculate delta tax revenue and take dempl_total, calculate new labour compensation based on those #endregion A_old = A_trade @@ -1250,7 +1252,8 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d cost_curves_impact_old = cost_curves_impact[['REG_imp','PROD_COMM','input_cost_change']].copy() cost_curves_impact_old = cost_curves_impact_old.rename(columns={'input_cost_change':'input_cost_change_old'}) - dempl_labour_supply_constraint = np.zeros_like(dempl_total) + if iter_run > 1: + dempl_labour_supply_constraint = np.zeros_like(dempl_total) # ? calculate new emission cost based on new trade and new output Energy_emissions.update_ind_base(A_trade, self.V.read_var("output", year-1) + self.V.read_var("dq_total", year)) @@ -1546,16 +1549,16 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d dy_trade_hh = fd_trade_response['dy']['dy_trade_hh'] dy_trade_fcf = fd_trade_response['dy']['dy_trade_fcf'] dy_trade_gov = fd_trade_response['dy']['dy_trade_gov'] - + # Build new A matrix due to trade (import) substitution - + # add scenario based changes # ind_trade = IO_model.io_change(io_changes, ind_trade) A_trade = IO_model.build_A_matrix(input_df=ind_trade, variable='IO_coef_trade') IO_model.update_Leontieff(A_trade) dq_trade_eff = IO_model.calc_dq_trade((dq_tech_eff)) self.V.write_var("dq_trade_eff", year, dq_trade_eff) - + Price_model.update_A_BASE(A_trade) Price_model.calc_positive_and_negative_L() @@ -1630,7 +1633,7 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d ind_ener_iter = IO_model.io_change(io_changes_, ind_trade) # ind_trade = ind_ener_iter - + # ! SET IO A_iochange = IO_model.build_A_matrix(input_df=ind_ener_iter, variable='IO_coef_trade') fd_vec_tmp = IO_model.calc_fd_vec(dq_total+IO_model.q_base_curr) @@ -1813,8 +1816,8 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d print(f"--- 2.2.15 Price change relative diff.: {round(price_cond * 100, 4)}% ---") print(f"--- 2.2.15 Labour constraint diff.: {round(labor_constraint_cond * 100, 4)}% ---") print(f"--- 2.2.15 MRIO matrix relative diff.: {round(np.max(np.abs(np.nan_to_num(A_old - A_trade, nan=0))) * 100, 4)}pp ---") - - if ((labor_cond < self.COND_LABOR and tax_rev_cond < self.COND_TAX and price_cond < self.COND_PRICE and + + if ((labor_cond < self.COND_LABOR and tax_rev_cond < self.COND_TAX and price_cond < self.COND_PRICE and np.allclose(A_old, A_trade, atol=self.COND_TRADE)) or (iter_run > self.ITER_MAX)): break From 50bc6e225b6bf692b842551fef368da03c335cae Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Thu, 4 Jun 2026 23:35:19 +0100 Subject: [PATCH 11/14] government: correct fillna defaults and protect against zero VIGA_nominal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After merging emission_cost_ with consumption_shares, the blanket .fillna(0) incorrectly zeroed out price_index for non-matched rows. price_index should default to 1.0 (no price change), not 0.0 (which would zero all nominal values downstream). Apply separate fillna calls: emission_cost → 0 price_index → 1.0 Protect the subsequent division by VIGA_nominal against rows where government expenditure is zero: replace zeros with NaN before dividing, then fillna(0) on the result so those rows contribute nothing to the emission cost share rather than producing inf. Add `import numpy as np` which was missing from the module header. --- SourceCode/government.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/SourceCode/government.py b/SourceCode/government.py index 40b6a19..e538af0 100644 --- a/SourceCode/government.py +++ b/SourceCode/government.py @@ -6,6 +6,7 @@ """ import pandas as pd +import numpy as np from SourceCode.utils import MRIO_df_to_vec, MRIO_vec_to_df import os @@ -181,9 +182,12 @@ def calc_gov_demand(self, government_spending_, price_index, emission_cost, new_ gov_base = gov_base[['REG_exp','REG_imp','TRAD_COMM','VIGA']].copy() consumption_shares = gov_base.merge(prices, how='left') - consumption_shares = consumption_shares.merge(emission_cost_, how='left', on=['REG_exp','REG_imp','TRAD_COMM']).fillna(0) + consumption_shares = consumption_shares.merge(emission_cost_, how='left', on=['REG_exp','REG_imp','TRAD_COMM']) + consumption_shares['emission_cost'] = consumption_shares['emission_cost'].fillna(0) + consumption_shares['price_index'] = consumption_shares['price_index'].fillna(1.0) consumption_shares['VIGA_nominal'] = consumption_shares['VIGA'] * consumption_shares['price_index'] - consumption_shares['emission_cost'] = consumption_shares['emission_cost'] / consumption_shares['VIGA_nominal'] + _denom = consumption_shares['VIGA_nominal'].replace(0, np.nan) + consumption_shares['emission_cost'] = (consumption_shares['emission_cost'] / _denom).fillna(0) consumption_shares['emission_cost'] = consumption_shares['emission_cost'].apply(lambda x: 0 if x < 0 else (3.0 if x > 3.0 else x)) consumption_shares['price_index'] = consumption_shares['price_index'] + consumption_shares['emission_cost'] consumption_shares['share'] = consumption_shares['VIGA'] / consumption_shares.groupby(['REG_imp'])['VIGA'].transform('sum') From 3dfaee238d988bb18f1029e9b830c991bc18b6cb Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Fri, 5 Jun 2026 12:41:14 +0100 Subject: [PATCH 12/14] Deal with setting passed to ftt --- MINDSET_FTT_Power/SourceCode/model_class.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/MINDSET_FTT_Power/SourceCode/model_class.py b/MINDSET_FTT_Power/SourceCode/model_class.py index 0caf332..6422598 100644 --- a/MINDSET_FTT_Power/SourceCode/model_class.py +++ b/MINDSET_FTT_Power/SourceCode/model_class.py @@ -20,7 +20,7 @@ from tqdm import tqdm from ftt_source.paths import set_paths -from ftt_source.Power.ftt_p_main import solve as ftt_p_solve +from ftt_source.Power.ftt_p_main import solve as ftt_p_solve, build_power_settings # Support modules import MINDSET_FTT_Power.SourceCode.support.input_functions as in_f @@ -120,6 +120,7 @@ def __init__(self): # Load classification titles self.titles = titles_f.load_titles() self.conv = titles_f.load_converters() + self.power_settings = build_power_settings(self.titles, config) # Load variable dimensions self.dims, self.histend, self.domain, self.forstart = dims_f.load_dims() @@ -256,10 +257,10 @@ def solve_year(self, year, y, scenario, max_iter=1): variables['FPIX'] = fpix_lag * (1.0 + fpi) self._fpix_carry = variables['FPIX'].copy() - # Call FTT_Standalone (drops iter_lag and conv; adds settings_path) + # Call FTT_Standalone variables = ftt_p_solve( variables, time_lags, self.titles, self.histend, - tl[y], self.domain, settings_path=self.ftt_settings_path + tl[y], self.domain, self.power_settings ) if not any(True for x in modules_list if x in self.ftt_modules): From 29b1d1a22e643b2c4739566b17d64c18d22ef00f Mon Sep 17 00:00:00 2001 From: ADanneaux Date: Fri, 5 Jun 2026 16:23:37 +0100 Subject: [PATCH 13/14] Solve PROD_COMM look up --- SourceCode/ftt_power.py | 19 +++++++++++-------- SourceCode/initiate_modules.py | 20 +++++++++++--------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/SourceCode/ftt_power.py b/SourceCode/ftt_power.py index 916cc3b..a516c28 100644 --- a/SourceCode/ftt_power.py +++ b/SourceCode/ftt_power.py @@ -59,20 +59,23 @@ def solve_year_ftt(self, year, ftt_model, DYNAMIC, Scenario, ener_base, IO_model # model_class.solve_year() applies the EUR2015→USD2013 conversion using current-year FTT exchange-rate variables. c_price_power = Scenario.tax_rate.loc[Scenario.tax_rate.PROD_COMM == 93].copy() reppx_arr = np.zeros((n_reg, n_tech, 1)) + # converters.csv uses numbered T2TI names matching titles['T2TI'] (converters.xlsx/ftt_t2ti_erti.csv use short names — wrong for this lookup) + _conv_csv = pd.read_csv(Path('MINDSET_FTT_Power/Utilities/titles/converters.csv')) + _t2ti_list = list(ftt_model.titles['T2TI']) + _erti_to_t2ti_idxs = {} + for _, row in _conv_csv.iterrows(): + _erti_to_t2ti_idxs.setdefault(row['ERTI'], []).append(_t2ti_list.index(row['T2TI'])) for fuel, sectors in ftt_model.ftt_fuel_converter.groupby('ERTI'): - if fuel not in ftt_model.conv['T2TI_ERTI'].ERTI.values: + if fuel not in _erti_to_t2ti_idxs: continue sec_c_price = c_price_power.loc[c_price_power.TRAD_COMM.isin(sectors.TRAD_COMM)] if len(sec_c_price) == 0: continue avg_by_reg = sec_c_price.groupby('REG_imp')['ctax'].mean() - tech = ftt_model.conv['T2TI_ERTI'].index[ - ftt_model.conv['T2TI_ERTI'].ERTI == fuel - ].values[0] - tech_idx = ftt_model.titles['T2TI'].index(tech) - for reg_idx, reg_short in enumerate(_rti_short): - if reg_short in avg_by_reg.index: - reppx_arr[reg_idx, tech_idx, 0] = avg_by_reg[reg_short] + for tech_idx in _erti_to_t2ti_idxs[fuel]: + for reg_idx, reg_short in enumerate(_rti_short): + if reg_short in avg_by_reg.index: + reppx_arr[reg_idx, tech_idx, 0] = avg_by_reg[reg_short] if np.any(reppx_arr != 0): ftt_model._mset_reppx = reppx_arr # else: leave _mset_reppx as None → CO2taxP stays zero (correct for no-tax years) diff --git a/SourceCode/initiate_modules.py b/SourceCode/initiate_modules.py index 01feeb4..3864590 100644 --- a/SourceCode/initiate_modules.py +++ b/SourceCode/initiate_modules.py @@ -400,7 +400,7 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d flow_price_impact_hh.loc[flow_price_impact_hh['flow_price_impact_hh']> 5, 'flow_price_impact_hh'] = 5.0 self.V.write_var('flow_price_impact_intermediates', year, flow_price_impact_intermediates, from_df=True) - self.V.write_var('flow_price_impact_hh', year, flow_price_impact_hh, from_df=True) + self.V.write_var('flow_price_impact_hh', year, flow_price_impact_hh.rename(columns={'PROD_COMM': 'FD'}), from_df=True) BTA_cou = BTA(Scenario, self.bta, EXOG_VARS.R, self.temp, EXOG_VARS) if year > self.model_start: @@ -409,7 +409,7 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d cbam_incidence = BTA_cou.calc_cbam_incidence(EXOG_VARS.IND_BASE, DYNAMIC['carbon_content'], EXOG_VARS.HH_BASE, EXOG_VARS.FCF_BASE, EXOG_VARS.GOV_BASE) # cbam incidence ['REG_imp','REG_exp','PROD_COMM','TRAD_COMM','cbam_cost'] cbam_cost_dfs = { - 'intermediates': cbam_incidence[~cbam_incidence['PROD_COMM'].str.contains("FD")].copy(), + 'intermediates': cbam_incidence[~cbam_incidence['PROD_COMM'].str.contains("FD")].copy().astype({'PROD_COMM': int, 'TRAD_COMM': int}), 'hh': cbam_incidence[cbam_incidence['PROD_COMM']=="FD_1"].copy(), 'fcf': cbam_incidence[cbam_incidence['PROD_COMM']=="FD_4"].copy(), 'gov': cbam_incidence[cbam_incidence['PROD_COMM']=="FD_3"].copy(), @@ -1259,16 +1259,18 @@ def initiate_modules(self, DYNAMIC, EXOG_VARS, MRIO_df_to_vec_DEF, MRIO_vec_to_d Energy_emissions.update_ind_base(A_trade, self.V.read_var("output", year-1) + self.V.read_var("dq_total", year)) tax_incidence = Energy_emissions.calculate_tax_incidence() self.V.write_var_df('emission_cost_intermediates', year, tax_incidence['tax_incidence_intermediates']) - self.V.write_var_df('emission_cost_hh', year, tax_incidence['tax_incidence_hh']) - self.V.write_var_df('emission_cost_fcf', year, tax_incidence['tax_incidence_fcf']) - self.V.write_var_df('emission_cost_gov', year, tax_incidence['tax_incidence_gov']) + self.V.write_var_df('emission_cost_hh', year, tax_incidence['tax_incidence_hh'].rename(columns={'PROD_COMM': 'FD'})) + self.V.write_var_df('emission_cost_fcf', year, tax_incidence['tax_incidence_fcf'].rename(columns={'PROD_COMM': 'FD'})) + self.V.write_var_df('emission_cost_gov', year, tax_incidence['tax_incidence_gov'].rename(columns={'PROD_COMM': 'FD'})) cbam_incidence = BTA_cou.calc_cbam_incidence(ind_ener_glo, DYNAMIC['carbon_content'], EXOG_VARS.HH_BASE, EXOG_VARS.FCF_BASE, EXOG_VARS.GOV_BASE) # cbam incidence ['REG_imp','REG_exp','PROD_COMM','TRAD_COMM','cbam_cost'] - self.V.write_var_df('cbam_cost_intermediates', year, cbam_incidence[~cbam_incidence['PROD_COMM'].str.contains("FD")].copy()) - self.V.write_var_df('cbam_cost_hh', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_1"].copy()) - self.V.write_var_df('cbam_cost_fcf', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_4"].copy()) - self.V.write_var_df('cbam_cost_gov', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_3"].copy()) + _cbam_interm = cbam_incidence[~cbam_incidence['PROD_COMM'].str.contains("FD")].copy() + _cbam_interm = _cbam_interm.astype({'PROD_COMM': int, 'TRAD_COMM': int}) + self.V.write_var_df('cbam_cost_intermediates', year, _cbam_interm) + self.V.write_var_df('cbam_cost_hh', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_1"].rename(columns={'PROD_COMM':'FD'})) + self.V.write_var_df('cbam_cost_fcf', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_4"].rename(columns={'PROD_COMM':'FD'})) + self.V.write_var_df('cbam_cost_gov', year, cbam_incidence[cbam_incidence['PROD_COMM']=="FD_3"].rename(columns={'PROD_COMM':'FD'})) # calculate within iteration delta of collected tax revenues dtax_rev = Tax_rev.calc_tax_iter_cond(emission_cost_old, self.V.read_var_df('emission_cost_intermediates', year)) From 4929439de5a8699fb7bd33a3b54f6d7fe393ffd1 Mon Sep 17 00:00:00 2001 From: Femke Nijsse Date: Mon, 8 Jun 2026 09:36:59 +0100 Subject: [PATCH 14/14] Align comment with code Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- MINDSET_FTT_Power/SourceCode/support/input_functions.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MINDSET_FTT_Power/SourceCode/support/input_functions.py b/MINDSET_FTT_Power/SourceCode/support/input_functions.py index 7d71a8e..046793c 100644 --- a/MINDSET_FTT_Power/SourceCode/support/input_functions.py +++ b/MINDSET_FTT_Power/SourceCode/support/input_functions.py @@ -61,8 +61,7 @@ def load_data(titles, dimensions, timeline, scenarios, ftt_modules, forstart): # Filter dims to variables whose dimensions are all resolvable in titles. # VariableListing.csv may contain incomplete-feature variables (e.g. battery_ages, - # SCA, MAN) that reference dimension keys not yet in classification_titles.xlsx. - known_dims = set(titles.keys()) # includes 'TIME' added above + # SCA, MAN) that reference dimension keys not yet in classification_titles.csv. dims = {var: dimensions[var] for var in dimensions if all(d in known_dims for d in dimensions[var])}