From e4683912d4fe6d2ebca43f940f5fcf6eb45a3144 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 25 Oct 2023 17:09:20 +1000 Subject: [PATCH 1/7] Add fcas requirements to class parameters --- nempy/markets.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nempy/markets.py b/nempy/markets.py index 8a5d354..56a59fe 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -80,6 +80,7 @@ class SpotMarket: def __init__(self, market_regions, unit_info, dispatch_interval=5): self.dispatch_interval = dispatch_interval self._unit_info = None + self._fcas_requirements = None self._decision_variables = {} self._variable_to_constraint_map = {'regional': {}, 'unit_level': {}} self._constraint_to_variable_map = {'regional': {}, 'unit_level': {}} @@ -1081,6 +1082,7 @@ def set_fcas_requirements_constraints(self, fcas_requirements): """ if self.validate_inputs: self._validate_fcas_requirements(fcas_requirements) + self._fcas_requirements = fcas_requirements rhs_and_type, variable_map = market_constraints.fcas(fcas_requirements, self._next_constraint_id) self._market_constraints_rhs_and_type['fcas'] = rhs_and_type self._constraint_to_variable_map['regional']['fcas'] = variable_map From e51f6fbdfa255ba13f2ca76ddd870d562df8a2b4 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Wed, 25 Oct 2023 17:11:52 +1000 Subject: [PATCH 2/7] Add limit setter logic to SpotMarket class --- nempy/markets.py | 302 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 302 insertions(+) diff --git a/nempy/markets.py b/nempy/markets.py index 56a59fe..829b831 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -3161,6 +3161,308 @@ def get_interconnector_flows(self): return flow.reset_index(drop=True) + def _summarise_generic_constraint_lhs(self): + """ + """ + columns = ['set', 'term', 'definition', 'coefficient'] + + units = ( + self._generic_constraint_lhs['unit'] + .assign(definition=lambda x: np.where(x['service'] != 'energy', 'unit_fcas', 'unit_energy')) + .rename(columns={'unit': 'term'}) + .get(columns) + ) + interconnectors = ( + self._generic_constraint_lhs['interconnectors'] + .assign(definition=lambda x: x['interconnector']) + .rename(columns={'interconnector': 'term'}) + .get(columns) + ) + fcas = ( + self._fcas_requirements + .assign( + coefficient=1.0, + definition='region_fcas', + ) + .rename(columns={'region': 'term'}) + .get(columns) + ) + lhs = pd.concat([units, interconnectors, fcas]) + + return lhs + + + def _summarise_generic_constraint_rhs(self): + """ + + """ + columns = ['set', 'constraint_id', 'type', 'rhs', 'slack'] + + fcas_constraints = self._market_constraints_rhs_and_type['fcas'].get(columns) + generic_constraints = self._constraints_rhs_and_type['generic'].get(columns) + + rhs = pd.concat([fcas_constraints, generic_constraints]) + return rhs + + def _summarise_generic_constraint(self): + lhs = self._summarise_generic_constraint_lhs() + rhs = self._summarise_generic_constraint_rhs() + + gencon = ( + lhs.merge(rhs, on=['set'], how='inner') + # For all constraint equations with ConstraintType of “≥”, convert to an “≤” inequality by + # multiplying both the LHS & RHS of equation in Step 4 by -1 (from AEMO's logic). + .assign( + coefficient=lambda x: np.where(x['type'] == '>=', x['coefficient'] * -1, x['coefficient']), + rhs=lambda x: np.where(x['type'] == '>=', x['rhs'] * -1, x['rhs']), + # If sign is reversed on both LHS and RHS, we can switch ">=" to "<=". + type=lambda x: np.where(x['type'] == '>=', '<=', x['type']), + lhs=lambda x: x['rhs'] - x['slack'], + ) + ) + return gencon + + def _priority_logic(self, x): + """Implements NEMMCO's priority logic in determining a single constraint responsible for setting an + interconnector import or export limit. + + Parameters + ---------- + x : pd.DataFrame + group by [`subject_interconnector`, `limit_type`] from `self._construct_frequency_matrix`. + + """ + interconnectors = list(self._decision_variables['interconnectors'].get('interconnector')) + subject = x.get('subject_interconnector').iloc[0] + + # Remove subject from interconnector list + interconnectors.remove(subject) + + priority_levels = { + # "If there are any constraint equations defined with only the Subject Interconnector as an LHS term". + 'priority_one': np.logical_and.reduce( + [ + x.get(subject) == 1.0, + x.get(interconnectors).sum(axis=1) == 0.0, + x.get(['unit_energy', 'unit_fcas', 'region_fcas']).sum(axis=1) == 0.0, + ] + ), + # "If there are any constraint equations defined with either:" + 'priority_two': np.logical_or( + # 1) Joint interconnector constraints + np.logical_and.reduce( + [ + x.get(subject) == 1.0, + x.get(interconnectors).sum(axis=1) >= 1.0, + ] + ), + # 2) Option 4 constraints + np.logical_and.reduce( + [ + x.get(subject) == 1.0, + x['unit_energy'] >= 1.0, + ] + ), + ), + # "If there are any constraint equations defined with either:" + 'priority_three': np.logical_or( + # 1) Islanding Risk FCAS requirement constraints + np.logical_and.reduce( + [ + x.get(subject) == 1.0, + x['region_fcas'] >= 1.0, + ] + ), + # 2) Other... + np.logical_and.reduce( + [ + x.get(subject) == 1.0, + x['unit_fcas'] >= 1.0, + ] + ), + ), + } + priority_none = ~np.logical_or.reduce( + np.vstack( + (priority_levels['priority_one'], priority_levels['priority_two'], priority_levels['priority_three']) + ) + ) + + # AEMO requires sorting as per ANSI SQL collation standard but pandas.sort_values() has been implemented for + # simplicity. This could be a source of error. Concatenate in order of priority. + priority_data = ( + pd.concat([ + x[priority_levels['priority_one']].assign(priority='1st').sort_values('set'), + x[priority_levels['priority_two']].assign(priority='2nd').sort_values('set'), + x[priority_levels['priority_three']].assign(priority='3rd').sort_values('set'), + x[priority_none].assign(priority='None').sort_values('set'), + ]) + ) + + return priority_data + + def _apply_default_limits(self, limits): + """Bound restrictive limits to the hard default limits to eliminate infeasible limit values. + + Parameters + ---------- + limits : pd.DataFrame + Return value from `SpotMarket._get_restrictive_limits()` + + """ + export_limits = limits.query('limit_type == "export"') + import_limits = limits.query('limit_type == "import"') + + feasible_export_limits = export_limits.assign(limit=lambda x: np.minimum(x['max'], x['limit'])) + feasible_import_limits = import_limits.assign(limit=lambda x: np.maximum(x['min'], x['limit'])) + + feasible_limits = pd.concat([feasible_export_limits, feasible_import_limits]) + + return feasible_limits + + def _get_restrictive_limits(self, limits): + """Retrieve the most restrictive limits (NEMMCO's upper and lower bounds). + There is a many-to-one relationship between constraint sets and interconnector terms. For example, + NSW1-QLD1 could have multiple constraints setting the most restrictive limit value in either direction. + + Parameters + ---------- + limits : pd.DataFrame + Return value from `self.get_interconnector_constraint_limits()` + + """ + export_limits = limits.query('limit_type == "export"') + import_limits = limits.query('limit_type == "import"') + + min_mask = export_limits['limit'] == export_limits.groupby('interconnector')['limit'].transform('min') + max_mask = import_limits['limit'] == import_limits.groupby('interconnector')['limit'].transform('max') + upper_bound = export_limits[min_mask] + lower_bound = import_limits[max_mask] + + restrictive_limits = pd.concat([upper_bound, lower_bound]) + + return restrictive_limits + + def get_interconnector_limits(self): + """Retrieves the import and export limits for each interconnector constraint. + + Parameters + ---------- + gencon : pd.DataFrame + Return value from `self._summarise_generic_constraint(). + + """ + gencon = self._summarise_generic_constraint() + hard_limits = ( + self._decision_variables['interconnectors'] + .get(['interconnector', 'lower_bound', 'upper_bound', 'value']) + ) + hard_limits.columns = ['term', 'min', 'max', 'flow'] + hard_limits = hard_limits.assign(min=lambda x: x['min'] * -1) + + constrained_limits = ( + gencon.merge(hard_limits, on=['term'], how='inner') + .rename(columns={'term': 'interconnector'}) + .assign( + interconnector_slack=lambda x: x['slack'] / x['coefficient'], + limit=lambda x: np.round(x['flow'] + x['interconnector_slack'], 4), + ) + ) + # For all constraints with a positive coefficient the effective RHS bound value (which may be negative) + # represents an upper bound (that is, contributes to setting the Export Limit) on that interconnector. + # Therefore, constraints where negative coefficients apply, the effective RHS bound value represents a lower + # bound (ie. `import_limit`). + + export_limits = constrained_limits.query('coefficient > 0.0').assign(limit_type='export') + import_limits = constrained_limits.query('coefficient < 0.0').assign(limit_type='import') + + limits = ( + pd.concat([export_limits, import_limits]) + .get(['interconnector', 'set', 'flow', 'min', 'max', 'limit', 'limit_type']) + ) + + return limits + + def _construct_frequency_matrix(self, limits, gencon): + """Construct's NEMMCO's 'master list' in the form of a frequency matrix according to the methodology. + """ + unique_ic = list(limits.get('interconnector').unique()) + column_index = unique_ic + ['unit_energy', 'unit_fcas', 'region_fcas'] + master_list = ( + limits + .rename(columns={'interconnector': 'subject_interconnector'}) + .get(['subject_interconnector', 'set', 'limit_type']) + .merge(gencon, on=['set'], how='inner') + ) + + # Construct frequency matrix + matrix = ( + master_list + .groupby(['subject_interconnector', 'set', 'definition', 'limit_type']) + .agg({'coefficient': 'count'}) + .squeeze() + .unstack('definition') + ) + # Extend for fcas and fill nulls with 0. + matrix_extended = ( + matrix + .reindex(column_index, axis=1) + .fillna(0) + .reset_index() + .rename_axis(None, axis=1) + ) + + return matrix_extended + + def _apply_setter_logic(self, matrix): + reordered_matrix = ( + matrix.groupby(['subject_interconnector', 'limit_type'], as_index=False) + .apply(self._priority_logic) + .reset_index(drop=True) + ) + filtered_matrix = ( + reordered_matrix + .groupby(['subject_interconnector', 'limit_type'], as_index=False) + .first() + .rename(columns={'subject_interconnector': 'interconnector'}) + ) + return filtered_matrix + + def get_interconnector_limit_setter(self): + """ + Retrieves the most restrictive import and export limits and their associated constraint for each interconnector. + + Methodology for calculation is described in the old NEMMCO document: + https://aemo.com.au/-/media/files/electricity/nem/security_and_reliability/dispatch/policy_and_process/ + interconnector-limit-setter-reporting-changes.pdf + + Each limit and associated constraint should match with the following fields from MMS table: + `DISPATCHINTERCONNECTORRES` in the fully constrained `nempy` scenario. + - `EXPORTLIMIT` and `IMPORTLIMIT`; and + - `IMPORTGENCONID` and `EXPORTGENCONID`. + + """ + gencon = self._summarise_generic_constraint() + limits = ( + self.get_interconnector_limits() + .pipe(self._get_restrictive_limits) + .pipe(self._apply_default_limits) + ) + matrix = ( + self._construct_frequency_matrix(limits, gencon) + .pipe(self._apply_setter_logic) + ) + published_setter = ( + matrix + .get(['interconnector', 'limit_type', 'set']) + .merge(limits, on=['interconnector', 'limit_type', 'set'], how='inner') + .get(['interconnector', 'set', 'flow', 'min', 'max', 'limit', 'limit_type']) + ) + + return published_setter + + + def get_region_dispatch_summary(self): """Calculates a dispatch summary at the regional level. From 26e5a634b86e92d533fb77f4d72f1987884b9adf Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Dec 2023 10:08:40 +1000 Subject: [PATCH 3/7] Fix import limit setting error --- nempy/markets.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nempy/markets.py b/nempy/markets.py index 829b831..e947261 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -3314,7 +3314,9 @@ def _apply_default_limits(self, limits): import_limits = limits.query('limit_type == "import"') feasible_export_limits = export_limits.assign(limit=lambda x: np.minimum(x['max'], x['limit'])) - feasible_import_limits = import_limits.assign(limit=lambda x: np.maximum(x['min'], x['limit'])) + + # Error in NEMMCO document; should be minimum, not maximum, as sign of limit has already been reversed. + feasible_import_limits = import_limits.assign(limit=lambda x: np.minimum(x['min'], x['limit'])) feasible_limits = pd.concat([feasible_export_limits, feasible_import_limits]) From 535814a6bf54d4f2abf67f96963ba086bd7d4906 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Dec 2023 11:00:01 +1000 Subject: [PATCH 4/7] Better conditional logic --- nempy/markets.py | 105 +++++++++++++++++++++++++++++------------------ 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/nempy/markets.py b/nempy/markets.py index e947261..ad2abfd 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -3166,27 +3166,39 @@ def _summarise_generic_constraint_lhs(self): """ columns = ['set', 'term', 'definition', 'coefficient'] - units = ( - self._generic_constraint_lhs['unit'] - .assign(definition=lambda x: np.where(x['service'] != 'energy', 'unit_fcas', 'unit_energy')) - .rename(columns={'unit': 'term'}) - .get(columns) - ) - interconnectors = ( - self._generic_constraint_lhs['interconnectors'] - .assign(definition=lambda x: x['interconnector']) - .rename(columns={'interconnector': 'term'}) - .get(columns) - ) - fcas = ( - self._fcas_requirements - .assign( - coefficient=1.0, - definition='region_fcas', + if 'unit' in self._generic_constraint_lhs: + units = ( + self._generic_constraint_lhs['unit'] + .assign(definition=lambda x: np.where(x['service'] != 'energy', 'unit_fcas', 'unit_energy')) + .rename(columns={'unit': 'term'}) + .get(columns) ) - .rename(columns={'region': 'term'}) - .get(columns) - ) + else: + units = pd.DataFrame(columns=columns) + + if 'interconnectors' in self._generic_constraint_lhs: + interconnectors = ( + self._generic_constraint_lhs['interconnectors'] + .assign(definition=lambda x: x['interconnector']) + .rename(columns={'interconnector': 'term'}) + .get(columns) + ) + else: + interconnectors = pd.DataFrame(columns=columns) + + if self._fcas_requirements is not None: + fcas = ( + self._fcas_requirements + .assign( + coefficient=1.0, + definition='region_fcas', + ) + .rename(columns={'region': 'term'}) + .get(columns) + ) + else: + fcas = pd.DataFrame(columns=columns) + lhs = pd.concat([units, interconnectors, fcas]) return lhs @@ -3197,9 +3209,15 @@ def _summarise_generic_constraint_rhs(self): """ columns = ['set', 'constraint_id', 'type', 'rhs', 'slack'] + if 'fcas' in self._market_constraints_rhs_and_type: + fcas_constraints = self._market_constraints_rhs_and_type['fcas'].get(columns) + else: + fcas_constraints = pd.DataFrame(columns=columns) - fcas_constraints = self._market_constraints_rhs_and_type['fcas'].get(columns) - generic_constraints = self._constraints_rhs_and_type['generic'].get(columns) + if 'generic' in self._constraints_rhs_and_type: + generic_constraints = self._constraints_rhs_and_type['generic'].get(columns) + else: + generic_constraints = pd.DataFrame(columns=columns) rhs = pd.concat([fcas_constraints, generic_constraints]) return rhs @@ -3354,7 +3372,9 @@ def get_interconnector_limits(self): Return value from `self._summarise_generic_constraint(). """ - gencon = self._summarise_generic_constraint() + if 'interconnectors' not in self._decision_variables: + raise NotImplementedError("No interconnectors have been set in this instance.") + hard_limits = ( self._decision_variables['interconnectors'] .get(['interconnector', 'lower_bound', 'upper_bound', 'value']) @@ -3362,26 +3382,31 @@ def get_interconnector_limits(self): hard_limits.columns = ['term', 'min', 'max', 'flow'] hard_limits = hard_limits.assign(min=lambda x: x['min'] * -1) - constrained_limits = ( - gencon.merge(hard_limits, on=['term'], how='inner') - .rename(columns={'term': 'interconnector'}) - .assign( - interconnector_slack=lambda x: x['slack'] / x['coefficient'], - limit=lambda x: np.round(x['flow'] + x['interconnector_slack'], 4), + if self._generic_constraint_lhs is not None: + gencon = self._summarise_generic_constraint() + + constrained_limits = ( + gencon.merge(hard_limits, on=['term'], how='inner') + .rename(columns={'term': 'interconnector'}) + .assign( + interconnector_slack=lambda x: x['slack'] / x['coefficient'], + limit=lambda x: np.round(x['flow'] + x['interconnector_slack'], 4), + ) ) - ) - # For all constraints with a positive coefficient the effective RHS bound value (which may be negative) - # represents an upper bound (that is, contributes to setting the Export Limit) on that interconnector. - # Therefore, constraints where negative coefficients apply, the effective RHS bound value represents a lower - # bound (ie. `import_limit`). + # For all constraints with a positive coefficient the effective RHS bound value (which may be negative) + # represents an upper bound (that is, contributes to setting the Export Limit) on that interconnector. + # Therefore, constraints where negative coefficients apply, the effective RHS bound value represents a lower + # bound (ie. `import_limit`). - export_limits = constrained_limits.query('coefficient > 0.0').assign(limit_type='export') - import_limits = constrained_limits.query('coefficient < 0.0').assign(limit_type='import') + export_limits = constrained_limits.query('coefficient > 0.0').assign(limit_type='export') + import_limits = constrained_limits.query('coefficient < 0.0').assign(limit_type='import') - limits = ( - pd.concat([export_limits, import_limits]) - .get(['interconnector', 'set', 'flow', 'min', 'max', 'limit', 'limit_type']) - ) + limits = ( + pd.concat([export_limits, import_limits]) + .get(['interconnector', 'set', 'flow', 'min', 'max', 'limit', 'limit_type']) + ) + else: + raise NotImplementedError('No generic constraints have been set in this instance.') return limits From dbe3e509a6c2d9c1ebce7295a24a686cb40b1db8 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Dec 2023 11:00:55 +1000 Subject: [PATCH 5/7] rename functions --- nempy/markets.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/nempy/markets.py b/nempy/markets.py index ad2abfd..210cf7f 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -3363,14 +3363,12 @@ def _get_restrictive_limits(self, limits): return restrictive_limits - def get_interconnector_limits(self): + def get_interconnector_energy_constraint_limits(self): """Retrieves the import and export limits for each interconnector constraint. - Parameters - ---------- - gencon : pd.DataFrame - Return value from `self._summarise_generic_constraint(). - + Requires: + - "interconnectors" in self._decision_variables; and + - Generic constraints to be set in "self._generic_constraint_lhs". """ if 'interconnectors' not in self._decision_variables: raise NotImplementedError("No interconnectors have been set in this instance.") @@ -3455,9 +3453,10 @@ def _apply_setter_logic(self, matrix): ) return filtered_matrix - def get_interconnector_limit_setter(self): + def get_interconnector_energy_constraint_setter(self): """ - Retrieves the most restrictive import and export limits and their associated constraint for each interconnector. + Retrieves the most restrictive import and export energy limits and their associated constraint for each + interconnector. Methodology for calculation is described in the old NEMMCO document: https://aemo.com.au/-/media/files/electricity/nem/security_and_reliability/dispatch/policy_and_process/ @@ -3471,7 +3470,7 @@ def get_interconnector_limit_setter(self): """ gencon = self._summarise_generic_constraint() limits = ( - self.get_interconnector_limits() + self.get_interconnector_energy_constraint_limits() .pipe(self._get_restrictive_limits) .pipe(self._apply_default_limits) ) From c15e5e4aa3f285b20ae9d6df80d502561920b61d Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Dec 2023 11:01:24 +1000 Subject: [PATCH 6/7] Add FCAS limit setter --- nempy/historical_inputs/interconnectors.py | 14 +++++--- nempy/historical_inputs/mms_db.py | 16 +++++---- nempy/markets.py | 40 ++++++++++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/nempy/historical_inputs/interconnectors.py b/nempy/historical_inputs/interconnectors.py index 90ce464..df962c3 100644 --- a/nempy/historical_inputs/interconnectors.py +++ b/nempy/historical_inputs/interconnectors.py @@ -306,8 +306,10 @@ def _format_interconnector_definitions(INTERCONNECTOR, INTERCONNECTORCONSTRAINT) """ interconnector_directions = INTERCONNECTOR.loc[:, ['INTERCONNECTORID', 'REGIONFROM', 'REGIONTO']] interconnector_directions.columns = ['interconnector', 'from_region', 'to_region'] - interconnector_paramaters = INTERCONNECTORCONSTRAINT.loc[:, ['INTERCONNECTORID', 'IMPORTLIMIT', 'EXPORTLIMIT']] - interconnector_paramaters.columns = ['interconnector', 'min', 'max'] + interconnector_paramaters = ( + INTERCONNECTORCONSTRAINT.loc[:, ['INTERCONNECTORID', 'IMPORTLIMIT', 'EXPORTLIMIT', 'FCASSUPPORTUNAVAILABLE']] + ) + interconnector_paramaters.columns = ['interconnector', 'min', 'max', 'fcas_support_unavailable'] interconnector_paramaters['min'] = -1 * interconnector_paramaters['min'] interconnectors = pd.merge(interconnector_directions, interconnector_paramaters, 'inner', on='interconnector') return interconnectors @@ -349,13 +351,15 @@ def _format_mnsp_transmission_loss_factors(MNSP_INTERCONNECTOR, INTERCONNECTORCO """ INTERCONNECTORCONSTRAINT = INTERCONNECTORCONSTRAINT[INTERCONNECTORCONSTRAINT['ICTYPE'] == 'MNSP'] MNSP_INTERCONNECTOR = pd.merge(MNSP_INTERCONNECTOR, INTERCONNECTORCONSTRAINT, on=['INTERCONNECTORID']) - MNSP_INTERCONNECTOR = MNSP_INTERCONNECTOR.loc[:, ['INTERCONNECTORID', 'LINKID', 'FROM_REGION_TLF', 'TO_REGION_TLF', - 'FROMREGION', 'TOREGION', 'LHSFACTOR', 'MAXCAPACITY']] + MNSP_INTERCONNECTOR = ( + MNSP_INTERCONNECTOR.loc[:, ['INTERCONNECTORID', 'LINKID', 'FROM_REGION_TLF', 'TO_REGION_TLF', + 'FROMREGION', 'TOREGION', 'LHSFACTOR', 'MAXCAPACITY', 'FCASSUPPORTUNAVAILABLE']] + ) mnsp_transmission_loss_factors = MNSP_INTERCONNECTOR.rename(columns={ 'INTERCONNECTORID': 'interconnector', 'LINKID': 'link', 'FROM_REGION_TLF': 'from_region_loss_factor', 'TO_REGION_TLF': 'to_region_loss_factor', 'FROMREGION': 'from_region', 'TOREGION': 'to_region', - 'LHSFACTOR': 'generic_constraint_factor', 'MAXCAPACITY': 'max' + 'LHSFACTOR': 'generic_constraint_factor', 'MAXCAPACITY': 'max', 'FCASSUPPORTUNAVAILABLE': 'fcas_support_unavailable', }) mnsp_transmission_loss_factors['min'] = 0.0 return mnsp_transmission_loss_factors diff --git a/nempy/historical_inputs/mms_db.py b/nempy/historical_inputs/mms_db.py index be59449..885622c 100644 --- a/nempy/historical_inputs/mms_db.py +++ b/nempy/historical_inputs/mms_db.py @@ -168,9 +168,10 @@ def __init__(self, connection): table_name='INTERCONNECTOR', table_columns=['INTERCONNECTORID', 'REGIONFROM', 'REGIONTO'], table_primary_keys=['INTERCONNECTORID'], con=self.con) self.INTERCONNECTORCONSTRAINT = InputsByEffectiveDateVersionNoAndDispatchInterconnector( - table_name='INTERCONNECTORCONSTRAINT', table_columns=['INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO', - 'FROMREGIONLOSSSHARE', 'LOSSCONSTANT', 'ICTYPE', - 'LOSSFLOWCOEFFICIENT', 'IMPORTLIMIT', 'EXPORTLIMIT'], + table_name='INTERCONNECTORCONSTRAINT', table_columns=[ + 'INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO', 'FROMREGIONLOSSSHARE', 'LOSSCONSTANT', 'ICTYPE', + 'LOSSFLOWCOEFFICIENT', 'IMPORTLIMIT', 'EXPORTLIMIT', 'FCASSUPPORTUNAVAILABLE', + ], table_primary_keys=['INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO'], con=self.con) self.LOSSMODEL = InputsByEffectiveDateVersionNoAndDispatchInterconnector( table_name='LOSSMODEL', table_columns=['INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO', 'LOSSSEGMENT', @@ -181,8 +182,10 @@ def __init__(self, connection): 'DEMANDCOEFFICIENT'], table_primary_keys=['INTERCONNECTORID', 'EFFECTIVEDATE', 'VERSIONNO'], con=self.con) self.DISPATCHINTERCONNECTORRES = InputsBySettlementDate( - table_name='DISPATCHINTERCONNECTORRES', table_columns=['INTERCONNECTORID', 'SETTLEMENTDATE', 'MWFLOW', - 'MWLOSSES'], + table_name='DISPATCHINTERCONNECTORRES', table_columns=[ + 'INTERCONNECTORID', 'SETTLEMENTDATE', 'MWFLOW', 'MWLOSSES', 'EXPORTLIMIT', 'IMPORTLIMIT', + 'EXPORTGENCONID', 'IMPORTGENCONID', 'FCASEXPORTLIMIT', 'FCASIMPORTLIMIT', + ], table_primary_keys=['INTERCONNECTORID', 'SETTLEMENTDATE'], con=self.con) self.MNSP_INTERCONNECTOR = InputsByEffectiveDateVersionNo( table_name='MNSP_INTERCONNECTOR', table_columns=['INTERCONNECTORID', 'LINKID', 'EFFECTIVEDATE', 'VERSIONNO', @@ -455,7 +458,8 @@ def __init__(self, table_name, table_columns, table_primary_keys, con): 'RAISE60SECROP': 'REAL', 'RAISE5MINROP': 'REAL', 'RAISEREGROP': 'REAL', 'LOWER6SECROP': 'REAL', 'LOWER60SECROP': 'REAL', 'LOWER5MINROP': 'REAL', 'LOWERREGROP': 'REAL', 'FROM_REGION_TLF': 'REAL', 'TO_REGION_TLF': 'REAL', 'ICTYPE': 'TEXT', 'LINKID': 'TEXT', 'FROMREGION': 'TEXT', 'TOREGION': 'TEXT', - 'REGISTEREDCAPACITY': 'REAL', 'LHSFACTOR': 'FACTOR', 'ROP': 'REAL' + 'REGISTEREDCAPACITY': 'REAL', 'LHSFACTOR': 'FACTOR', 'ROP': 'REAL', 'FCASSUPPORTUNAVAILABLE': 'REAL', + 'FCASEXPORTLIMIT': 'REAL', 'FCASIMPORTLIMIT': 'REAL', 'EXPORTGENCONID': 'TEXT', 'IMPORTGENCONID': 'TEXT', } def create_table_in_sqlite_db(self): diff --git a/nempy/markets.py b/nempy/markets.py index 210cf7f..2966843 100644 --- a/nempy/markets.py +++ b/nempy/markets.py @@ -1758,6 +1758,8 @@ def _validate_interconnector_definitions(self, interconnector_directions_and_lim must_be_real_number=True, not_negative=True)) schema.add_column(dv.SeriesSchema(name='to_region_loss_factor', data_type=np.float64, must_be_real_number=True, not_negative=True)) + schema.add_column(dv.SeriesSchema(name='fcas_support_unavailable', data_type=np.float64, must_be_real_number=True, + not_negative=True)) schema.validate(interconnector_directions_and_limits) def set_interconnector_losses(self, loss_functions, interpolation_break_points): @@ -3487,6 +3489,44 @@ def get_interconnector_energy_constraint_setter(self): return published_setter + + def get_interconnector_fcas_constraint_setter(self): + """ + Retrieves the most restrictive import and export FCAS limits for each interconnector. + + Methodology for calculation is described in the old NEMMCO document: + https://aemo.com.au/-/media/files/electricity/nem/security_and_reliability/dispatch/policy_and_process/ + interconnector-limit-setter-reporting-changes.pdf + + Each limit and associated constraint should match with the following fields from MMS table: + `DISPATCHINTERCONNECTORRES` in the fully constrained `nempy` scenario. + - `FCASEXPORTLIMIT` and `FCASIMPORTLIMIT` + + """ + interconnector_energy_constraint_setter = self.get_interconnector_energy_constraint_setter() + fcas_support = ( + self._interconnector_directions.get(['interconnector', 'fcas_support_unavailable']).drop_duplicates() + ) + interconnector_fcas_constraint_setter = ( + interconnector_energy_constraint_setter + .merge(fcas_support, on=['interconnector'], how='inner') + ) + export_fcas_constraint_setter = ( + interconnector_fcas_constraint_setter.query('limit_type == "export"') + .assign(limit=lambda x: np.where(x['fcas_support_unavailable'] == 0.0, x['max'], x['limit'])) + ) + import_fcas_constraint_setter = ( + interconnector_fcas_constraint_setter.query('limit_type == "import"') + .assign(limit=lambda x: np.where(x['fcas_support_unavailable'] == 0.0, x['min'] * -1, x['limit'])) + ) + + published_setter = ( + pd.concat([export_fcas_constraint_setter, import_fcas_constraint_setter]) + .drop(['fcas_support_unavailable'], axis=1) + .sort_values(by=['interconnector']) + ) + + return published_setter def get_region_dispatch_summary(self): From 570e44e0b115bbd46ec29f90726f3ccea3032bd3 Mon Sep 17 00:00:00 2001 From: Michael Lee Date: Thu, 7 Dec 2023 11:01:40 +1000 Subject: [PATCH 7/7] Fix bug that prevents joining on DISPATCHINTERCONNECTORRES --- nempy/historical_inputs/mms_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nempy/historical_inputs/mms_db.py b/nempy/historical_inputs/mms_db.py index 885622c..0d638d9 100644 --- a/nempy/historical_inputs/mms_db.py +++ b/nempy/historical_inputs/mms_db.py @@ -1283,7 +1283,7 @@ def get_data(self, date_time): # Inner join the most recent data with the interconnectors used in the actual interval of interest. query = """SELECT {cols} FROM temp4 - INNER JOIN (SELECT * + INNER JOIN (SELECT INTERCONNECTORID FROM DISPATCHINTERCONNECTORRES WHERE SETTLEMENTDATE == '{datetime}') USING (INTERCONNECTORID);"""