Model curtailment on a negative export price (#3986)

apps/predbat/config.py

@@ -602,6 +602,20 @@
         "enable": "expert_mode",
         "default": 0.0,
     },
+    {
+        # Model curtailment of export when the export price is negative (issue #3986). Predbat does not drive
+        # the curtailment, it only models it - the actual curtailment stays with the user's inverter or automation.
+        # "off" (the default) disables the feature. "curtail_excess" models grid export as blocked while PV still
+        # charges the battery and supplies the house. "solar_production_off" models the PV as fully off, for
+        # inverters that can only disable generation rather than limit export.
+        "name": "curtail_on_negative_export_price",
+        "friendly_name": "Curtail on negative export price",
+        "type": "select",
+        "options": ["off", "curtail_excess", "solar_production_off"],
+        "icon": "mdi:transmission-tower-export",
+        "enable": "expert_mode",
+        "default": "off",
+    },
     {
         "name": "car_charging_hold",
         "friendly_name": "Car charging hold (remove car charging energy from load data)",

apps/predbat/fetch.py

@@ -2319,6 +2319,7 @@ def fetch_config_options(self):
         self.iboost_solar_excess = self.get_arg("iboost_solar_excess")
         self.iboost_rate_threshold = self.get_arg("iboost_rate_threshold")
         self.iboost_rate_threshold_export = self.get_arg("iboost_rate_threshold_export")
+        self.curtail_on_negative_export_price = self.get_arg("curtail_on_negative_export_price", "off")
         self.iboost_charging = self.get_arg("iboost_charging")
         self.iboost_gas_scale = self.get_arg("iboost_gas_scale")
         self.iboost_max_energy = self.get_arg("iboost_max_energy")

apps/predbat/output.py

@@ -1178,6 +1178,12 @@ def publish_html_plan(self, pv_forecast_minute_step, pv_forecast_minute_step10,
             elif pv_forecast == 0.0:
                 pv_forecast = "⚊"
 
+            # Mark PV modelled as curtailed on a negative export price with a distinct colour, so curtailed solar
+            # is visually separated from normally-exported solar (issue #3986). With the feature off this never
+            # triggers, leaving the plan unchanged for existing users.
+            if raw_pv_forecast > 0.0 and self.curtail_on_negative_export_price != "off" and rate_value_export < 0:
+                pv_color = "#C8A2C8"
+
             pv_forecast = str(pv_forecast)
             if plan_debug and pv_forecast10 > 0.0:
                 pv_forecast += " (%s)" % (str(pv_forecast10))

apps/predbat/predbat.py

@@ -686,6 +686,7 @@ def reset(self):
         self.iboost_smart_threshold = 0
         self.iboost_rate_threshold = 9999
         self.iboost_rate_threshold_export = 9999
+        self.curtail_on_negative_export_price = "off"
         self.iboost_plan = []
         self.iboost_energy_subtract = True
         self.iboost_running = False

apps/predbat/prediction.py

@@ -157,6 +157,7 @@ def __init__(self, base=None, pv_forecast_minute_step=None, pv_forecast_minute10
             self.inverter_hybrid = base.inverter_hybrid
             self.inverter_limit = base.inverter_limit
             self.export_limit = base.export_limit
+            self.curtail_on_negative_export_price = base.curtail_on_negative_export_price
             self.pv_ac_limit = base.pv_ac_limit
             self.battery_rate_min = base.battery_rate_min
             self.battery_rate_max_charge = base.battery_rate_max_charge
@@ -513,6 +514,7 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
         car_energy_reported_load = self.car_energy_reported_load
         inverter_limit = self.inverter_limit * step
         export_limit = self.export_limit * step
+        curtail_on_negative_export_price = self.curtail_on_negative_export_price
         pv_ac_limit = self.pv_ac_limit * step
         set_charge_low_power = self.set_charge_window and self.set_charge_low_power and (save in ["best", "best10", "test"])
         carbon_enable = self.carbon_enable
@@ -579,6 +581,13 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
                 import_rate = self.rate_max  # Assume in worst case that slot goes away and max rate applies
             export_rate = rate_export.get(minute_absolute, 0)
 
+            # Model curtailment of export when the export price is negative (issue #3986). In curtail_excess
+            # mode the export is modelled as curtailed to net-zero while PV still charges the battery and serves
+            # the house; in solar_production_off mode the PV is modelled as fully off (see where pv_now is read).
+            # Predbat does not drive the curtailment, it only models it.
+            export_curtailed = curtail_on_negative_export_price != "off" and export_rate < 0
+            effective_export_limit = 0 if export_curtailed else export_limit
+
             # Alert?
             alert_keep = all_active_keep.get(minute_absolute, 0)
 
@@ -656,6 +665,9 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
 
             # Get load and pv forecast, total up for all values in the step
             pv_now = pv_forecast_minute_step_flat[minute]
+            # Model PV as fully off during curtailed slots for inverters that can only disable generation (issue #3986)
+            if export_curtailed and curtail_on_negative_export_price == "solar_production_off":
+                pv_now = 0
             load_yesterday = load_minutes_step_flat[minute]
 
             # Count PV kWh
@@ -812,8 +824,8 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
 
                 # Exceed export limit?
                 diff = get_diff(battery_draw, pv_dc, pv_ac, load_yesterday, inverter_loss, inverter_loss_recp)
-                if diff < 0 and abs(diff) > export_limit:
-                    over_limit = abs(diff) - export_limit
+                if diff < 0 and abs(diff) > effective_export_limit:
+                    over_limit = abs(diff) - effective_export_limit
                     reduce_by = over_limit
 
                     if reduce_by > battery_draw:
@@ -1027,8 +1039,8 @@ def run_prediction(self, charge_limit, charge_window, export_window, export_limi
 
             # Export limit, clip PV output
             diff = get_diff(battery_draw, pv_dc, pv_ac, load_yesterday, inverter_loss, inverter_loss_recp)
-            if diff < 0 and abs(diff) > export_limit:
-                over_limit = abs(diff) - export_limit
+            if diff < 0 and abs(diff) > effective_export_limit:
+                over_limit = abs(diff) - effective_export_limit
                 # Only solar PV is truly "clipped" (lost energy); excess battery discharge just gets limited
                 pv_ac_before = pv_ac
                 pv_ac = max(pv_ac - over_limit, 0)

apps/predbat/solcast.py

@@ -882,6 +882,15 @@ def publish_pv_stats(self, pv_forecast_data, divide_by, period):
                     app="solar",
                 )
 
+    def slot_export_curtailed(self, rate):
+        """
+        Return True if the slot (given its historical export rate) was curtailed under the negative-export-price
+        curtailment and should be excluded from PV calibration (issue #3986). Curtailment applies when the
+        feature is enabled and the export rate was negative. Slots with no known historical rate (rate is None)
+        are treated as not curtailed (conservative), and with the feature off nothing is excluded.
+        """
+        return self.base.curtail_on_negative_export_price != "off" and rate is not None and rate < 0
+
     def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10, divide_by, max_kwh, forecast_days, period=None):
         """
         Perform PV calibration based on historical data and forecast data.
@@ -920,6 +929,13 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d
             self.now_utc_exact, prune_today(history_attribute(self.get_history_wrapper("sensor." + self.prefix + "_pv_forecast_h0", days + 1, required=False)), self.now_utc_exact, self.midnight_utc, prune=False, intermediate=True)
         )
 
+        # Find the historical export rates, used to exclude slots that were curtailed under the rate-conditional
+        # export limit from calibration (issue #3986). The rates_export sensor's state is the export rate at each
+        # point in time, so its recorded history gives the historical rate per slot keyed the same way as pv_forecast.
+        rate_export_hist, _ = history_attribute_to_minute_data(
+            self.now_utc_exact, prune_today(history_attribute(self.get_history_wrapper(self.prefix + ".rates_export", days + 1, required=False)), self.now_utc_exact, self.midnight_utc, prune=False, intermediate=True)
+        )
+
         hist_days = min(pv_today_hist_days, pv_forecast_hist_days, days)
         enabled_calibration = True
         if hist_days < 3:
@@ -941,6 +957,10 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d
                 days_prev = int(abs(minute_absolute) / (24 * 60)) + 1
                 slot_abs = minute_absolute % (24 * 60)
                 slot = int(slot_abs / self.plan_interval_minutes) * self.plan_interval_minutes
+                # Exclude slots curtailed under the rate-conditional export limit so deliberate curtailment
+                # is not read as panel underperformance and dragged into the forecast (issue #3986).
+                if self.slot_export_curtailed(rate_export_hist.get(minute, None)):
+                    continue
                 pv_power_hist_by_slot[slot] = pv_power_hist_by_slot.get(slot, 0) + pv_power_hist[minute]
                 pv_power_hist_by_slot_count[slot] = pv_power_hist_by_slot_count.get(slot, 0) + 1
                 past_day_actual[days_prev] = past_day_actual.get(days_prev, 0) + pv_power_hist[minute]
@@ -958,6 +978,10 @@ def pv_calibration(self, pv_forecast_minute, pv_forecast_minute10, pv_forecast_d
             if minute_absolute < 0:
                 slot_abs = minute_absolute % (24 * 60)
                 slot = int(slot_abs / self.plan_interval_minutes) * self.plan_interval_minutes
+                # Mirror the actual-history exclusion so curtailed slots drop out of both aggregates,
+                # keeping the per-day and per-slot scaling factors consistent (issue #3986).
+                if self.slot_export_curtailed(rate_export_hist.get(minute, None)):
+                    continue
                 pv_forecast_by_slot[slot] = pv_forecast_by_slot.get(slot, 0) + pv_forecast[minute]
                 pv_forecast_by_slot_count[slot] = pv_forecast_by_slot_count.get(slot, 0) + 1
                 max_pv_power_forecast = max(max_pv_power_forecast, pv_forecast[minute])

apps/predbat/tests/test_infra.py

@@ -547,6 +547,7 @@ def simple_scenario(
     battery_soc=0.0,
     hybrid=False,
     export_limit=10.0,
+    curtail_on_negative_export_price="off",
     inverter_limit=1.0,
     reserve=0.0,
     charge=0,
@@ -660,6 +661,7 @@ def simple_scenario(
     my_predbat.soc_kw = battery_soc
     my_predbat.inverter_hybrid = hybrid
     my_predbat.export_limit = export_limit / 60.0
+    my_predbat.curtail_on_negative_export_price = curtail_on_negative_export_price
     my_predbat.inverter_limit = inverter_limit / 60.0
     my_predbat.pv_ac_limit = pv_ac_limit / 60.0
     my_predbat.reserve = reserve

apps/predbat/tests/test_model.py

@@ -489,6 +489,49 @@ def run_model_tests(my_predbat):
     failed |= simple_scenario("pv_only_bat_ac_clips2c", my_predbat, 0, 2, assert_final_metric=-export_rate * 24, assert_final_soc=24, with_battery=True, battery_rate_max_charge=2.0)
     failed |= simple_scenario("pv_only_bat_ac_clips3", my_predbat, 0, 3, assert_final_metric=-export_rate * 48, assert_final_soc=24, with_battery=True)
     failed |= simple_scenario("pv_only_bat_ac_export_limit", my_predbat, 0, 3, assert_final_metric=-export_rate * 24 * 0.5, assert_final_soc=24, with_battery=True, export_limit=0.5, assert_clipped=24 * 1.5)
+    # Curtailment on negative export price (issue #3986). Switch the export rate negative so curtailment triggers.
+    reset_rates(my_predbat, import_rate, -export_rate)
+    # curtail_excess: grid export is modelled as blocked, so the surplus PV that can no longer be exported is all
+    # clipped (no export, and no negative-price cost), while the battery still charges from PV (soc 24).
+    failed |= simple_scenario(
+        "curtail_excess_negative",
+        my_predbat,
+        0,
+        3,
+        assert_final_metric=0,
+        assert_final_soc=24,
+        with_battery=True,
+        export_limit=10.0,
+        curtail_on_negative_export_price="curtail_excess",
+        assert_clipped=24 * 2,
+    )
+    # solar_production_off: the PV is modelled as fully off, so the battery never charges (soc 0), nothing is
+    # exported or imported, and there is no PV to clip.
+    failed |= simple_scenario(
+        "curtail_solar_off_negative",
+        my_predbat,
+        0,
+        3,
+        assert_final_metric=0,
+        assert_final_soc=0,
+        with_battery=True,
+        export_limit=10.0,
+        curtail_on_negative_export_price="solar_production_off",
+        assert_clipped=0,
+    )
+    reset_rates(my_predbat, import_rate, export_rate)
+    # Positive export price: even with the feature enabled, curtailment does NOT trigger and export proceeds as
+    # normal (matches pv_only_bat_ac_clips3 above). Guards against false triggering when the price is not negative.
+    failed |= simple_scenario(
+        "curtail_excess_positive_rate",
+        my_predbat,
+        0,
+        3,
+        assert_final_metric=-export_rate * 48,
+        assert_final_soc=24,
+        with_battery=True,
+        curtail_on_negative_export_price="curtail_excess",
+    )
     failed |= simple_scenario(
         "pv_only_bat_ac_export_limit_loss",
         my_predbat,

apps/predbat/tests/test_solcast.py

@@ -62,6 +62,8 @@ def __init__(self):
         self.mock_history = {}
         self.fatal_error = False
         self.args = {}
+        # Mirror the real PredBat default used by pv_calibration's negative-export-price curtailment exclusion.
+        self.curtail_on_negative_export_price = "off"
         self.currency_symbols = ["p", "£"]
         self.arg_errors = []
         self.components = MockComponents(StorageLocalFiles(self.config_root, self.log))
@@ -2082,9 +2084,16 @@ def mock_minute_data_import_export(max_days_previous, now_utc, key, scale=1.0, r
 
         base.minute_data_import_export = mock_minute_data_import_export
 
-        # No forecast history → enabled_calibration will be False (< 3 days), but power
-        # conversion and capped_data paths still execute.
-        solar.get_history_wrapper = lambda entity_id, days, required=False: []
+        # No forecast history → enabled_calibration will be False (< 3 days), but power conversion and capped_data
+        # paths still execute. Record which entities are queried so we can assert the historical export rate is
+        # fetched from the correct entity (issue #3986 regression guard - a wrong name silently disables it).
+        requested_history = []
+
+        def mock_get_history(entity_id, days, required=False):
+            requested_history.append(entity_id)
+            return []
+
+        solar.get_history_wrapper = mock_get_history
 
         # Build a simple pv_forecast_minute: constant 0.05 kW per minute for 4 days
         total_minutes = 4 * 24 * 60
@@ -2094,6 +2103,12 @@ def mock_minute_data_import_export(max_days_previous, now_utc, key, scale=1.0, r
 
         adj_minute, adj_minute10, adj_data = solar.pv_calibration(pv_forecast_minute, pv_forecast_minute10, pv_forecast_data, create_pv10=False, divide_by=1.0, max_kwh=5.0, forecast_days=solar.forecast_days)
 
+        # The historical export rate must be read from the predbat.rates_export entity so curtailed slots can be
+        # excluded from calibration; a wrong entity name would silently disable the exclusion (issue #3986).
+        if (base.prefix + ".rates_export") not in requested_history:
+            print("ERROR: pv_calibration did not query {}.rates_export for curtailment exclusion; queried {}".format(base.prefix, requested_history))
+            failed = True
+
         # Returned minute data must be non-negative
         if any(v < 0 for v in adj_minute.values()):
             print("ERROR: pv_calibration returned negative adjusted forecast values")
@@ -2246,6 +2261,51 @@ def mock_get_history(entity_id, days, required=False, _h0=h0_ha_history):
     return failed
 
 
+def test_pv_calibration_curtailment_exclusion(my_predbat):
+    """
+    Test slot_export_curtailed, which decides whether a slot (given its historical export rate) was curtailed
+    on a negative export price and must therefore be excluded from PV calibration (issue #3986).
+
+    Verifies: with the feature off nothing is curtailed; in the curtail_excess and solar_production_off modes
+    only genuinely negative export rates are curtailed (strict <, so exactly 0 is kept); and slots with no known
+    historical rate (None) are kept (conservative).
+    """
+    print("  - test_pv_calibration_curtailment_exclusion")
+    failed = False
+
+    test_api = create_test_solar_api()
+    try:
+        solar = test_api.solar
+        base = test_api.mock_base
+
+        # Feature off: nothing is curtailed, whatever the rate.
+        base.curtail_on_negative_export_price = "off"
+        for rate in (-5.0, 0.0, 3.0, None):
+            if solar.slot_export_curtailed(rate):
+                print("ERROR: rate {} flagged curtailed with the feature off".format(rate))
+                failed = True
+
+        # Both active modes curtail exactly the negative-price slots.
+        for mode in ("curtail_excess", "solar_production_off"):
+            base.curtail_on_negative_export_price = mode
+            if not solar.slot_export_curtailed(-5.0):
+                print("ERROR: -5 should be curtailed in mode {}".format(mode))
+                failed = True
+            if solar.slot_export_curtailed(0.0):
+                print("ERROR: 0 should NOT be curtailed in mode {} (strict <)".format(mode))
+                failed = True
+            if solar.slot_export_curtailed(3.0):
+                print("ERROR: 3 should NOT be curtailed in mode {}".format(mode))
+                failed = True
+            if solar.slot_export_curtailed(None):
+                print("ERROR: a slot with no known historical rate should NOT be curtailed in mode {}".format(mode))
+                failed = True
+    finally:
+        test_api.cleanup()
+
+    return failed
+
+
 def test_pv_calibration_capped_data_clamp(my_predbat):
     """
     Test that the capped_data clamp in pv_calibration correctly limits the
@@ -2939,6 +2999,7 @@ def run_solcast_tests(my_predbat):
 
     # Calibration tests
     failed |= test_pv_calibration_power_conversion(my_predbat)
+    failed |= test_pv_calibration_curtailment_exclusion(my_predbat)
     failed |= test_pv_calibration_capped_data_clamp(my_predbat)
     failed |= test_pv_calibration_partial_history(my_predbat)
     failed |= test_pv_calibration_synthetic_values(my_predbat)

docs/apps-yaml.md

@@ -947,6 +947,8 @@ approval was lower than your maximum inverter power (check your install informat
 
 If you do not set an export limit then it is assumed to be unlimited (and thus limited by your inverter or PV system).
 
+If your export is curtailed when the export price is negative, see [curtailment on a negative export price](energy-rates.md#curtailment-on-a-negative-export-price), which models that curtailment without affecting the normal export limit.
+
 ### **inverter_limit_charge** and **inverter_limit_discharge**
 
 An optional list of values with one entry per inverter.