Feature: Automated Solar Clipping Mitigation (using native clear-sky models)

.cspell/custom-dictionary-workspace.txt

@@ -54,6 +54,7 @@ chargehold
 chargelater
 Chrg
 citem
+clearsky
 Codespaces
 collapsable
 compareform
@@ -270,6 +271,7 @@ oncharge
 oninput
 onmouseout
 onmouseover
+openmeteo
 openweathermap
 overfitting
 overvoltage

apps/predbat/components.py

@@ -108,11 +108,15 @@
             "pv_forecast_tomorrow": {"required": False, "config": "pv_forecast_tomorrow"},
             "pv_forecast_d3": {"required": False, "config": "pv_forecast_d3"},
             "pv_forecast_d4": {"required": False, "config": "pv_forecast_d4"},
+            "pv_clearsky_today": {"required": False, "config": "pv_clearsky_today"},
+            "pv_clearsky_tomorrow": {"required": False, "config": "pv_clearsky_tomorrow"},
+            "pv_clearsky_d3": {"required": False, "config": "pv_clearsky_d3"},
+            "pv_clearsky_d4": {"required": False, "config": "pv_clearsky_d4"},
             "pv_scaling": {"required": False, "config": "pv_scaling", "default": 1.0},
             "open_meteo_forecast": {"required": False, "config": "open_meteo_forecast", "default": False},
             "open_meteo_forecast_max_age": {"required": False, "config": "open_meteo_forecast_max_age", "default": 4},
         },
-        "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast"],
+        "required_or": ["solcast_api_key", "forecast_solar", "pv_forecast_today", "open_meteo_forecast", "pv_clearsky_today"],
         "phase": 2,  # Solar component moved to phase 2 so that any Predbat cloud components (such as GEcloud) have been started and initialised pv_today, etc
     },
     "gecloud": {

apps/predbat/config.py

@@ -487,6 +487,104 @@
         "default": True,
         "enable": "expert_mode",
     },
+    {
+        "name": "clipping_buffer_enable",
+        "friendly_name": "Clipping Buffer Enable",
+        "type": "switch",
+        "default": False,
+        "icon": "mdi:chart-bell-curve-cumulative",
+    },
+    {
+        "name": "clipping_clearsky_source",
+        "friendly_name": "Clipping Clear-Sky Source",
+        "type": "select",
+        "options": ["auto", "ha_solcast_clearsky", "solcast_api", "openmeteo"],
+        "icon": "mdi:cloud-search",
+        "default": "auto",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_use_clearsky_peaks",
+        "friendly_name": "Clipping Use ClearSky Peaks",
+        "type": "switch",
+        "default": True,
+        "icon": "mdi:weather-sunny-alert",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_auto_tune",
+        "friendly_name": "Clipping Auto-Tune",
+        "type": "switch",
+        "default": True,
+        "icon": "mdi:auto-fix",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_cost_weight",
+        "friendly_name": "Clipping Cost Weight",
+        "type": "input_number",
+        "min": 0,
+        "max": 10.0,
+        "step": 0.1,
+        "unit": "x",
+        "icon": "mdi:multiplication",
+        "enable": "clipping_buffer_enable",
+        "default": 1.0,
+    },
+    {
+        "name": "clipping_amplification",
+        "friendly_name": "Clipping Amplification",
+        "type": "input_number",
+        "min": 0.5,
+        "max": 3.0,
+        "step": 0.1,
+        "unit": "x",
+        "icon": "mdi:arrow-expand-vertical",
+        "enable": "clipping_buffer_enable",
+        "default": 1.0,
+    },
+    {
+        "name": "clipping_limit_override",
+        "friendly_name": "Clipping Limit Override",
+        "type": "input_number",
+        "min": 0,
+        "max": 50000,
+        "step": 100,
+        "unit": "W",
+        "icon": "mdi:flash-alert",
+        "enable": "clipping_buffer_enable",
+        "default": 0,
+    },
+    {
+        "name": "clipping_buffer_max_kwh",
+        "friendly_name": "Clipping Buffer Max kWh (Manual Override)",
+        "type": "input_number",
+        "min": 0,
+        "max": 50.0,
+        "step": 0.1,
+        "unit": "kWh",
+        "icon": "mdi:battery-50",
+        "default": 0,
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_buffer_start_time",
+        "friendly_name": "Clipping Buffer Start Time",
+        "type": "select",
+        "options": ["None"] + OPTIONS_TIME,
+        "icon": "mdi:clock-start",
+        "default": "None",
+        "enable": "clipping_buffer_enable",
+    },
+    {
+        "name": "clipping_buffer_end_time",
+        "friendly_name": "Clipping Buffer End Time",
+        "type": "select",
+        "options": ["None"] + OPTIONS_TIME,
+        "icon": "mdi:clock-end",
+        "default": "None",
+        "enable": "clipping_buffer_enable",
+    },
     {
         "name": "metric_pv_calibration_enable",
         "friendly_name": "Enable use of Calibrated PV data",
@@ -2234,4 +2332,16 @@
     "gateway_mqtt_host": {"type": "string", "empty": False},
     "gateway_mqtt_port": {"type": "integer", "zero": False},
     "gateway_mqtt_token": {"type": "string", "empty": False},
+    "clipping_buffer_enable": {"type": "boolean"},
+    "clipping_use_clearsky_peaks": {"type": "boolean"},
+    "clipping_cost_weight": {"type": "float"},
+    "clipping_amplification": {"type": "float"},
+    "clipping_limit_override": {"type": "integer"},
+    "clipping_buffer_max_kwh": {"type": "float"},
+    "clipping_buffer_start_time": {"type": "string"},
+    "clipping_buffer_end_time": {"type": "string"},
+    "pv_clearsky_today": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_tomorrow": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_d3": {"type": "sensor", "sensor_type": "float"},
+    "pv_clearsky_d4": {"type": "sensor", "sensor_type": "float"},
 }

apps/predbat/config/apps.yaml

@@ -267,7 +267,6 @@ pred_bat:
   # Only use this for workarounds if your inverter time is correct but Predbat is somehow wrong (AppDaemon issue)
   # 1 means add 1 minute to AppDaemon time, -1 takes it away
   clock_skew: 0
-
   # Solcast cloud interface, set this or the local interface below
   #solcast_host: 'https://api.solcast.com.au/'
   #solcast_api_key: !secret solcast_api_key
@@ -280,7 +279,25 @@ pred_bat:
   #  azimuth: 180
   #  declination: 10
   #  api_key: !secret forecast_solar_api_key
-  #  azimuth_zero_south: false  # Set to true if azimuth is already in Forecast.solar/Open-Meteo convention (0=South) rather than Predbat convention (0=North)
+
+  # Open-Meteo interface (used for primary PV forecast OR for ClearSky baseline)
+  #open_meteo_forecast:
+  #  - latitude: 55.9711067
+  #    longitude: -3.1925157
+  #    declination: 35
+  #    azimuth: 180
+  #    kwp: 4.0
+
+  # Determines which source Predbat uses for the primary PV forecast.
+  # Options: "auto" (default), "ha", "forecast_solar", "openmeteo", "solcast_api"
+  # If you are using Open-Meteo ONLY for the ClearSky baseline below, it's recommended to explicitly set this to "ha" if using the HA integration.
+  #pv_forecast_primary: "auto"
+
+  # --- Automated Solar Clipping Mitigation ---
+  # Enable clipping mitigation to automatically force exports prior to peaks
+  #clipping_buffer_enable: True
+  # Set the source for the theoretical ClearSky maximum (e.g. "openmeteo", "ha_solcast_clearsky", "solcast_api")
+  #clipping_clearsky_source: "ha_solcast_clearsky"
 
   # Set these to match solcast sensor names if not using the cloud interface
   # The regular expression (re:) makes the solcast bit optional
@@ -290,6 +307,12 @@ pred_bat:
   pv_forecast_d3: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_3|d3))
   pv_forecast_d4: re:(sensor.(solcast_|)(pv_forecast_|)forecast_(day_4|d4))
 
+  # ClearSky sensors (Required if clipping_clearsky_source is set to ha_solcast_clearsky)
+  #pv_clearsky_today: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_today)
+  #pv_clearsky_tomorrow: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_tomorrow)
+  #pv_clearsky_d3: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_3|d3))
+  #pv_clearsky_d4: re:(sensor.(solcast_|)(pv_forecast_|)clearsky_(day_4|d4))
+
   # Ohme EV charger cloud direct integration
   #ohme_login: !secret ohme_login
   #ohme_password: !secret ohme_password

apps/predbat/fetch.py

@@ -572,7 +572,7 @@ def minute_data_import_export(self, max_days_previous, now_utc, key, scale=1.0,
             if history and len(history) > 0 and len(history[0]) > 0:
                 item = history[0][0]
                 try:
-                    last_updated_time = str2time(item["last_updated"])
+                    last_updated_time = str2time(item.get("last_updated", item.get("state_class", "")))
                 except (ValueError, TypeError):
                     last_updated_time = now_utc
                 age_days = max_days_previous
@@ -639,7 +639,7 @@ def minute_data_load(self, now_utc, entity_name, max_days_previous, load_scaling
             if isinstance(history, list) and history and history[0]:
                 item = history[0][0]
                 try:
-                    last_updated_time = str2time(item["last_updated"])
+                    last_updated_time = str2time(item.get("last_updated", item.get("state_class", "")))
                 except (ValueError, TypeError):
                     last_updated_time = now_utc
                 age = now_utc - last_updated_time
@@ -1027,7 +1027,7 @@ def fetch_sensor_data(self, save=True):
             self.cost_today_sofar, self.carbon_today_sofar = self.today_cost(self.import_today, self.export_today, self.car_charging_energy, self.load_minutes, save=save)
 
         # Fetch PV forecast if enabled, today must be enabled, other days are optional
-        self.pv_forecast_minute, self.pv_forecast_minute10 = self.fetch_pv_forecast()
+        self.pv_forecast_minute, self.pv_forecast_minute10, self.pv_forecast_minute90, self.pv_forecast_minuteCS, self.pv_forecast_minuteMAX = self.fetch_pv_forecast()
 
         if self.load_minutes and not self.load_forecast_only:
             # Apply modal filter to historical data
@@ -1240,11 +1240,21 @@ def fetch_pv_forecast(self):
         """
         pv_forecast_minute = {}
         pv_forecast_minute10 = {}
+        pv_forecast_minute90 = {}
+        pv_forecast_minuteCS = {}
+        pv_forecast_minuteHIST = {}
 
         # Get data from forecast sensor
         entity_id = "sensor." + self.prefix + "_pv_forecast_raw"
         pv_forecast_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast")
         pv_forecast10_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast10")
+        pv_forecast90_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast90")
+        pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_clearsky")
+        pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecast_historical")
+        if pv_forecastCS_packed_ld is None:
+            pv_forecastCS_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastCS")
+        if pv_forecastHIST_packed_ld is None:
+            pv_forecastHIST_packed_ld = self.get_state_wrapper(entity_id=entity_id, attribute="forecastMAX")
         relative_time = self.get_state_wrapper(entity_id=entity_id, attribute="relative_time")
         try:
             relative_time = datetime.strptime(relative_time, TIME_FORMAT)
@@ -1255,6 +1265,9 @@ def fetch_pv_forecast(self):
         # Convert keys to integers and values to floats
         pv_forecast_packed = {}
         pv_forecast10_packed = {}
+        pv_forecast90_packed = {}
+        pv_forecastCS_packed = {}
+        pv_forecastHIST_packed = {}
 
         if pv_forecast_packed_ld:
             for key, value in pv_forecast_packed_ld.items():
@@ -1272,10 +1285,37 @@ def fetch_pv_forecast(self):
                 except (ValueError, TypeError):
                     pass
 
+        if pv_forecast90_packed_ld:
+            for key, value in pv_forecast90_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecast90_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
+        if pv_forecastCS_packed_ld:
+            for key, value in pv_forecastCS_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecastCS_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
+        if pv_forecastHIST_packed_ld:
+            for key, value in pv_forecastHIST_packed_ld.items():
+                try:
+                    minute = int(key)
+                    pv_forecastHIST_packed[minute] = float(value)
+                except (ValueError, TypeError):
+                    pass
+
         # Unpack the forecast data
         max_minute = max(pv_forecast_packed.keys()) if pv_forecast_packed else 0
         last_value = 0
         last_value10 = 0
+        last_value90 = 0
+        last_valueCS = 0
+        last_valueHIST = 0
         # The forecast could be for a different time to our relative time, so we need to offset the minutes to align with our midnight_utc.
         # relative_time is the midnight at which the forecast was saved, so stored minute keys are relative to that midnight.
         # We subtract the offset so that stored minute X (= relative_time + X) maps to (relative_time + X - midnight_utc) minutes from today's midnight.
@@ -1284,10 +1324,16 @@ def fetch_pv_forecast(self):
             target_minute = minute - minute_offset
             last_value = pv_forecast_packed.get(minute, last_value)
             last_value10 = pv_forecast10_packed.get(minute, last_value10)
+            last_value90 = pv_forecast90_packed.get(minute, last_value90)
+            last_valueCS = pv_forecastCS_packed.get(minute, last_valueCS)
+            last_valueHIST = pv_forecastHIST_packed.get(minute, last_valueHIST)
             pv_forecast_minute[target_minute] = last_value
             pv_forecast_minute10[target_minute] = last_value10
+            pv_forecast_minute90[target_minute] = last_value90
+            pv_forecast_minuteCS[target_minute] = last_valueCS
+            pv_forecast_minuteHIST[target_minute] = last_valueHIST
 
-        return pv_forecast_minute, pv_forecast_minute10
+        return pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_minuteCS, pv_forecast_minuteHIST
 
     def predict_battery_temperature(self, battery_temperature_history, step):
         """
@@ -2317,6 +2363,14 @@ def fetch_config_options(self):
         self.carbon_enable = self.get_arg("carbon_enable")
         self.carbon_metric = self.get_arg("carbon_metric")
 
+        # Clipping peak cost penalty model
+        self.clipping_buffer_enable = self.get_arg("clipping_buffer_enable")
+        self.clipping_use_clearsky_peaks = self.get_arg("clipping_use_clearsky_peaks")
+        self.clipping_auto_tune = self.get_arg("clipping_auto_tune")
+        self.clipping_cost_weight = self.get_arg("clipping_cost_weight")
+        self.clipping_amplification = self.get_arg("clipping_amplification")
+        self.clipping_limit_override = self.get_arg("clipping_limit_override") / MINUTE_WATT if self.get_arg("clipping_limit_override") else 0
+
         # iBoost solar diverter model
         self.iboost_enable = self.get_arg("iboost_enable")
         self.iboost_gas = self.get_arg("iboost_gas")

apps/predbat/load_ml_component.py

@@ -405,7 +405,7 @@ async def _fetch_load_data(self):
                 energy = self.get_from_incrementing(pv_data_cumulative, m, PREDICT_STEP, backwards=True)
                 pv_data[m] = dp4(energy)
 
-            pv_forecast_minute, pv_forecast_minute10 = self.base.fetch_pv_forecast()
+            pv_forecast_minute, pv_forecast_minute10, pv_forecast_minute90, pv_forecast_minuteCS, pv_forecast_minuteHIST = self.base.fetch_pv_forecast()
             # Add future PV forecast as per-5-min energy with negative keys (negative = future)
             # key -5 = first future step, -10 = second, etc.
             if pv_forecast_minute:

apps/predbat/output.py

@@ -939,6 +939,35 @@ def short_textual_plan(self, soc_min, soc_min_minute, pv_forecast_minute_step, p
                 export_type, self.duration_string(self.export_window_best[export_window_n_next]["start"] - self.minutes_now), self.get_rate_text(self.export_window_best[export_window_n_next]["start"], export=True, with_value=True)
             )
 
+        # Clipping summary
+        if getattr(self, "clipping_buffer_enable", False):
+            clipping_status = getattr(self, "clipping_status", "No clipping forecast.")
+            sentence += "- Clipping status: {}\n".format(clipping_status)
+
+        predict_clipped_best = getattr(self, "predict_clipped_best", {})
+        if predict_clipped_best:
+            clipping_total = predict_clipped_best.get(max(predict_clipped_best.keys()), 0.0)
+            if clipping_total > 0.01:
+                clipping_mode = getattr(self, "clipping_limit_mode", "Unknown")
+                start_str = ""
+                end_str = ""
+                start_stamp = None
+                end_stamp = None
+
+                prev_val = 0.0
+                for min_key, val in sorted(predict_clipped_best.items()):
+                    if val > prev_val + 0.001:
+                        if start_stamp is None:
+                            start_stamp = self.midnight_utc + timedelta(minutes=min_key)
+                        end_stamp = self.midnight_utc + timedelta(minutes=min_key)
+                    prev_val = val
+
+                if start_stamp and end_stamp:
+                    start_str = start_stamp.strftime("%H:%M")
+                    end_str = end_stamp.strftime("%H:%M")
+
+                sentence += "- Forecast {} kWh clipping, exceeding {} limit from {} to {}. Plan penalized to mitigate.\n".format(dp2(clipping_total), clipping_mode, start_str, end_str)
+
         if publish:
             self.text_plan = self.get_text_plan_html(sentence)
 
@@ -2439,6 +2468,9 @@ def record_status(self, message, debug="", had_errors=False, notify=False, extra
             },
         )
 
+        # Clipping Status and PV Peak Forecast sensors are published in plan.py run_prediction(save="best")
+        # to avoid duplicate dashboard_item writes to the same sensor.
+
         if had_errors:
             self.log("Warn: record_status {}".format(message + extra))
         else: