From 8859b83425711835893cbbd46db9e5ce65dff6df Mon Sep 17 00:00:00 2001 From: Aditya Mehra Date: Fri, 10 Oct 2025 09:02:07 -0400 Subject: [PATCH 1/2] Improve error messages for insufficient future_covariates in backtest() and fix residuals() bug 1. Add validation for future_covariates coverage in backtest() - Clear error messages when covariates start too late or don't extend far enough - Replaces cryptic IndexError/TypeError with actionable ValueError - Only validates when covariates are provided; model handles None case - Validation only in backtest() to preserve flexibility in historical_forecasts() 2. Fix bug in residuals() method - residuals() now properly passes past_covariates and future_covariates to backtest() - Previously these parameters were accepted but not forwarded 3. Add comprehensive test suite - 5 tests covering insufficient covariate scenarios in backtest() - Tests coverage validation (too short, starts too late) - Validates no regression for models without covariates - Tests exact boundary cases Fixes #2846 --- darts/models/forecasting/forecasting_model.py | 103 ++++++- .../test_backtest_error_messages.py | 251 ++++++++++++++++++ 2 files changed, 353 insertions(+), 1 deletion(-) create mode 100644 darts/tests/models/forecasting/test_backtest_error_messages.py diff --git a/darts/models/forecasting/forecasting_model.py b/darts/models/forecasting/forecasting_model.py index f3d837405c..a5271f10ed 100644 --- a/darts/models/forecasting/forecasting_model.py +++ b/darts/models/forecasting/forecasting_model.py @@ -1254,6 +1254,100 @@ def retrain_func( forecasts = forecasts_list return series2seq(forecasts, seq_type_out=sequence_type_in) + def _validate_future_covariates_for_forecast( + self, + series: Union[TimeSeries, Sequence[TimeSeries]], + future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]], + forecast_horizon: int, + method_name: str = "forecast", + ) -> None: + """ + Validate that future_covariates have proper time coverage. + + This validation detects when covariates are provided but don't extend far enough, + which would otherwise cause cryptic IndexError or TypeError. If covariates are None, + the model's own validation will handle it. + + Parameters + ---------- + series + The target series or sequence of series + future_covariates + The future covariates to validate (can be None) + forecast_horizon + Number of time steps to forecast + method_name + Name of calling method (for error messages) + + Raises + ------ + ValueError + If future_covariates don't start early enough or don't extend far enough + """ + # Only validate if model uses future covariates + if not (hasattr(self, 'uses_future_covariates') and self.uses_future_covariates): + return + + # Only validate coverage if covariates are provided + # If None, let the model's own validation handle it + if future_covariates is None: + return + + # Validate time coverage + from darts import TimeSeries + + model_name = self.__class__.__name__ + + # Handle both single series and sequences + series_seq = series if isinstance(series, (list, tuple)) else [series] + cov_seq = future_covariates if isinstance(future_covariates, (list, tuple)) else [future_covariates] + + for idx, (ts, fc) in enumerate(zip(series_seq, cov_seq)): + series_label = f" (series {idx})" if len(series_seq) > 1 else "" + + # Check 1: Covariates must START at or before series start + if fc.start_time() > ts.start_time(): + raise ValueError( + f"{model_name} requires future_covariates to cover the entire series time range{series_label}.\n\n" + f"Problem:\n" + f" - Series starts at: {ts.start_time()}\n" + f" - Future covariates start at: {fc.start_time()}\n" + f" - Gap: Covariates start {(fc.start_time() - ts.start_time()) // ts.freq} time steps AFTER series start\n\n" + f"Solution: Ensure future_covariates begin at or before the series start time.\n\n" + f"Example:\n" + f" model.{method_name}(\n" + f" series=my_series,\n" + f" future_covariates=my_covariates, # Must cover from series start\n" + f" forecast_horizon={forecast_horizon}\n" + f" )" + ) + + # Check 2: Covariates must END after series end + forecast_horizon + required_end = ts.end_time() + ts.freq * forecast_horizon + actual_end = fc.end_time() + + if actual_end < required_end: + missing_steps = int((required_end - actual_end) / ts.freq) + + raise ValueError( + f"{model_name} requires future_covariates to extend beyond the series end{series_label}.\n\n" + f"Problem:\n" + f" - Series ends at: {ts.end_time()}\n" + f" - Future covariates end at: {actual_end}\n" + f" - Required: {required_end} (series end + {forecast_horizon} steps)\n" + f" - Missing: {missing_steps} time steps\n\n" + f"Solution: Extend future_covariates to cover the full series + {forecast_horizon} steps.\n\n" + f"Example:\n" + f" cov_dates = pd.date_range(\n" + f" start=series.start_time(),\n" + f" periods=len(series) + {forecast_horizon},\n" + f" freq=series.freq\n" + f" )\n" + f" future_covariates = TimeSeries.from_times_and_values(\n" + f" times=cov_dates, values=your_data\n" + f" )\n" + f" model.{method_name}(series, future_covariates, forecast_horizon={forecast_horizon})" + ) def backtest( self, series: Union[TimeSeries, Sequence[TimeSeries]], @@ -1456,7 +1550,6 @@ def backtest( computed per time `series`. random_state Controls the randomness of probabilistic predictions. - Returns ------- float @@ -1481,6 +1574,12 @@ def backtest( Same as for type `np.ndarray` but for a sequence of series. The returned metric list has length `len(series)` with the `np.ndarray` metrics for each input `series`. """ + self._validate_future_covariates_for_forecast( + series, + future_covariates, + forecast_horizon, + "backtest" + ) metric_kwargs = metric_kwargs or dict() if not isinstance(metric_kwargs, list): metric_kwargs = [metric_kwargs] @@ -2278,6 +2377,8 @@ def residuals( residuals = self.backtest( series=series, + past_covariates=past_covariates, + future_covariates=future_covariates, historical_forecasts=historical_forecasts, last_points_only=False, metric=metric, diff --git a/darts/tests/models/forecasting/test_backtest_error_messages.py b/darts/tests/models/forecasting/test_backtest_error_messages.py new file mode 100644 index 0000000000..f73997914d --- /dev/null +++ b/darts/tests/models/forecasting/test_backtest_error_messages.py @@ -0,0 +1,251 @@ +"""Tests for improved error messages in backtest() and historical_forecasts().""" + +import pytest +import pandas as pd +import numpy as np +from darts import TimeSeries +from darts.models import TFTModel, NBEATSModel +from darts.metrics import mape + + +class TestBacktestErrorMessages: + """Test that backtest() and historical_forecasts() provide helpful error messages.""" + + @pytest.fixture + def sample_data(self): + """Create sample time series data for testing.""" + dates = pd.date_range('2020-01-01', periods=100, freq='D') + np.random.seed(42) + target_data = np.sin(np.linspace(0, 10, 100)) + np.random.normal(0, 0.1, 100) + future_cov_data = np.cos(np.linspace(0, 10, 100)) + np.random.normal(0, 0.1, 100) + + series = TimeSeries.from_times_and_values(dates, target_data.astype(np.float32)) + future_covariates = TimeSeries.from_times_and_values( + dates, + future_cov_data.astype(np.float32) + ) + + return series, future_covariates + + def test_backtest_missing_future_covariates_error(self, sample_data): + """ + Test that calling backtest() without future_covariates on a model + that requires them produces an error message. + + Note: We only validate coverage (too short/late start), not None. + When covariates are None, the model's own validation handles it. + This test verifies that SOME error is raised (could be from model or our validation). + """ + series, future_covariates = sample_data + + # Create and train model that requires future covariates + model = TFTModel( + input_chunk_length=12, + output_chunk_length=6, + n_epochs=1, + random_state=42, + pl_trainer_kwargs={ + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + } + ) + + # Train with future covariates + model.fit(series[:70], future_covariates=future_covariates[:70]) + + # Test that calling backtest without future_covariates raises an error + with pytest.raises((ValueError, RuntimeError)) as exc_info: + model.backtest( + series=series, + # Intentionally NOT providing future_covariates + forecast_horizon=6, + metric=mape + ) + + error_message = str(exc_info.value) + + # Verify error mentions future covariates (could be from model or our validation) + assert ("future" in error_message.lower() and "covariate" in error_message.lower()) or \ + "future_covariates" in error_message.lower(), \ + "Error should mention future covariates" + + # The error can come from either: + # 1. The model's own validation (mentions "future covariates", "encoders") + # 2. Our validation (mentions "extend", "missing steps") if covariates were partially provided + assert any(keyword in error_message.lower() for keyword in + ["covariate", "encoder", "extend", "missing"]), \ + "Error should provide actionable information" + def test_backtest_future_covariates_too_short(self, sample_data): + """ + Test that backtest() with future_covariates that don't extend far enough + produces a clear error message. + + Addresses issue #2846 - Scenario 2: IndexError when covariates are too short. + """ + series, future_covariates = sample_data + + # Create covariates that are too short + future_covariates_short = future_covariates[:75] # Only 75 days, need 100 + 6 + + model = TFTModel( + input_chunk_length=12, + output_chunk_length=6, + n_epochs=1, + random_state=42, + pl_trainer_kwargs={ + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + } + ) + + model.fit(series[:70], future_covariates=future_covariates_short[:70]) + + # Should raise helpful error instead of IndexError + with pytest.raises(ValueError) as exc_info: + model.backtest( + series=series, + future_covariates=future_covariates_short, + forecast_horizon=6, + metric=mape + ) + + error_message = str(exc_info.value) + + # Verify error message is helpful + assert "extend" in error_message.lower() or "missing" in error_message.lower(), \ + "Error should mention that covariates don't extend far enough" + + assert "Series ends at" in error_message, \ + "Error should show series end time" + + assert "Future covariates end at" in error_message, \ + "Error should show covariates end time" + + assert "31" in error_message or "missing" in error_message.lower(), \ + "Error should indicate how many steps are missing" + + def test_backtest_future_covariates_start_too_late(self, sample_data): + """ + Test that backtest() with future_covariates that start after series start + produces a clear error message. + + Addresses issue #2846 - Scenario 3: TypeError when covariates start too late. + """ + series, _ = sample_data + + # Create covariates that start AFTER series starts + cov_dates = pd.date_range('2020-03-01', periods=80, freq='D') # Starts March 1 + cov_data = np.random.randn(80).astype(np.float32) + future_covariates_late = TimeSeries.from_times_and_values(cov_dates, cov_data) + + model = TFTModel( + input_chunk_length=12, + output_chunk_length=6, + n_epochs=1, + random_state=42, + pl_trainer_kwargs={ + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + } + ) + + # Train with shorter series that fits within covariates + model.fit(series[60:80], future_covariates=future_covariates_late[:20]) + + # Should raise helpful error when series starts before covariates + with pytest.raises(ValueError) as exc_info: + model.backtest( + series=series, # Starts Jan 1 + future_covariates=future_covariates_late, # Starts March 1 + forecast_horizon=6, + metric=mape + ) + + error_message = str(exc_info.value) + + # Verify error message is helpful + assert "Series starts at" in error_message, \ + "Error should show series start time" + + assert "Future covariates start at" in error_message, \ + "Error should show covariates start time" + + assert "gap" in error_message.lower() or "after" in error_message.lower(), \ + "Error should mention the gap/timing issue" + + def test_model_without_future_covariates_works_normally(self): + """ + Test that models not requiring future_covariates work normally. + + Ensures our validation doesn't break models that don't use future covariates. + """ + dates = pd.date_range('2020-01-01', periods=100, freq='D') + np.random.seed(42) + target_data = np.random.randn(100).astype(np.float32) + series = TimeSeries.from_times_and_values(dates, target_data) + + # NBEATSModel doesn't require future covariates + model = NBEATSModel( + input_chunk_length=12, + output_chunk_length=6, + n_epochs=1, + random_state=42, + pl_trainer_kwargs={ + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + } + ) + + model.fit(series[:70]) + + # Should work fine without future_covariates + result = model.backtest( + series=series, + forecast_horizon=6, + metric=mape + ) + + assert result is not None, "Backtest should succeed for models without future covariates" + + def test_exact_boundary_case(self, sample_data): + """ + Test that covariates extending exactly to required length work correctly. + + Edge case: covariates are exactly long enough (not too short, not extra long). + """ + series, _ = sample_data + + forecast_horizon = 6 + # Create covariates that are exactly the right length + exact_length = len(series) + forecast_horizon + cov_dates = pd.date_range('2020-01-01', periods=exact_length, freq='D') + cov_data = np.random.randn(exact_length).astype(np.float32) + future_covariates_exact = TimeSeries.from_times_and_values(cov_dates, cov_data) + + model = TFTModel( + input_chunk_length=12, + output_chunk_length=6, + n_epochs=1, + random_state=42, + pl_trainer_kwargs={ + "accelerator": "cpu", + "enable_progress_bar": False, + "enable_model_summary": False, + } + ) + + model.fit(series[:70], future_covariates=future_covariates_exact[:70]) + + # Should work - covariates are exactly long enough + result = model.backtest( + series=series, + future_covariates=future_covariates_exact, + forecast_horizon=forecast_horizon, + metric=mape + ) + + assert result is not None, "Backtest should succeed when covariates are exactly long enough" \ No newline at end of file From 212cd836f6b5167b20bd9625276b727135c88143 Mon Sep 17 00:00:00 2001 From: Aditya Mehra Date: Fri, 10 Oct 2025 09:10:38 -0400 Subject: [PATCH 2/2] Add CHANGELOG entry for #2846 --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fa62cb4c84..36729164d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ but cannot always guarantee backwards compatibility. Changes that may **break co **Improved** **Fixed** +- `backtest()` now provides clear, actionable error messages when `future_covariates` don't extend far enough or start too late, replacing cryptic `IndexError` and `TypeError` with detailed `ValueError` messages that include timestamps, missing steps, and code examples. Also fixed bug in `residuals()` method where `past_covariates` and `future_covariates` weren't being forwarded to `backtest()`. [#2846](https://github.com/unit8co/darts/issues/2846) by [Aditya Mehra](https://github.com/addym). **Dependencies**