Improve error messages for insufficient future_covariates in backtest() and fix residuals()

CHANGELOG.md

@@ -14,6 +14,7 @@ but cannot always guarantee backwards compatibility. Changes that may **break co
 - 🔴 Improved the performance of the `TimeSeries.map()` method for functions that take two arguments. The mapping is now applied on the entire time index and values array which requires users to reshape the time index explicitly within the function. See more information in the `TimeSeries.map()` method documentation. [#2911](https://github.com/unit8co/darts/pull/2911) by [Jakub Chłapek](https://github.com/jakubchlapek)
 
 **Fixed**
+- `backtest()` now provides clear, actionable error messages when `future_covariates` don't extend far enough or start too late, replacing cryptic `IndexError` and `TypeError` with detailed `ValueError` messages that include timestamps, missing steps, and code examples. Also fixed bug in `residuals()` method where `past_covariates` and `future_covariates` weren't being forwarded to `backtest()`. [#2846](https://github.com/unit8co/darts/issues/2846) by [Aditya Mehra](https://github.com/addym).
 
 **Dependencies**
 

darts/models/forecasting/forecasting_model.py

@@ -1254,6 +1254,100 @@ def retrain_func(
             forecasts = forecasts_list
         return series2seq(forecasts, seq_type_out=sequence_type_in)
 
+    def _validate_future_covariates_for_forecast(
+            self,
+            series: Union[TimeSeries, Sequence[TimeSeries]],
+            future_covariates: Optional[Union[TimeSeries, Sequence[TimeSeries]]],
+            forecast_horizon: int,
+            method_name: str = "forecast",
+    ) -> None:
+        """
+        Validate that future_covariates have proper time coverage.
+
+        This validation detects when covariates are provided but don't extend far enough,
+        which would otherwise cause cryptic IndexError or TypeError. If covariates are None,
+        the model's own validation will handle it.
+
+        Parameters
+        ----------
+        series
+            The target series or sequence of series
+        future_covariates
+            The future covariates to validate (can be None)
+        forecast_horizon
+            Number of time steps to forecast
+        method_name
+            Name of calling method (for error messages)
+
+        Raises
+        ------
+        ValueError
+            If future_covariates don't start early enough or don't extend far enough
+        """
+        # Only validate if model uses future covariates
+        if not (hasattr(self, 'uses_future_covariates') and self.uses_future_covariates):
+            return
+
+        # Only validate coverage if covariates are provided
+        # If None, let the model's own validation handle it
+        if future_covariates is None:
+            return
+
+        # Validate time coverage
+        from darts import TimeSeries
+
+        model_name = self.__class__.__name__
+
+        # Handle both single series and sequences
+        series_seq = series if isinstance(series, (list, tuple)) else [series]
+        cov_seq = future_covariates if isinstance(future_covariates, (list, tuple)) else [future_covariates]
+
+        for idx, (ts, fc) in enumerate(zip(series_seq, cov_seq)):
+            series_label = f" (series {idx})" if len(series_seq) > 1 else ""
+
+            # Check 1: Covariates must START at or before series start
+            if fc.start_time() > ts.start_time():
+                raise ValueError(
+                    f"{model_name} requires future_covariates to cover the entire series time range{series_label}.\n\n"
+                    f"Problem:\n"
+                    f"  - Series starts at: {ts.start_time()}\n"
+                    f"  - Future covariates start at: {fc.start_time()}\n"
+                    f"  - Gap: Covariates start {(fc.start_time() - ts.start_time()) // ts.freq} time steps AFTER series start\n\n"
+                    f"Solution: Ensure future_covariates begin at or before the series start time.\n\n"
+                    f"Example:\n"
+                    f"    model.{method_name}(\n"
+                    f"        series=my_series,\n"
+                    f"        future_covariates=my_covariates,  # Must cover from series start\n"
+                    f"        forecast_horizon={forecast_horizon}\n"
+                    f"    )"
+                )
+
+            # Check 2: Covariates must END after series end + forecast_horizon
+            required_end = ts.end_time() + ts.freq * forecast_horizon
+            actual_end = fc.end_time()
+
+            if actual_end < required_end:
+                missing_steps = int((required_end - actual_end) / ts.freq)
+
+                raise ValueError(
+                    f"{model_name} requires future_covariates to extend beyond the series end{series_label}.\n\n"
+                    f"Problem:\n"
+                    f"  - Series ends at: {ts.end_time()}\n"
+                    f"  - Future covariates end at: {actual_end}\n"
+                    f"  - Required: {required_end} (series end + {forecast_horizon} steps)\n"
+                    f"  - Missing: {missing_steps} time steps\n\n"
+                    f"Solution: Extend future_covariates to cover the full series + {forecast_horizon} steps.\n\n"
+                    f"Example:\n"
+                    f"    cov_dates = pd.date_range(\n"
+                    f"        start=series.start_time(),\n"
+                    f"        periods=len(series) + {forecast_horizon},\n"
+                    f"        freq=series.freq\n"
+                    f"    )\n"
+                    f"    future_covariates = TimeSeries.from_times_and_values(\n"
+                    f"        times=cov_dates, values=your_data\n"
+                    f"    )\n"
+                    f"    model.{method_name}(series, future_covariates, forecast_horizon={forecast_horizon})"
+                )
     def backtest(
         self,
         series: Union[TimeSeries, Sequence[TimeSeries]],
@@ -1456,7 +1550,6 @@ def backtest(
             computed per time `series`.
         random_state
             Controls the randomness of probabilistic predictions.
-
         Returns
         -------
         float
@@ -1481,6 +1574,12 @@ def backtest(
             Same as for type `np.ndarray` but for a sequence of series. The returned metric list has length
             `len(series)` with the `np.ndarray` metrics for each input `series`.
         """
+        self._validate_future_covariates_for_forecast(
+            series,
+            future_covariates,
+            forecast_horizon,
+            "backtest"
+        )
         metric_kwargs = metric_kwargs or dict()
         if not isinstance(metric_kwargs, list):
             metric_kwargs = [metric_kwargs]
@@ -2278,6 +2377,8 @@ def residuals(
 
         residuals = self.backtest(
             series=series,
+            past_covariates=past_covariates,
+            future_covariates=future_covariates,
             historical_forecasts=historical_forecasts,
             last_points_only=False,
             metric=metric,

darts/tests/models/forecasting/test_backtest_error_messages.py

@@ -0,0 +1,251 @@
+"""Tests for improved error messages in backtest() and historical_forecasts()."""
+
+import pytest
+import pandas as pd
+import numpy as np
+from darts import TimeSeries
+from darts.models import TFTModel, NBEATSModel
+from darts.metrics import mape
+
+
+class TestBacktestErrorMessages:
+    """Test that backtest() and historical_forecasts() provide helpful error messages."""
+
+    @pytest.fixture
+    def sample_data(self):
+        """Create sample time series data for testing."""
+        dates = pd.date_range('2020-01-01', periods=100, freq='D')
+        np.random.seed(42)
+        target_data = np.sin(np.linspace(0, 10, 100)) + np.random.normal(0, 0.1, 100)
+        future_cov_data = np.cos(np.linspace(0, 10, 100)) + np.random.normal(0, 0.1, 100)
+
+        series = TimeSeries.from_times_and_values(dates, target_data.astype(np.float32))
+        future_covariates = TimeSeries.from_times_and_values(
+            dates,
+            future_cov_data.astype(np.float32)
+        )
+
+        return series, future_covariates
+
+    def test_backtest_missing_future_covariates_error(self, sample_data):
+        """
+        Test that calling backtest() without future_covariates on a model
+        that requires them produces an error message.
+
+        Note: We only validate coverage (too short/late start), not None.
+        When covariates are None, the model's own validation handles it.
+        This test verifies that SOME error is raised (could be from model or our validation).
+        """
+        series, future_covariates = sample_data
+
+        # Create and train model that requires future covariates
+        model = TFTModel(
+            input_chunk_length=12,
+            output_chunk_length=6,
+            n_epochs=1,
+            random_state=42,
+            pl_trainer_kwargs={
+                "accelerator": "cpu",
+                "enable_progress_bar": False,
+                "enable_model_summary": False,
+            }
+        )
+
+        # Train with future covariates
+        model.fit(series[:70], future_covariates=future_covariates[:70])
+
+        # Test that calling backtest without future_covariates raises an error
+        with pytest.raises((ValueError, RuntimeError)) as exc_info:
+            model.backtest(
+                series=series,
+                # Intentionally NOT providing future_covariates
+                forecast_horizon=6,
+                metric=mape
+            )
+
+        error_message = str(exc_info.value)
+
+        # Verify error mentions future covariates (could be from model or our validation)
+        assert ("future" in error_message.lower() and "covariate" in error_message.lower()) or \
+               "future_covariates" in error_message.lower(), \
+            "Error should mention future covariates"
+
+        # The error can come from either:
+        # 1. The model's own validation (mentions "future covariates", "encoders")
+        # 2. Our validation (mentions "extend", "missing steps") if covariates were partially provided
+        assert any(keyword in error_message.lower() for keyword in
+                   ["covariate", "encoder", "extend", "missing"]), \
+            "Error should provide actionable information"
+    def test_backtest_future_covariates_too_short(self, sample_data):
+        """
+        Test that backtest() with future_covariates that don't extend far enough
+        produces a clear error message.
+
+        Addresses issue #2846 - Scenario 2: IndexError when covariates are too short.
+        """
+        series, future_covariates = sample_data
+
+        # Create covariates that are too short
+        future_covariates_short = future_covariates[:75]  # Only 75 days, need 100 + 6
+
+        model = TFTModel(
+            input_chunk_length=12,
+            output_chunk_length=6,
+            n_epochs=1,
+            random_state=42,
+            pl_trainer_kwargs={
+                "accelerator": "cpu",
+                "enable_progress_bar": False,
+                "enable_model_summary": False,
+            }
+        )
+
+        model.fit(series[:70], future_covariates=future_covariates_short[:70])
+
+        # Should raise helpful error instead of IndexError
+        with pytest.raises(ValueError) as exc_info:
+            model.backtest(
+                series=series,
+                future_covariates=future_covariates_short,
+                forecast_horizon=6,
+                metric=mape
+            )
+
+        error_message = str(exc_info.value)
+
+        # Verify error message is helpful
+        assert "extend" in error_message.lower() or "missing" in error_message.lower(), \
+            "Error should mention that covariates don't extend far enough"
+
+        assert "Series ends at" in error_message, \
+            "Error should show series end time"
+
+        assert "Future covariates end at" in error_message, \
+            "Error should show covariates end time"
+
+        assert "31" in error_message or "missing" in error_message.lower(), \
+            "Error should indicate how many steps are missing"
+
+    def test_backtest_future_covariates_start_too_late(self, sample_data):
+        """
+        Test that backtest() with future_covariates that start after series start
+        produces a clear error message.
+
+        Addresses issue #2846 - Scenario 3: TypeError when covariates start too late.
+        """
+        series, _ = sample_data
+
+        # Create covariates that start AFTER series starts
+        cov_dates = pd.date_range('2020-03-01', periods=80, freq='D')  # Starts March 1
+        cov_data = np.random.randn(80).astype(np.float32)
+        future_covariates_late = TimeSeries.from_times_and_values(cov_dates, cov_data)
+
+        model = TFTModel(
+            input_chunk_length=12,
+            output_chunk_length=6,
+            n_epochs=1,
+            random_state=42,
+            pl_trainer_kwargs={
+                "accelerator": "cpu",
+                "enable_progress_bar": False,
+                "enable_model_summary": False,
+            }
+        )
+
+        # Train with shorter series that fits within covariates
+        model.fit(series[60:80], future_covariates=future_covariates_late[:20])
+
+        # Should raise helpful error when series starts before covariates
+        with pytest.raises(ValueError) as exc_info:
+            model.backtest(
+                series=series,  # Starts Jan 1
+                future_covariates=future_covariates_late,  # Starts March 1
+                forecast_horizon=6,
+                metric=mape
+            )
+
+        error_message = str(exc_info.value)
+
+        # Verify error message is helpful
+        assert "Series starts at" in error_message, \
+            "Error should show series start time"
+
+        assert "Future covariates start at" in error_message, \
+            "Error should show covariates start time"
+
+        assert "gap" in error_message.lower() or "after" in error_message.lower(), \
+            "Error should mention the gap/timing issue"
+
+    def test_model_without_future_covariates_works_normally(self):
+        """
+        Test that models not requiring future_covariates work normally.
+
+        Ensures our validation doesn't break models that don't use future covariates.
+        """
+        dates = pd.date_range('2020-01-01', periods=100, freq='D')
+        np.random.seed(42)
+        target_data = np.random.randn(100).astype(np.float32)
+        series = TimeSeries.from_times_and_values(dates, target_data)
+
+        # NBEATSModel doesn't require future covariates
+        model = NBEATSModel(
+            input_chunk_length=12,
+            output_chunk_length=6,
+            n_epochs=1,
+            random_state=42,
+            pl_trainer_kwargs={
+                "accelerator": "cpu",
+                "enable_progress_bar": False,
+                "enable_model_summary": False,
+            }
+        )
+
+        model.fit(series[:70])
+
+        # Should work fine without future_covariates
+        result = model.backtest(
+            series=series,
+            forecast_horizon=6,
+            metric=mape
+        )
+
+        assert result is not None, "Backtest should succeed for models without future covariates"
+
+    def test_exact_boundary_case(self, sample_data):
+        """
+        Test that covariates extending exactly to required length work correctly.
+
+        Edge case: covariates are exactly long enough (not too short, not extra long).
+        """
+        series, _ = sample_data
+
+        forecast_horizon = 6
+        # Create covariates that are exactly the right length
+        exact_length = len(series) + forecast_horizon
+        cov_dates = pd.date_range('2020-01-01', periods=exact_length, freq='D')
+        cov_data = np.random.randn(exact_length).astype(np.float32)
+        future_covariates_exact = TimeSeries.from_times_and_values(cov_dates, cov_data)
+
+        model = TFTModel(
+            input_chunk_length=12,
+            output_chunk_length=6,
+            n_epochs=1,
+            random_state=42,
+            pl_trainer_kwargs={
+                "accelerator": "cpu",
+                "enable_progress_bar": False,
+                "enable_model_summary": False,
+            }
+        )
+
+        model.fit(series[:70], future_covariates=future_covariates_exact[:70])
+
+        # Should work - covariates are exactly long enough
+        result = model.backtest(
+            series=series,
+            future_covariates=future_covariates_exact,
+            forecast_horizon=forecast_horizon,
+            metric=mape
+        )
+
+        assert result is not None, "Backtest should succeed when covariates are exactly long enough"