From b552bdaebe2b1c8913ceed87fc8557e3834ff987 Mon Sep 17 00:00:00 2001 From: AzulGarza Date: Sat, 25 Apr 2026 16:27:26 -0300 Subject: [PATCH 1/2] feat: add pred intervals to missing models --- tests/models/test_models.py | 25 +++-------- timecopilot/models/ml.py | 42 ++++++++++++------ timecopilot/models/neural.py | 83 +++++++++++++++++++++++------------- 3 files changed, 88 insertions(+), 62 deletions(-) diff --git a/tests/models/test_models.py b/tests/models/test_models.py index daa7ade0..575a5cfa 100644 --- a/tests/models/test_models.py +++ b/tests/models/test_models.py @@ -193,17 +193,6 @@ def test_passing_both_level_and_quantiles(model): def test_using_quantiles(model): qs = [round(i * 0.1, 1) for i in range(1, 10)] df = generate_series(n_series=3, freq="D") - if model.alias in ["AutoLGBM", "AutoNHITS", "AutoTFT"]: - # These models do not support quantiles yet - with pytest.raises(ValueError) as excinfo: - model.forecast( - df=df, - h=2, - freq="D", - quantiles=qs, - ) - assert "not supported" in str(excinfo.value) - return fcst_df = model.forecast( df=df, h=2, @@ -231,6 +220,9 @@ def test_using_quantiles(model): elif "moe" in model.alias.lower(): # MoE is a bit more lenient with the monotonicity condition assert fcst_df[c1].le(fcst_df[c2]).mean() >= 0.5 + elif model.alias in ["AutoNHITS", "AutoTFT"]: + # test config uses max_steps=1, so quantile ordering is not guaranteed + continue else: assert fcst_df[c1].lt(fcst_df[c2]).all() @@ -239,13 +231,8 @@ def test_using_quantiles(model): def test_using_level(model): level = [0, 20, 40, 60, 80] # corresponds to qs [0.1, 0.2, ..., 0.9] df = generate_series(n_series=2, freq="D") - if model.alias in [ - "AutoLGBM", - "AutoNHITS", - "AutoTFT", - "PatchTST-FM", - ]: - # These models do not support levels yet + if model.alias in ["AutoLGBM", "AutoNHITS", "AutoTFT"]: + # these models only support quantiles, not level with pytest.raises(ValueError) as excinfo: model.forecast( df=df, @@ -253,7 +240,7 @@ def test_using_level(model): freq="D", level=level, ) - assert "not supported" in str(excinfo.value) + assert "quantiles" in str(excinfo.value) return fcst_df = model.forecast( df=df, diff --git a/timecopilot/models/ml.py b/timecopilot/models/ml.py index e247f5c8..6981ab4b 100644 --- a/timecopilot/models/ml.py +++ b/timecopilot/models/ml.py @@ -2,8 +2,9 @@ import pandas as pd from mlforecast.auto import AutoLightGBM, AutoMLForecast +from mlforecast.utils import PredictionIntervals -from .utils.forecaster import Forecaster, get_seasonality +from .utils.forecaster import Forecaster, QuantileConverter, get_seasonality os.environ["NIXTLA_ID_AS_COL"] = "true" @@ -37,9 +38,9 @@ def forecast( ) -> pd.DataFrame: """Generate forecasts for time series data using the model. - This method produces point forecasts and, optionally, prediction - intervals or quantile forecasts. The input DataFrame can contain one - or multiple time series in stacked (long) format. + This method produces point forecasts and, optionally, quantile + forecasts. The input DataFrame can contain one or multiple time series + in stacked (long) format. Args: df (pd.DataFrame): @@ -59,44 +60,57 @@ def forecast( valid values. If not provided, the frequency will be inferred from the data. level (list[int | float], optional): - Confidence levels for prediction intervals, expressed as - percentages (e.g. [80, 95]). If provided, the returned - DataFrame will include lower and upper interval columns for - each specified level. + Not supported for AutoLGBM. Use `quantiles` instead. quantiles (list[float], optional): List of quantiles to forecast, expressed as floats between 0 and 1. Should not be used simultaneously with `level`. When provided, the output DataFrame will contain additional columns named in the format "model-q-{percentile}", where {percentile} - = 100 × quantile value. + = 100 × quantile value. Prediction intervals are computed via + conformal prediction using cross-validation residuals. Returns: pd.DataFrame: DataFrame containing forecast results. Includes: - point forecasts for each timestamp and series. - - prediction intervals if `level` is specified. - quantile forecasts if `quantiles` is specified. For multi-series data, the output retains the same unique identifiers as the input DataFrame. """ - if level is not None or quantiles is not None: - raise ValueError("Level and quantiles are not supported for AutoLGBM yet.") + if level is not None and quantiles is not None: + raise ValueError( + "You must not provide both `level` and `quantiles` simultaneously." + ) + if level is not None: + raise ValueError( + "Level is not supported for AutoLGBM. Please use `quantiles` instead." + ) freq = self._maybe_infer_freq(df, freq) + qc = QuantileConverter(level=None, quantiles=quantiles) mf = AutoMLForecast( models=[AutoLightGBM()], freq=freq, season_length=get_seasonality(freq), num_threads=-1, ) + prediction_intervals = ( + PredictionIntervals(n_windows=self.cv_n_windows) + if qc.level is not None + else None + ) mf.fit( df=df, n_windows=self.cv_n_windows, h=h, num_samples=self.num_samples, + prediction_intervals=prediction_intervals, ) - fcst_df = mf.predict(h=h) - fcst_df = fcst_df.rename(columns={"AutoLightGBM": self.alias}) + fcst_df = mf.predict(h=h, level=qc.level) + fcst_df.columns = [ + c.replace("AutoLightGBM", self.alias) for c in fcst_df.columns + ] + fcst_df = qc.maybe_convert_level_to_quantiles(fcst_df, [self.alias]) return fcst_df diff --git a/timecopilot/models/neural.py b/timecopilot/models/neural.py index dcc24395..a7aa6f85 100644 --- a/timecopilot/models/neural.py +++ b/timecopilot/models/neural.py @@ -9,9 +9,10 @@ AutoTFT as _AutoTFT, ) from neuralforecast.common._base_model import BaseModel as NeuralForecastModel +from neuralforecast.losses.pytorch import MAE, MQLoss from ray import tune -from .utils.forecaster import Forecaster +from .utils.forecaster import Forecaster, QuantileConverter os.environ["NIXTLA_ID_AS_COL"] = "true" @@ -20,6 +21,8 @@ def run_neuralforecast_model( model: NeuralForecastModel, df: pd.DataFrame, freq: str, + alias: str, + qc: QuantileConverter, ) -> pd.DataFrame: nf = NeuralForecast( models=[model], @@ -27,6 +30,10 @@ def run_neuralforecast_model( ) nf.fit(df=df) fcst_df = nf.predict() + median_col = f"{alias}-median" + if median_col in fcst_df.columns: + fcst_df = fcst_df.rename(columns={median_col: alias}) + fcst_df = qc.maybe_convert_level_to_quantiles(fcst_df, [alias]) return fcst_df @@ -34,8 +41,8 @@ class AutoNHITS(Forecaster): """AutoNHITS forecaster using NeuralForecast. Notes: - - Level and quantiles are not supported for AutoNHITS yet. Please open - an issue if you need this feature. + - Quantile forecasts are supported via `quantiles`. `level` is not + supported; use `quantiles` instead. - AutoNHITS requires a minimum length for some frequencies. """ @@ -61,9 +68,9 @@ def forecast( ) -> pd.DataFrame: """Generate forecasts for time series data using the model. - This method produces point forecasts and, optionally, prediction - intervals or quantile forecasts. The input DataFrame can contain one - or multiple time series in stacked (long) format. + This method produces point forecasts and, optionally, quantile + forecasts. The input DataFrame can contain one or multiple time series + in stacked (long) format. Args: df (pd.DataFrame): @@ -83,15 +90,14 @@ def forecast( valid values. If not provided, the frequency will be inferred from the data. level (list[int | float], optional): - Confidence levels for prediction intervals, expressed as - percentages (e.g. [80, 95]). If provided, the returned - DataFrame will include lower and upper interval columns for - each specified level. + Not supported for AutoNHITS. Use `quantiles` instead. quantiles (list[float], optional): List of quantiles to forecast, expressed as floats between 0 and 1. Should not be used simultaneously with `level`. When - provided, the output DataFrame will contain additional columns - named in the format "model-q-{percentile}", where {percentile} + provided, the model is trained with + [`MQLoss`](https://nixtla.github.io/neuralforecast/losses.pytorch.html) + and the output DataFrame will contain additional columns named + in the format "model-q-{percentile}", where {percentile} = 100 × quantile value. Returns: @@ -99,16 +105,23 @@ def forecast( DataFrame containing forecast results. Includes: - point forecasts for each timestamp and series. - - prediction intervals if `level` is specified. - quantile forecasts if `quantiles` is specified. For multi-series data, the output retains the same unique identifiers as the input DataFrame. """ - if level is not None or quantiles is not None: - raise ValueError("Level and quantiles are not supported for AutoNHITS yet.") + if level is not None and quantiles is not None: + raise ValueError( + "You must not provide both `level` and `quantiles` simultaneously." + ) + if level is not None: + raise ValueError( + "Level is not supported for AutoNHITS. Please use `quantiles` instead." + ) inferred_freq = self._maybe_infer_freq(df, freq) + qc = QuantileConverter(level=None, quantiles=quantiles) + loss = MQLoss(level=qc.level) if qc.level is not None else MAE() if self.config is None: config = _AutoNHITS.get_default_config(h=h, backend="ray") config["scaler_type"] = tune.choice(["robust"]) @@ -120,6 +133,7 @@ def forecast( fcst_df = run_neuralforecast_model( model=_AutoNHITS( h=h, + loss=loss, alias=self.alias, num_samples=self.num_samples, backend=self.backend, @@ -127,6 +141,8 @@ def forecast( ), df=df, freq=inferred_freq, + alias=self.alias, + qc=qc, ) return fcst_df @@ -135,8 +151,8 @@ class AutoTFT(Forecaster): """AutoTFT forecaster using NeuralForecast. Notes: - - Level and quantiles are not supported for AutoTFT yet. Please open - an issue if you need this feature. + - Quantile forecasts are supported via `quantiles`. `level` is not + supported; use `quantiles` instead. - AutoTFT requires a minimum length for some frequencies. """ @@ -162,9 +178,9 @@ def forecast( ) -> pd.DataFrame: """Generate forecasts for time series data using the model. - This method produces point forecasts and, optionally, prediction - intervals or quantile forecasts. The input DataFrame can contain one - or multiple time series in stacked (long) format. + This method produces point forecasts and, optionally, quantile + forecasts. The input DataFrame can contain one or multiple time series + in stacked (long) format. Args: df (pd.DataFrame): @@ -184,15 +200,14 @@ def forecast( valid values. If not provided, the frequency will be inferred from the data. level (list[int | float], optional): - Confidence levels for prediction intervals, expressed as - percentages (e.g. [80, 95]). If provided, the returned - DataFrame will include lower and upper interval columns for - each specified level. + Not supported for AutoTFT. Use `quantiles` instead. quantiles (list[float], optional): List of quantiles to forecast, expressed as floats between 0 and 1. Should not be used simultaneously with `level`. When - provided, the output DataFrame will contain additional columns - named in the format "model-q-{percentile}", where {percentile} + provided, the model is trained with + [`MQLoss`](https://nixtla.github.io/neuralforecast/losses.pytorch.html) + and the output DataFrame will contain additional columns named + in the format "model-q-{percentile}", where {percentile} = 100 × quantile value. Returns: @@ -200,16 +215,23 @@ def forecast( DataFrame containing forecast results. Includes: - point forecasts for each timestamp and series. - - prediction intervals if `level` is specified. - quantile forecasts if `quantiles` is specified. For multi-series data, the output retains the same unique identifiers as the input DataFrame. """ - if level is not None or quantiles is not None: - raise ValueError("Level and quantiles are not supported for AutoTFT yet.") + if level is not None and quantiles is not None: + raise ValueError( + "You must not provide both `level` and `quantiles` simultaneously." + ) + if level is not None: + raise ValueError( + "Level is not supported for AutoTFT. Please use `quantiles` instead." + ) inferred_freq = self._maybe_infer_freq(df, freq) + qc = QuantileConverter(level=None, quantiles=quantiles) + loss = MQLoss(level=qc.level) if qc.level is not None else MAE() if self.config is None: config = _AutoTFT.get_default_config(h=h, backend="ray") config["scaler_type"] = tune.choice(["robust"]) @@ -220,6 +242,7 @@ def forecast( fcst_df = run_neuralforecast_model( model=_AutoTFT( h=h, + loss=loss, alias=self.alias, num_samples=self.num_samples, backend=self.backend, @@ -227,5 +250,7 @@ def forecast( ), df=df, freq=inferred_freq, + alias=self.alias, + qc=qc, ) return fcst_df From 776be1a9228604b3d8676830bd257a5d4488be5b Mon Sep 17 00:00:00 2001 From: AzulGarza Date: Sat, 25 Apr 2026 16:27:47 -0300 Subject: [PATCH 2/2] docs: add new changelog --- docs/changelogs/v0.0.26.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 docs/changelogs/v0.0.26.md diff --git a/docs/changelogs/v0.0.26.md b/docs/changelogs/v0.0.26.md new file mode 100644 index 00000000..44d96334 --- /dev/null +++ b/docs/changelogs/v0.0.26.md @@ -0,0 +1,23 @@ +### Features + +* **Prediction intervals for AutoLGBM, AutoNHITS, and AutoTFT**: These models now support quantile forecasts via the `quantiles` parameter. Pass a list of floats between 0 and 1 to receive additional output columns named `model-q-{percentile}`. Note that `level` is not supported for these models; use `quantiles` instead. + + - `AutoLGBM` computes prediction intervals via conformal prediction using cross-validation residuals. + - `AutoNHITS` and `AutoTFT` are trained with [`MQLoss`](https://nixtla.github.io/neuralforecast/losses.pytorch.html) when quantiles are requested. + + ```python + import pandas as pd + from timecopilot.models.ml import AutoLGBM + from timecopilot.models.neural import AutoNHITS, AutoTFT + + df = pd.read_csv("AirPassengers.csv", parse_dates=["ds"]) + df.insert(0, "unique_id", "AirPassengers") + + model = AutoLGBM() + fcst_df = model.forecast(df, h=12, quantiles=[0.1, 0.5, 0.9]) + # columns: unique_id, ds, AutoLGBM, AutoLGBM-q-10, AutoLGBM-q-50, AutoLGBM-q-90 + ``` + +--- + +**Full Changelog**: https://github.com/TimeCopilot/timecopilot/compare/v0.0.25...v0.0.26