Merge pull request #34 from pyronear/arthur/leaderboard-add-temporal-models

Chouffe · web-flow · commit 2acd049be429 · 2026-04-02T14:13:32.000+02:00
feat(leaderboard): add mtb-change-detection and pyro-detector-baseline
diff --git a/experiments/temporal-models/README.md b/experiments/temporal-models/README.md
@@ -11,7 +11,7 @@ Experiments exploring temporal/video-based approaches to reduce false positives
 
 ## 🏆 Leaderboard
 
-Current rankings on the test set (298 sequences: 149 wildfire + 149 false positive):
+Current rankings on the [pyro-dataset](https://github.com/pyronear/pyro-dataset) **v2.2.0** test set (298 sequences: 149 wildfire + 149 false positive):
 
 | Rank | Model | Precision | Recall | F1 | FPR | Mean TTD | Median TTD |
 |------|-------|-----------|--------|----|-----|----------|------------|
diff --git a/experiments/temporal-models/temporal-model-leaderboard/README.md b/experiments/temporal-models/temporal-model-leaderboard/README.md
@@ -1,43 +1,47 @@
-# Temporal Model Leaderboard
+# 🏆 Temporal Model Leaderboard
 
-Standardized evaluation and ranking of `TemporalModel` implementations on the [pyro-dataset](https://github.com/pyronear/pyro-dataset) sequential test set.
+Standardized evaluation and ranking of `TemporalModel` implementations on the [pyro-dataset](https://github.com/pyronear/pyro-dataset) **v2.2.0** sequential test set.
 
-## Leaderboard
+## 📊 Leaderboard
 
 | Rank | Model | Precision | Recall | F1 | FPR | Mean TTD (s) | Median TTD (s) |
 |------|-------|-----------|--------|----|-----|--------------|----------------|
 | 1 | [FSM Tracking Baseline](../tracking-fsm-baseline/) | 0.9474 | 0.9664 | 0.9568 | 0.0537 | 142.0 | 58.0 |
+| 2 | [Pyro-Detector Baseline](../pyro-detector-baseline/) | 0.8563 | 0.9597 | 0.9051 | 0.1611 | 27.0 | 7.0 |
+| 3 | [MTB Change Detection](../mtb-change-detection/) | 0.7165 | 0.9329 | 0.8105 | 0.3691 | 85.4 | 25.0 |
 
-*Evaluated on 298 sequences (149 wildfire + 149 false positive). Last updated: 2026-03-31.*
+*Evaluated on 298 sequences (149 wildfire + 149 false positive). Last updated: 2026-04-02.*
 
-## Models
+## 🤖 Models
 
 | Model | Description | Paper |
 |-------|-------------|-------|
 | [FSM Tracking Baseline](../tracking-fsm-baseline/) | YOLO11s detector + IoU-based FSM tracker. Requires temporal persistence (5 consecutive frames) before raising an alarm. Rule-based, no ML training. | [FLAME (Gragnaniello et al., 2024)](https://doi.org/10.1007/s00521-024-10963-z) |
+| [Pyro-Detector Baseline](../pyro-detector-baseline/) | Production pyro-predictor: YOLO ONNX + per-camera sliding-window temporal smoothing. Alarm when aggregated confidence crosses threshold over N consecutive frames. | -- |
+| [MTB Change Detection](../mtb-change-detection/) | YOLO11s + pixel-wise frame differencing (MTB ratio) to reject static FPs, followed by IoU-based FSM tracker. | [SlowFastMTB (Choi, Kim & Oh, 2022)](https://doi.org/10.3390/s22155602) |
 
-## Metrics
+## 📏 Metrics
 
 - **Precision, Recall, F1** -- sequence-level binary classification (smoke vs. no smoke)
 - **FPR** -- false positive rate
 - **Mean / Median TTD** -- time-to-detection in seconds for true positives (time from first frame to trigger frame)
 
-## Data
+## 📦 Data
 
 Test set imported via DVC from [pyro-dataset](https://github.com/pyronear/pyro-dataset):
 - 149 wildfire (positive) + 149 false positive (negative) sequences
 - Ground truth determined by directory structure (`wildfire/` vs `fp/`)
 - Max 20 frames per sequence, 30s apart
 
-## How to Reproduce
+## 🔄 How to Reproduce
 
 ```bash
 make install
 uv run dvc pull            # pull test set + model packages from S3
 uv run dvc repro           # run evaluation pipeline
 ```
 
-## Adding a New Model
+## ➕ Adding a New Model
 
 1. Implement `TemporalModel` in a new experiment under `experiments/temporal-models/`
 2. Package the model (see [tracking-fsm-baseline](../tracking-fsm-baseline/) for the zip format)
diff --git a/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/.gitignore b/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/.gitignore
@@ -1 +1,3 @@
 /fsm-tracking-baseline.zip
+/mtb-change-detection.zip
+/pyro-detector-baseline.zip
diff --git a/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/mtb-change-detection.zip.dvc b/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/mtb-change-detection.zip.dvc
@@ -0,0 +1,5 @@
+outs:
+- md5: af4db66fccf5be457dff99c0b66a30f5
+  size: 19260701
+  hash: md5
+  path: mtb-change-detection.zip
diff --git a/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/pyro-detector-baseline.zip.dvc b/experiments/temporal-models/temporal-model-leaderboard/data/01_raw/models/pyro-detector-baseline.zip.dvc
@@ -0,0 +1,5 @@
+outs:
+- md5: 8df77be9c5cd9a120569730c4160098a
+  size: 38513179
+  hash: md5
+  path: pyro-detector-baseline.zip
diff --git a/experiments/temporal-models/temporal-model-leaderboard/data/07_model_output/.gitignore b/experiments/temporal-models/temporal-model-leaderboard/data/07_model_output/.gitignore
@@ -1 +1,3 @@
 /fsm-tracking-baseline
+/mtb-change-detection
+/pyro-detector-baseline
diff --git a/experiments/temporal-models/temporal-model-leaderboard/data/08_reporting/leaderboard.json b/experiments/temporal-models/temporal-model-leaderboard/data/08_reporting/leaderboard.json
@@ -12,5 +12,33 @@
     "fpr": 0.0537,
     "mean_ttd_seconds": 142.0,
     "median_ttd_seconds": 58.0
+  },
+  {
+    "model_name": "pyro-detector-baseline",
+    "num_sequences": 298,
+    "tp": 143,
+    "fp": 24,
+    "fn": 6,
+    "tn": 125,
+    "precision": 0.8563,
+    "recall": 0.9597,
+    "f1": 0.9051,
+    "fpr": 0.1611,
+    "mean_ttd_seconds": 27.0,
+    "median_ttd_seconds": 7.0
+  },
+  {
+    "model_name": "mtb-change-detection",
+    "num_sequences": 298,
+    "tp": 139,
+    "fp": 55,
+    "fn": 10,
+    "tn": 94,
+    "precision": 0.7165,
+    "recall": 0.9329,
+    "f1": 0.8105,
+    "fpr": 0.3691,
+    "mean_ttd_seconds": 85.4,
+    "median_ttd_seconds": 25.0
   }
 ]
diff --git a/experiments/temporal-models/temporal-model-leaderboard/dvc.lock b/experiments/temporal-models/temporal-model-leaderboard/dvc.lock
@@ -18,12 +18,12 @@ stages:
       nfiles: 32640
     - path: scripts/evaluate.py
       hash: md5
-      md5: 75f32f92f6ac1d1ba28eb9c7bf5343e1
-      size: 3761
+      md5: b0f7e0612d85e1691ef78aed9bd2c118
+      size: 3979
     - path: src/temporal_model_leaderboard
       hash: md5
-      md5: cbdb9e92766ca170b1bc87485d74c633.dir
-      size: 32176
+      md5: 423d337f89bfe4e71fb126ec3c665aae.dir
+      size: 32284
       nfiles: 12
     outs:
     - path: data/07_model_output/fsm-tracking-baseline
@@ -37,24 +37,86 @@ stages:
     deps:
     - path: data/07_model_output
       hash: md5
-      md5: 6af82611b2a69282012915498c595d56.dir
-      size: 46667
-      nfiles: 4
+      md5: 8679635cba572131844bd723271f57d5.dir
+      size: 139733
+      nfiles: 8
     - path: scripts/leaderboard.py
       hash: md5
       md5: 81dec9584c04f465c99ced1b8c733a08
       size: 2507
     - path: src/temporal_model_leaderboard
       hash: md5
-      md5: cbdb9e92766ca170b1bc87485d74c633.dir
-      size: 32176
+      md5: 423d337f89bfe4e71fb126ec3c665aae.dir
+      size: 32284
       nfiles: 12
     outs:
     - path: data/08_reporting/leaderboard.json
       hash: md5
-      md5: bdfd6120841f4649413c6426e8cd04aa
-      size: 283
+      md5: a0333c0f68c720abf9000938e20ad851
+      size: 842
     - path: data/08_reporting/leaderboard.txt
       hash: md5
-      md5: cc2e649cfd237e8d4d7108e3efee5707
-      size: 279
+      md5: 49c9c9ac97a732437dfee797ee73c523
+      size: 470
+  evaluate_mtb_change_detection:
+    cmd: uv run python scripts/evaluate.py --model-name mtb-change-detection 
+      --model-type mtb-change-detection --model-package 
+      data/01_raw/models/mtb-change-detection.zip --test-dir 
+      data/01_raw/sequential_test/test --output-dir 
+      data/07_model_output/mtb-change-detection
+    deps:
+    - path: data/01_raw/models/mtb-change-detection.zip
+      hash: md5
+      md5: af4db66fccf5be457dff99c0b66a30f5
+      size: 19260701
+    - path: data/01_raw/sequential_test
+      hash: md5
+      md5: 828edb7fb8f0909f0d6115d67ffbddc9.dir
+      size: 1594484536
+      nfiles: 32640
+    - path: scripts/evaluate.py
+      hash: md5
+      md5: b0f7e0612d85e1691ef78aed9bd2c118
+      size: 3979
+    - path: src/temporal_model_leaderboard
+      hash: md5
+      md5: 423d337f89bfe4e71fb126ec3c665aae.dir
+      size: 32284
+      nfiles: 12
+    outs:
+    - path: data/07_model_output/mtb-change-detection
+      hash: md5
+      md5: 7b4f525b949035c54a0de70a635456c1.dir
+      size: 46552
+      nfiles: 2
+  evaluate_pyro_detector:
+    cmd: uv run python scripts/evaluate.py --model-name pyro-detector-baseline 
+      --model-type pyro-detector-baseline --model-package 
+      data/01_raw/models/pyro-detector-baseline.zip --test-dir 
+      data/01_raw/sequential_test/test --output-dir 
+      data/07_model_output/pyro-detector-baseline
+    deps:
+    - path: data/01_raw/models/pyro-detector-baseline.zip
+      hash: md5
+      md5: 8df77be9c5cd9a120569730c4160098a
+      size: 38513179
+    - path: data/01_raw/sequential_test
+      hash: md5
+      md5: 828edb7fb8f0909f0d6115d67ffbddc9.dir
+      size: 1594484536
+      nfiles: 32640
+    - path: scripts/evaluate.py
+      hash: md5
+      md5: b0f7e0612d85e1691ef78aed9bd2c118
+      size: 3979
+    - path: src/temporal_model_leaderboard
+      hash: md5
+      md5: 423d337f89bfe4e71fb126ec3c665aae.dir
+      size: 32284
+      nfiles: 12
+    outs:
+    - path: data/07_model_output/pyro-detector-baseline
+      hash: md5
+      md5: 0ff18e7a674f5a7c8b0452f3ce20cc4a.dir
+      size: 46514
+      nfiles: 2
diff --git a/experiments/temporal-models/temporal-model-leaderboard/dvc.yaml b/experiments/temporal-models/temporal-model-leaderboard/dvc.yaml
@@ -15,6 +15,38 @@ stages:
     outs:
       - data/07_model_output/fsm-tracking-baseline
 
+  evaluate_mtb_change_detection:
+    cmd: >-
+      uv run python scripts/evaluate.py
+      --model-name mtb-change-detection
+      --model-type mtb-change-detection
+      --model-package data/01_raw/models/mtb-change-detection.zip
+      --test-dir data/01_raw/sequential_test/test
+      --output-dir data/07_model_output/mtb-change-detection
+    deps:
+      - scripts/evaluate.py
+      - src/temporal_model_leaderboard
+      - data/01_raw/sequential_test
+      - data/01_raw/models/mtb-change-detection.zip
+    outs:
+      - data/07_model_output/mtb-change-detection
+
+  evaluate_pyro_detector:
+    cmd: >-
+      uv run python scripts/evaluate.py
+      --model-name pyro-detector-baseline
+      --model-type pyro-detector-baseline
+      --model-package data/01_raw/models/pyro-detector-baseline.zip
+      --test-dir data/01_raw/sequential_test/test
+      --output-dir data/07_model_output/pyro-detector-baseline
+    deps:
+      - scripts/evaluate.py
+      - src/temporal_model_leaderboard
+      - data/01_raw/sequential_test
+      - data/01_raw/models/pyro-detector-baseline.zip
+    outs:
+      - data/07_model_output/pyro-detector-baseline
+
   leaderboard:
     cmd: >-
       uv run python scripts/leaderboard.py
diff --git a/experiments/temporal-models/temporal-model-leaderboard/pyproject.toml b/experiments/temporal-models/temporal-model-leaderboard/pyproject.toml
@@ -6,11 +6,15 @@ requires-python = ">=3.11"
 dependencies = [
     "pyrocore",
     "tracking-fsm-baseline",
+    "mtb-change-detection",
+    "pyro-detector-baseline",
 ]
 
 [tool.uv.sources]
 pyrocore = { path = "../../../lib/pyrocore" }
 tracking-fsm-baseline = { path = "../tracking-fsm-baseline" }
+mtb-change-detection = { path = "../mtb-change-detection" }
+pyro-detector-baseline = { path = "../pyro-detector-baseline" }
 
 [dependency-groups]
 dev = [
diff --git a/experiments/temporal-models/temporal-model-leaderboard/scripts/evaluate.py b/experiments/temporal-models/temporal-model-leaderboard/scripts/evaluate.py
@@ -29,6 +29,14 @@
         "tracking_fsm_baseline.model",
         "FsmTrackingModel",
     ),
+    "mtb-change-detection": (
+        "mtb_change_detection.model",
+        "MtbChangeDetectionModel",
+    ),
+    "pyro-detector-baseline": (
+        "pyro_detector_baseline.model",
+        "PyroDetectorModel",
+    ),
 }
 
 logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
diff --git a/experiments/temporal-models/temporal-model-leaderboard/uv.lock b/experiments/temporal-models/temporal-model-leaderboard/uv.lock

Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
`1`	`1`	`/fsm-tracking-baseline.zip`
	`2`	`+/mtb-change-detection.zip`
	`3`	`+/pyro-detector-baseline.zip`
Original file line number	Diff line number	Diff line change
`@@ -1 +1,3 @@`
`1`	`1`	`/fsm-tracking-baseline`
	`2`	`+/mtb-change-detection`
	`3`	`+/pyro-detector-baseline`