From 1b380c5816fa6ef2a9f3db231bf151f1439cfaa6 Mon Sep 17 00:00:00 2001 From: SeaCelo Date: Fri, 12 Jun 2026 17:16:38 -0400 Subject: [PATCH 1/3] fix: compress case backup archives with DEFLATE instead of storing uncompressed --- API/Routes/Upload/UploadRoute.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/API/Routes/Upload/UploadRoute.py b/API/Routes/Upload/UploadRoute.py index 3205df9f8..dcc04446e 100644 --- a/API/Routes/Upload/UploadRoute.py +++ b/API/Routes/Upload/UploadRoute.py @@ -1,6 +1,6 @@ import shutil from flask import Blueprint, request, jsonify, send_file, after_this_request -from zipfile import ZipFile +from zipfile import ZipFile, ZIP_DEFLATED from pathlib import Path, PurePosixPath from werkzeug.utils import secure_filename import os, time, json, glob @@ -281,7 +281,13 @@ def backupCase(): zippedFile = Path(Config.validate_path(Config.DATA_STORAGE, f"{case}.zip")) '''File system data storage''' - with ZipFile(zippedFile, 'w') as zipObj: + # DEFLATE rather than the ZipFile default of ZIP_STORED. Case backups are + # JSON/CSV/text (model inputs, solver results, pivot views) which compress + # ~15-30x; storing them uncompressed made the demo archive 48 MB and a real + # country case ~944 MB. Every standard reader (Python zipfile, unzip, OS + # archivers) handles DEFLATE, and the read/extract paths are unchanged, so + # existing STORED backups still restore. + with ZipFile(zippedFile, 'w', compression=ZIP_DEFLATED) as zipObj: # Iterate over all the files in directory for folderName, subfolders, filenames in os.walk(str(casePath)): From 9b6c69e35b7742a3ffe12b52ffa72338721565a4 Mon Sep 17 00:00:00 2001 From: SeaCelo Date: Fri, 12 Jun 2026 17:28:47 -0400 Subject: [PATCH 2/3] build: point demo-data archive at recompressed DEFLATE asset (48.7MB -> 3.2MB), update pinned SHA-256 --- README.md | 2 +- scripts/setup_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b43b24126..766e8dada 100644 --- a/README.md +++ b/README.md @@ -68,7 +68,7 @@ For setup options, use the "--help" flag: The demo dataset (`CLEWs.Demo.zip`) is hosted as a [GitHub release asset](https://github.com/EAPD-DRB/MUIOGO/releases/tag/demo-data) and downloaded automatically during setup when not already cached locally. -- SHA-256: `facf4bda703f67b3c8b8697fea19d7d49be72bc2029fc05a68c61fd12ba7edde` +- SHA-256: `db92d380b0448f767c4ba56eea5c79b14bcae8fbf8e05a6a0d92d5345bb742c1` Setup installs demo data by default. The archive is downloaded once, cached in `assets/demo-data/`, and reused on subsequent runs. diff --git a/scripts/setup_dev.py b/scripts/setup_dev.py index 6c1559664..8b0eecc65 100644 --- a/scripts/setup_dev.py +++ b/scripts/setup_dev.py @@ -52,7 +52,7 @@ MAX_PYTHON = (3, 13) # exclusive DATA_STORAGE_DIR = PROJECT_ROOT / "WebAPP" / "DataStorage" DEMO_DATA_ARCHIVE = PROJECT_ROOT / "assets" / "demo-data" / "CLEWs.Demo.zip" -DEMO_DATA_ARCHIVE_SHA256 = "facf4bda703f67b3c8b8697fea19d7d49be72bc2029fc05a68c61fd12ba7edde" +DEMO_DATA_ARCHIVE_SHA256 = "db92d380b0448f767c4ba56eea5c79b14bcae8fbf8e05a6a0d92d5345bb742c1" DEMO_DATA_ARCHIVE_URL = ( "https://github.com/EAPD-DRB/MUIOGO/releases/download/demo-data/CLEWs.Demo.zip" ) From 580535918c475414a35e937904422cbda943a84e Mon Sep 17 00:00:00 2001 From: SeaCelo Date: Mon, 15 Jun 2026 09:52:09 -0400 Subject: [PATCH 3/3] fix: re-download stale cached demo-data archive instead of failing checksum --- scripts/setup_dev.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/setup_dev.py b/scripts/setup_dev.py index 8b0eecc65..09d642dd6 100644 --- a/scripts/setup_dev.py +++ b/scripts/setup_dev.py @@ -450,6 +450,16 @@ def install_demo_data(force: bool, yes: bool) -> bool: _print_pass("Demo data already installed", str(DEMO_DATA_REQUIRED_DIRS[0])) return True + # Self-heal a stale cache: if the cached archive exists but no longer matches + # the pinned hash (e.g. the release asset was recompressed/updated), drop it so + # the download path below re-fetches the current one. Without this a stale + # assets/demo-data/CLEWs.Demo.zip makes the checksum verification further down + # fail hard instead of recovering -- notably on --force-demo-data, which clears + # the extracted dirs but not the cached archive. + if DEMO_DATA_ARCHIVE.exists() and _sha256(DEMO_DATA_ARCHIVE) != DEMO_DATA_ARCHIVE_SHA256: + print(" Cached demo-data archive is stale (hash mismatch); re-downloading ...") + DEMO_DATA_ARCHIVE.unlink() + if not DEMO_DATA_ARCHIVE.exists(): print(" Demo-data archive not found locally; downloading from release asset ...") DEMO_DATA_ARCHIVE.parent.mkdir(parents=True, exist_ok=True)