From 83efd78885f94a5d545394c79867c7d53de7ad91 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 20:25:37 -0500 Subject: [PATCH 01/38] Edge Case Submission Fixes --- services/api/routers/admin.py | 30 ++++++++++++++++++++++++++++- services/api/routers/submissions.py | 21 ++++++++++++++++++++ services/nginx/nginx.conf | 1 + services/worker/worker/tasks.py | 30 +++++++++++++++++++++-------- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/services/api/routers/admin.py b/services/api/routers/admin.py index 5074fec..c91cfd4 100644 --- a/services/api/routers/admin.py +++ b/services/api/routers/admin.py @@ -1521,7 +1521,7 @@ def activate_submission( sub_row = db.execute( text(""" - SELECT id, user_id, submission_type + SELECT id, user_id, submission_type, status FROM submissions WHERE id = CAST(:sid AS uuid) AND deleted_at IS NULL """), @@ -1530,6 +1530,13 @@ def activate_submission( if sub_row is None: raise HTTPException(status_code=404, detail="Submission not found") + status: str = sub_row["status"] + if status not in ("validated", "evaluated"): + raise HTTPException( + status_code=409, + detail="Submission must be validated or evaluated before it can be set as active", + ) + user_id: str = str(sub_row["user_id"]) submission_type: str = sub_row["submission_type"] @@ -1560,6 +1567,27 @@ def activate_submission( ) db.commit() + # Setting to active mimics an initial submission + from routers.queue import _insert_job, _publish_task + from schemas.jobs import JobType + + if submission_type == "defense": + j_type = JobType.DEFENSE + payload = {"defense_submission_id": submission_id} + else: + j_type = JobType.ATTACK + payload = {"attack_submission_id": submission_id} + + job_id = _insert_job( + db=db, + job_type=j_type.value, + payload=payload, + requested_by_user_id=current_user.user_id, + ) + _publish_task(job_type=j_type, job_id=job_id, payload=payload) + + logger.info(f"Enqueued {submission_type} job {job_id} after admin set active") + client_ip, user_agent = _request_meta(request) log_audit_event( event_type=event_type, diff --git a/services/api/routers/submissions.py b/services/api/routers/submissions.py index 60df912..293189a 100644 --- a/services/api/routers/submissions.py +++ b/services/api/routers/submissions.py @@ -720,6 +720,27 @@ def set_active_submission( except Exception: logger.warning("Failed to publish leaderboard update after active submission change") + # Setting to active mimics an initial submission + from routers.queue import _insert_job, _publish_task + from schemas.jobs import JobType + + if sub_type == "defense": + j_type = JobType.DEFENSE + payload = {"defense_submission_id": submission_id} + else: + j_type = JobType.ATTACK + payload = {"attack_submission_id": submission_id} + + job_id = _insert_job( + db=db, + job_type=j_type.value, + payload=payload, + requested_by_user_id=current_user.user_id, + ) + _publish_task(job_type=j_type, job_id=job_id, payload=payload) + + logger.info(f"Enqueued {sub_type} job {job_id} after setting active") + return SetActiveResponse( submission_id=submission_id, submission_type=sub_type, diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index 36827d3..438d754 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -8,6 +8,7 @@ upstream frontend { server { listen 80; + listen [::]:80; client_max_body_size 512M; location = /health { diff --git a/services/worker/worker/tasks.py b/services/worker/worker/tasks.py index 1f0a980..a314522 100644 --- a/services/worker/worker/tasks.py +++ b/services/worker/worker/tasks.py @@ -455,22 +455,36 @@ async def _body() -> None: f"Registered worker {worker_id} with Redis for {len(defense_submission_ids)} defenses" ) - all_unevaluated_attacks: set[str] = set() - for dsid in defense_submission_ids: - all_unevaluated_attacks.update(get_unevaluated_attacks(dsid)) - - logger.info(f"Found {len(all_unevaluated_attacks)} unique unevaluated attacks for batch") - for attack_id in all_unevaluated_attacks: - registry.add_attack_to_queue(worker_id, attack_id) - defense_specs: list[tuple[str, bool, str, dict]] = [] + all_pending_attacks: set[str] = set() + for dsid in defense_submission_ids: needs_validation = check_if_needs_validation(dsid) + pending_attacks = get_unevaluated_attacks(dsid) + + if not needs_validation and not pending_attacks: + logger.info(f"Skipping container setup for defense {dsid}: already validated and no pending evaluations.") + continue + if needs_validation: mark_defense_validating(dsid) + source_type, source_data = get_defense_submission_source(dsid) defense_specs.append((dsid, needs_validation, source_type, source_data)) + + for attack_id in pending_attacks: + if attack_id not in all_pending_attacks: + registry.add_attack_to_queue(worker_id, attack_id) + all_pending_attacks.add(attack_id) + + if not defense_specs: + logger.info(f"No defenses in batch job {job_id} require container setup. Finishing job.") + set_job_status(job_id=job_id, status="done") + return + logger.info( + f"Found {len(all_pending_attacks)} unique unevaluated attacks for {len(defense_specs)} active defense containers" + ) logger.info(f"Setting up {len(defense_specs)} defense containers concurrently") setup_results = await asyncio.gather(*[ _setup_defense_container( From 8981d31ec18451c308084e8b48258a1d1e560b3a Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 20:34:27 -0500 Subject: [PATCH 02/38] Test fixes --- services/api/tests/conftest.py | 14 ++++++++++++++ .../worker/tests/test_defense_job_integration.py | 14 ++++++++++---- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/services/api/tests/conftest.py b/services/api/tests/conftest.py index 42d2b7b..c857ef6 100644 --- a/services/api/tests/conftest.py +++ b/services/api/tests/conftest.py @@ -5,8 +5,14 @@ from sqlalchemy.orm import sessionmaker from fastapi.testclient import TestClient +import os +os.environ["CELERY_BROKER_URL"] = "memory://" +os.environ["CELERY_DEFAULT_QUEUE"] = "mlsec" + from main import app from core.database import Base, get_db +from core.celery_app import get_celery +from unittest.mock import MagicMock TEST_DB_URL = "postgresql://postgres:password123@localhost:5433/mlsec_test" @@ -94,6 +100,14 @@ def override_get_db(): app.dependency_overrides.pop(get_db, None) +@pytest.fixture(autouse=True) +def mock_celery(monkeypatch): + """Mock Celery to prevent physical broker connections during tests.""" + mock_app = MagicMock() + monkeypatch.setattr("core.celery_app.get_celery", lambda: mock_app) + return mock_app + + @pytest.fixture() def fake_redis(): """Provide fake Redis client for testing without external dependency.""" diff --git a/services/worker/tests/test_defense_job_integration.py b/services/worker/tests/test_defense_job_integration.py index 6c016ae..bfc8e2c 100644 --- a/services/worker/tests/test_defense_job_integration.py +++ b/services/worker/tests/test_defense_job_integration.py @@ -412,11 +412,11 @@ def fake_init(self): monkeypatch.setattr(WorkerRegistry, "__init__", fake_init) - # Create defense with GitHub source + # Create defense with GitHub source (not validated yet to ensure setup runs) defense_id = test_helpers.create_defense( source_type="github", git_repo="https://github.com/user/defense", - is_functional=True + is_functional=None ) # Create job @@ -480,11 +480,11 @@ def fake_init(self): monkeypatch.setattr(WorkerRegistry, "__init__", fake_init) - # Create defense with ZIP source + # Create defense with ZIP source (not validated yet to ensure setup runs) defense_id = test_helpers.create_defense( source_type="zip", object_key="defenses/test-defense.zip", - is_functional=True + is_functional=None ) # Create job @@ -555,6 +555,9 @@ def fake_init(self): is_functional=True ) + # Create attack so evaluation starts + test_helpers.create_attack() + # Create job job_id = test_helpers.create_job( job_type="defense", @@ -643,6 +646,9 @@ def fake_init(self): is_functional=True ) + # Create attack so evaluation starts + test_helpers.create_attack() + # Create job job_id = test_helpers.create_job( job_type="defense", From 2b877265a2f70cb04589ce7c4f81d372692b1220 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 20:52:25 -0500 Subject: [PATCH 03/38] Remove Redundant Imports --- services/api/routers/admin.py | 3 --- services/api/routers/submissions.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/services/api/routers/admin.py b/services/api/routers/admin.py index c91cfd4..70bcc73 100644 --- a/services/api/routers/admin.py +++ b/services/api/routers/admin.py @@ -1568,9 +1568,6 @@ def activate_submission( db.commit() # Setting to active mimics an initial submission - from routers.queue import _insert_job, _publish_task - from schemas.jobs import JobType - if submission_type == "defense": j_type = JobType.DEFENSE payload = {"defense_submission_id": submission_id} diff --git a/services/api/routers/submissions.py b/services/api/routers/submissions.py index 293189a..26cd5b7 100644 --- a/services/api/routers/submissions.py +++ b/services/api/routers/submissions.py @@ -721,9 +721,6 @@ def set_active_submission( logger.warning("Failed to publish leaderboard update after active submission change") # Setting to active mimics an initial submission - from routers.queue import _insert_job, _publish_task - from schemas.jobs import JobType - if sub_type == "defense": j_type = JobType.DEFENSE payload = {"defense_submission_id": submission_id} From 7aa85ec3effcc23696c2efdc4c7cd8ca176bbb58 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 20:55:59 -0500 Subject: [PATCH 04/38] They Were Not Redundant This reverts commit 2b877265a2f70cb04589ce7c4f81d372692b1220. --- services/api/routers/admin.py | 3 +++ services/api/routers/submissions.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/services/api/routers/admin.py b/services/api/routers/admin.py index 70bcc73..c91cfd4 100644 --- a/services/api/routers/admin.py +++ b/services/api/routers/admin.py @@ -1568,6 +1568,9 @@ def activate_submission( db.commit() # Setting to active mimics an initial submission + from routers.queue import _insert_job, _publish_task + from schemas.jobs import JobType + if submission_type == "defense": j_type = JobType.DEFENSE payload = {"defense_submission_id": submission_id} diff --git a/services/api/routers/submissions.py b/services/api/routers/submissions.py index 26cd5b7..293189a 100644 --- a/services/api/routers/submissions.py +++ b/services/api/routers/submissions.py @@ -721,6 +721,9 @@ def set_active_submission( logger.warning("Failed to publish leaderboard update after active submission change") # Setting to active mimics an initial submission + from routers.queue import _insert_job, _publish_task + from schemas.jobs import JobType + if sub_type == "defense": j_type = JobType.DEFENSE payload = {"defense_submission_id": submission_id} From 22ef034197a81832fe1d3ce042c622e07e00d496 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 21:18:34 -0500 Subject: [PATCH 05/38] Attack Submission Fixes --- services/worker/worker/db.py | 11 + services/worker/worker/tasks.py | 382 ++++++++++++++++---------------- 2 files changed, 205 insertions(+), 188 deletions(-) diff --git a/services/worker/worker/db.py b/services/worker/worker/db.py index c9a6dff..ca165ee 100644 --- a/services/worker/worker/db.py +++ b/services/worker/worker/db.py @@ -49,6 +49,17 @@ def set_job_status(*, job_id: str, status: str, error: str | None = None) -> Non ) +def get_submission_status(submission_id: str) -> str | None: + from sqlalchemy import text + engine = get_engine() + with engine.connect() as conn: + result = conn.execute( + text("SELECT status FROM submissions WHERE id = :id"), + {"id": submission_id}, + ).scalar() + return result + + def get_defense_docker_image(*, submission_id: str) -> str | None: from sqlalchemy import text engine = get_engine() diff --git a/services/worker/worker/tasks.py b/services/worker/worker/tasks.py index a314522..70670e3 100644 --- a/services/worker/worker/tasks.py +++ b/services/worker/worker/tasks.py @@ -682,6 +682,7 @@ def run_attack_job(self, *, job_id: str, attack_submission_id: str) -> None: get_template_reports, get_all_validated_defenses, is_evaluation_in_progress, + get_submission_status, ) from worker.minio_client import get_minio_client, get_bucket_name from worker.attack.validation import ( @@ -701,208 +702,213 @@ def run_attack_job(self, *, job_id: str, attack_submission_id: str) -> None: logger.info( f"Starting attack job {job_id} for submission {attack_submission_id}") - # Attack validation logic - logger.info("Starting attack ZIP validation") - - # Get attack source information - attack_source = get_attack_submission_source(attack_submission_id) - zip_object_key = attack_source["zip_object_key"] - logger.info(f"Attack ZIP object key: {zip_object_key}") - - # Initialize MinIO client - minio_client = get_minio_client() - bucket_name = get_bucket_name() - - # Download ZIP to temporary file - # TODO: Store as individual files instead of zips, maybe? - temp_zip = tempfile.NamedTemporaryFile( - suffix='.zip', - prefix=f'attack_{attack_submission_id}_', - delete=False - ) - temp_zip.close() - - temp_extract_dir = None - - try: - logger.info(f"Downloading {zip_object_key} from MinIO") - minio_client.fget_object( - bucket_name, zip_object_key, temp_zip.name) - logger.info(f"Downloaded to {temp_zip.name}") - - # Functional validation (ZIP structure, password, safety) - attack_cfg = config.worker.attack - active_template = get_active_template() - - # Defer job if behavioral seeding is still in progress - if ( - active_template is not None - and attack_cfg.check_similarity - and not is_template_fully_seeded(active_template["id"]) - ): - logger.info( - "Template seeding in progress, deferring attack job %s", job_id - ) - raise self.retry(countdown=60) - - if active_template is None: - if attack_cfg.check_similarity: - error_msg = "No attack template is configured." - logger.warning( - "Attack %s rejected: %s", attack_submission_id, error_msg - ) - mark_attack_failed(attack_submission_id, error_msg) - set_job_status(job_id=job_id, status="failed", error=error_msg) - return - expected_files: set[str] = set() - else: - template_file_rows = get_template_files(active_template["id"]) - expected_files = {f["filename"] for f in template_file_rows} - - try: - validate_attack_functional( - temp_zip.name, - expected_files, - attack_cfg.max_zip_size_mb, - ) - except AttackValidationError as e: - error_msg = str(e) - logger.warning( - "Attack %s failed functional validation: %s", - attack_submission_id, - error_msg, - ) - mark_attack_failed(attack_submission_id, error_msg) - set_job_status(job_id=job_id, status="failed", error=error_msg) - return - - # Extract ZIP with password "infected" - temp_extract_dir = tempfile.mkdtemp( - prefix=f"attack_{attack_submission_id}_extract_" + # Check if validation is already complete + status = get_submission_status(attack_submission_id) + if status in ["validated", "evaluated"]: + logger.info(f"Attack {attack_submission_id} is already in '{status}' state, skipping validation logic.") + else: + # Attack validation logic + logger.info("Starting attack ZIP validation") + + # Get attack source information + attack_source = get_attack_submission_source(attack_submission_id) + zip_object_key = attack_source["zip_object_key"] + logger.info(f"Attack ZIP object key: {zip_object_key}") + + # Initialize MinIO client + minio_client = get_minio_client() + bucket_name = get_bucket_name() + + # Download ZIP to temporary file + # TODO: Store as individual files instead of zips, maybe? + temp_zip = tempfile.NamedTemporaryFile( + suffix='.zip', + prefix=f'attack_{attack_submission_id}_', + delete=False ) - logger.info(f"Extracting ZIP to {temp_extract_dir}") + temp_zip.close() + + temp_extract_dir = None try: - import pyzipper - with pyzipper.AESZipFile(temp_zip.name, 'r') as zf: - zf.setpassword(b'infected') - zf.extractall(temp_extract_dir) - logger.info("Successfully extracted attack ZIP") - except RuntimeError as e: - if "password" in str(e).lower(): - raise ValueError( - "Wrong password for attack ZIP (expected 'infected')") - raise ValueError(f"Failed to extract ZIP: {e}") - - # Scan extracted files and populate attack_files table - extract_path = Path(temp_extract_dir) - extracted_files = [] - submission_files: list[tuple[str, str]] = [] - - for file_path in extract_path.rglob('*'): - if file_path.is_file(): - # Calculate SHA256 - sha256_hash = hashlib.sha256() - with open(file_path, 'rb') as f: - for chunk in iter(lambda: f.read(8192), b''): - sha256_hash.update(chunk) - - # Get relative path - rel_path = file_path.relative_to(extract_path) - - # Inner filename (strips top-level wrapping folder) - inner_name = _inner_filename(file_path, extract_path) - submission_files.append((inner_name, str(file_path))) - - file_object_key = f"attack/{attack_submission_id}/{rel_path}" - minio_client.fput_object(bucket_name, file_object_key, str(file_path)) - - extracted_files.append({ - "filename": str(rel_path), - "sha256": sha256_hash.hexdigest(), - "byte_size": file_path.stat().st_size, - "object_key": file_object_key, - "is_malware": True # Default assumption for attack samples - }) - - logger.info(f"Found {len(extracted_files)} files in attack ZIP") - - # Insert files into database - if extracted_files: - inserted_count = insert_attack_files( - attack_submission_id, extracted_files) - logger.info( - f"Inserted {inserted_count} attack files into database") + logger.info(f"Downloading {zip_object_key} from MinIO") + minio_client.fget_object( + bucket_name, zip_object_key, temp_zip.name) + logger.info(f"Downloaded to {temp_zip.name}") + + # Functional validation (ZIP structure, password, safety) + attack_cfg = config.worker.attack + active_template = get_active_template() + + # Defer job if behavioral seeding is still in progress + if ( + active_template is not None + and attack_cfg.check_similarity + and not is_template_fully_seeded(active_template["id"]) + ): + logger.info( + "Template seeding in progress, deferring attack job %s", job_id + ) + raise self.retry(countdown=60) - # Heuristic validation (behavioral similarity against template) - if attack_cfg.check_similarity: if active_template is None: - template_reports = {} + if attack_cfg.check_similarity: + error_msg = "No attack template is configured." + logger.warning( + "Attack %s rejected: %s", attack_submission_id, error_msg + ) + mark_attack_failed(attack_submission_id, error_msg) + set_job_status(job_id=job_id, status="failed", error=error_msg) + return + expected_files: set[str] = set() else: - template_reports = get_template_reports_for_template( - active_template["id"] + template_file_rows = get_template_files(active_template["id"]) + expected_files = {f["filename"] for f in template_file_rows} + + try: + validate_attack_functional( + temp_zip.name, + expected_files, + attack_cfg.max_zip_size_mb, ) - if not template_reports: + except AttackValidationError as e: + error_msg = str(e) logger.warning( - "No template reports available, skipping heuristic " - "validation for attack %s.", + "Attack %s failed functional validation: %s", attack_submission_id, + error_msg, ) - else: - sandbox = get_sandbox_backend(attack_cfg) - try: - avg_similarity = validate_heuristic( - submission_files, sandbox, template_reports - ) - except SandboxUnavailableError as e: - error_msg = str(e) - logger.error( - "Sandbox unavailable during heuristic validation " - "for attack %s: %s", - attack_submission_id, - error_msg, - ) - mark_attack_failed(attack_submission_id, error_msg) - raise - - if ( - attack_cfg.reject_dissimilar_attacks - and avg_similarity < attack_cfg.minimum_attack_similarity - ): - error_msg = ( - f"Behavioral similarity {avg_similarity:.1f}% is below " - f"the minimum threshold of " - f"{attack_cfg.minimum_attack_similarity}%." + mark_attack_failed(attack_submission_id, error_msg) + set_job_status(job_id=job_id, status="failed", error=error_msg) + return + + # Extract ZIP with password "infected" + temp_extract_dir = tempfile.mkdtemp( + prefix=f"attack_{attack_submission_id}_extract_" + ) + logger.info(f"Extracting ZIP to {temp_extract_dir}") + + try: + import pyzipper + with pyzipper.AESZipFile(temp_zip.name, 'r') as zf: + zf.setpassword(b'infected') + zf.extractall(temp_extract_dir) + logger.info("Successfully extracted attack ZIP") + except RuntimeError as e: + if "password" in str(e).lower(): + raise ValueError( + "Wrong password for attack ZIP (expected 'infected')") + raise ValueError(f"Failed to extract ZIP: {e}") + + # Scan extracted files and populate attack_files table + extract_path = Path(temp_extract_dir) + submission_files: list[tuple[str, str]] = [] + extracted_files = [] + + for file_path in extract_path.rglob('*'): + if file_path.is_file(): + # Calculate SHA256 + sha256_hash = hashlib.sha256() + with open(file_path, 'rb') as f: + for chunk in iter(lambda: f.read(8192), b''): + sha256_hash.update(chunk) + + # Get relative path + rel_path = file_path.relative_to(extract_path) + + # Inner filename (strips top-level wrapping folder) + inner_name = _inner_filename(file_path, extract_path) + submission_files.append((inner_name, str(file_path))) + + file_object_key = f"attack/{attack_submission_id}/{rel_path}" + minio_client.fput_object(bucket_name, file_object_key, str(file_path)) + + extracted_files.append({ + "filename": str(rel_path), + "sha256": sha256_hash.hexdigest(), + "byte_size": file_path.stat().st_size, + "object_key": file_object_key, + "is_malware": True # Default assumption for attack samples + }) + + logger.info(f"Found {len(extracted_files)} files in attack ZIP") + + # Insert files into database + if extracted_files: + inserted_count = insert_attack_files( + attack_submission_id, extracted_files) + logger.info( + f"Inserted {inserted_count} attack files into database") + + # Heuristic validation (behavioral similarity against template) + if attack_cfg.check_similarity: + if active_template is None: + template_reports = {} + else: + template_reports = get_template_reports_for_template( + active_template["id"] ) + if not template_reports: logger.warning( - "Attack %s rejected: %s", + "No template reports available, skipping heuristic " + "validation for attack %s.", attack_submission_id, - error_msg, ) - mark_attack_failed(attack_submission_id, error_msg) - set_job_status( - job_id=job_id, status="failed", error=error_msg - ) - return - elif not attack_cfg.reject_dissimilar_attacks: - logger.info( - "Attack %s heuristic similarity=%.1f%% " - "(reject_dissimilar_attacks=False, accepting).", - attack_submission_id, - avg_similarity, - ) - - # Mark attack as validated - mark_attack_validated(attack_submission_id) - logger.info(f"Attack {attack_submission_id} marked as validated") - - finally: - # Cleanup temporary files - if os.path.exists(temp_zip.name): - os.unlink(temp_zip.name) - if temp_extract_dir and os.path.exists(temp_extract_dir): - import shutil - shutil.rmtree(temp_extract_dir) + else: + sandbox = get_sandbox_backend(attack_cfg) + try: + avg_similarity = validate_heuristic( + submission_files, sandbox, template_reports + ) + except SandboxUnavailableError as e: + error_msg = str(e) + logger.error( + "Sandbox unavailable during heuristic validation " + "for attack %s: %s", + attack_submission_id, + error_msg, + ) + mark_attack_failed(attack_submission_id, error_msg) + raise + + if ( + attack_cfg.reject_dissimilar_attacks + and avg_similarity < attack_cfg.minimum_attack_similarity + ): + error_msg = ( + f"Behavioral similarity {avg_similarity:.1f}% is below " + f"the minimum threshold of " + f"{attack_cfg.minimum_attack_similarity}%." + ) + logger.warning( + "Attack %s rejected: %s", + attack_submission_id, + error_msg, + ) + mark_attack_failed(attack_submission_id, error_msg) + set_job_status( + job_id=job_id, status="failed", error=error_msg + ) + return + elif not attack_cfg.reject_dissimilar_attacks: + logger.info( + "Attack %s heuristic similarity=%.1f%% " + "(reject_dissimilar_attacks=False, accepting).", + attack_submission_id, + avg_similarity, + ) + + # Mark attack as validated + mark_attack_validated(attack_submission_id) + logger.info(f"Attack {attack_submission_id} marked as validated") + + finally: + # Cleanup temporary files + if os.path.exists(temp_zip.name): + os.unlink(temp_zip.name) + if temp_extract_dir and os.path.exists(temp_extract_dir): + import shutil + shutil.rmtree(temp_extract_dir) # Initialize Redis client # Use a temporary worker ID for API-side operations From 50bf3dfea00119aaa439e4ce26bba63b4b85281a Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 21:30:15 -0500 Subject: [PATCH 06/38] I'm Losing It --- services/worker/worker/tasks.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/services/worker/worker/tasks.py b/services/worker/worker/tasks.py index 70670e3..9d4e0fb 100644 --- a/services/worker/worker/tasks.py +++ b/services/worker/worker/tasks.py @@ -698,16 +698,22 @@ def run_attack_job(self, *, job_id: str, attack_submission_id: str) -> None: try: set_job_status(job_id=job_id, status="running") - mark_attack_validating(attack_submission_id) + + # Check if validation or evaluation is already complete + status = get_submission_status(attack_submission_id) + if status == "evaluated": + logger.info(f"Attack {attack_submission_id} is already evaluated, skipping job.") + set_job_status(job_id=job_id, status="done") + return + logger.info( f"Starting attack job {job_id} for submission {attack_submission_id}") - # Check if validation is already complete - status = get_submission_status(attack_submission_id) - if status in ["validated", "evaluated"]: - logger.info(f"Attack {attack_submission_id} is already in '{status}' state, skipping validation logic.") + if status == "validated": + logger.info(f"Attack {attack_submission_id} is already validated, skipping validation logic.") else: # Attack validation logic + mark_attack_validating(attack_submission_id) logger.info("Starting attack ZIP validation") # Get attack source information From 2166ffc8417614c9f97504d38a8284cb11992ae1 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 21:39:53 -0500 Subject: [PATCH 07/38] Test Fixes --- .../worker/tests/test_attack_job_integration.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/services/worker/tests/test_attack_job_integration.py b/services/worker/tests/test_attack_job_integration.py index f6df04b..c6c0ca0 100644 --- a/services/worker/tests/test_attack_job_integration.py +++ b/services/worker/tests/test_attack_job_integration.py @@ -551,8 +551,13 @@ def fake_init(self): monkeypatch.setattr(WorkerRegistry, "__init__", fake_init) - # Create attack + # Create attack in submitted state to trigger validation logic attack_id = test_helpers.create_attack() + db_session.execute( + text("UPDATE submissions SET status = 'submitted' WHERE id = CAST(:id AS uuid)"), + {"id": attack_id} + ) + db_session.commit() # Create job job_id = test_helpers.create_job( @@ -613,8 +618,15 @@ def fake_redis_init(self): self.client = fake_redis monkeypatch.setattr(WorkerRegistry, "__init__", fake_redis_init) - + + # Create attack in submitted state to trigger validation logic attack_id = test_helpers.create_attack() + db_session.execute( + text("UPDATE submissions SET status = 'submitted' WHERE id = CAST(:id AS uuid)"), + {"id": attack_id} + ) + db_session.commit() + job_id = test_helpers.create_job( job_type="attack", status="queued", From 10e252ea9b14cd4a265f998c760cc8ebad6b9d30 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Sun, 5 Apr 2026 22:15:51 -0500 Subject: [PATCH 08/38] Multiple Worker Containers --- Makefile | 23 +++++++++++++++++++++++ config.yaml | 4 ++-- docker-compose.yaml | 2 +- services/worker/Dockerfile | 2 +- 4 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0c2e987 --- /dev/null +++ b/Makefile @@ -0,0 +1,23 @@ +.PHONY: up down build ps logs worker-count + +# Parse num_workers from config.yaml +NUM_WORKERS=$(shell awk '/worker:/ {found=1} found && /num_workers:/ {print $$2; exit}' config.yaml) + +up: + @echo "Starting platform with $(NUM_WORKERS) workers..." + docker compose up --scale worker=$(NUM_WORKERS) + +down: + docker compose down + +build: + docker compose build + +ps: + docker compose ps + +logs: + docker compose logs -f + +worker-count: + @echo $(NUM_WORKERS) diff --git a/config.yaml b/config.yaml index b08aedd..50b9e6f 100644 --- a/config.yaml +++ b/config.yaml @@ -1,5 +1,5 @@ worker: - num_workers: 4 # number of concurrent Celery worker processes (maps to --concurrency) + num_workers: 4 # number of concurrent Worker containers (Total Possible Competitor Containers = num_workers*batch_size) defense_job: mem_limit: "1g" nano_cpus: 1000000000 @@ -8,7 +8,7 @@ worker: max_uncompressed_size_mb: 1024 evaluation: requests_timeout_seconds: 5 - batch_size: 4 + batch_size: 2 defense_max_ram: 1024 # MB - soft RAM threshold; sample marked evaded and container restarted if exceeded defense_max_time: 5000 # ms - per-sample time limit; exceeded = evaded defense_max_timeout: 20000 # ms - forced restart threshold (must be >= defense_max_time) diff --git a/docker-compose.yaml b/docker-compose.yaml index ea9aebf..e0f00ca 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -136,7 +136,7 @@ services: MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} MINIO_SECURE: "false" VIRUSTOTAL_API_KEY: ${VIRUSTOTAL_API_KEY:-} - CELERY_CONCURRENCY: "${WORKER_CONCURRENCY:-4}" + CELERY_CONCURRENCY: "${WORKER_CONCURRENCY:-1}" # Forces use of a single thread per Worker container volumes: - /var/run/docker.sock:/var/run/docker.sock - ./config.yaml:/app/config.yaml:ro diff --git a/services/worker/Dockerfile b/services/worker/Dockerfile index 49d414b..1d1a85e 100644 --- a/services/worker/Dockerfile +++ b/services/worker/Dockerfile @@ -21,5 +21,5 @@ COPY . /app # RUN useradd --create-home --uid 10001 appuser # USER appuser -CMD ["sh", "-c", "celery -A worker.celery_app:celery_app worker --loglevel=INFO --concurrency=${CELERY_CONCURRENCY:-4}"] +CMD ["sh", "-c", "celery -A worker.celery_app:celery_app worker --loglevel=INFO --concurrency=${CELERY_CONCURRENCY:-1}"] From 03b6c7a12a52b07b53a296b0a2f5af9a384a43c9 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 12:28:44 -0500 Subject: [PATCH 09/38] Added optional seeding flag --- config.yaml | 1 + services/worker/worker/config.py | 4 ++++ services/worker/worker/tasks.py | 9 ++++++++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index b08aedd..8b6271a 100644 --- a/config.yaml +++ b/config.yaml @@ -44,6 +44,7 @@ worker: access_key: "mlsec_minio_admin" secret_key: "mlsec_minio_password_change_in_production" attack: + skip_seeding: false # true = skip template seeding and all behavioral checks check_similarity: false # false = skip evaluation, accept all validated attacks reject_dissimilar_attacks: false # only applies when check_similarity=true # true = reject if score < minimum_attack_similarity diff --git a/services/worker/worker/config.py b/services/worker/worker/config.py index 62d6d1d..10e3550 100644 --- a/services/worker/worker/config.py +++ b/services/worker/worker/config.py @@ -73,6 +73,10 @@ class SourceConfig(BaseModel): class AttackConfig(BaseModel): """Configuration for attack validation and evaluation.""" + # True = skip template seeding and all behavioral checks entirely. + # Overrides check_similarity when set to True. + skip_seeding: bool = False + # Whether to run similarity evaluation at all. # False = skip evaluation, accept all attacks that pass validation. check_similarity: bool = True diff --git a/services/worker/worker/tasks.py b/services/worker/worker/tasks.py index 9d4e0fb..794de5d 100644 --- a/services/worker/worker/tasks.py +++ b/services/worker/worker/tasks.py @@ -623,6 +623,12 @@ def seed_attack_template(self, *, template_id: str, job_id: str | None = None) - if job_id: set_job_status(job_id=job_id, status="running") + if config.worker.attack.skip_seeding: + logger.info("skip_seeding=true; skipping template seeding for template %s.", template_id) + if job_id: + set_job_status(job_id=job_id, status="done") + return + try: template_files = get_template_files(template_id) if not template_files: @@ -750,6 +756,7 @@ def run_attack_job(self, *, job_id: str, attack_submission_id: str) -> None: if ( active_template is not None and attack_cfg.check_similarity + and not attack_cfg.skip_seeding and not is_template_fully_seeded(active_template["id"]) ): logger.info( @@ -847,7 +854,7 @@ def run_attack_job(self, *, job_id: str, attack_submission_id: str) -> None: f"Inserted {inserted_count} attack files into database") # Heuristic validation (behavioral similarity against template) - if attack_cfg.check_similarity: + if attack_cfg.check_similarity and not attack_cfg.skip_seeding: if active_template is None: template_reports = {} else: From 6eedf3e06efea26accd2ee516ddf6e7716e324f3 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 13:01:03 -0500 Subject: [PATCH 10/38] Added more error handling to evaluate task --- config.yaml | 2 +- services/worker/worker/defense/evaluate.py | 28 ++++++++++++++++++++++ services/worker/worker/tasks.py | 1 + 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 8b6271a..0a26b30 100644 --- a/config.yaml +++ b/config.yaml @@ -44,7 +44,7 @@ worker: access_key: "mlsec_minio_admin" secret_key: "mlsec_minio_password_change_in_production" attack: - skip_seeding: false # true = skip template seeding and all behavioral checks + skip_seeding: true # true = skip template seeding and all behavioral checks check_similarity: false # false = skip evaluation, accept all validated attacks reject_dissimilar_attacks: false # only applies when check_similarity=true # true = reject if score < minimum_attack_similarity diff --git a/services/worker/worker/defense/evaluate.py b/services/worker/worker/defense/evaluate.py index 7d7c320..f02c75a 100644 --- a/services/worker/worker/defense/evaluate.py +++ b/services/worker/worker/defense/evaluate.py @@ -194,6 +194,34 @@ async def evaluate_sample_against_container( "Extended wait request failed for %s: %s", container_url, exc ) + except httpx.NetworkError as exc: + # Connection dropped mid-request (ReadError, ConnectError, etc.). + # Treat this the same as a container crash: mark evaded, restart. + evaded_reason = "connection_error" + model_output = 0 + logger.warning( + "Container %s dropped connection (%s: %s); restarting.", + container_name, + type(exc).__name__, + exc or "no message", + ) + restart_count_ref[0] += 1 + if restart_count_ref[0] > eval_cfg.defense_max_restarts: + raise ContainerRestartError( + f"Container {container_name!r} exceeded maximum restarts " + f"({eval_cfg.defense_max_restarts})." + ) + try: + ctx.pop("container_obj", None) # Force re-fetch after restart + ctx["container_obj"] = await asyncio.to_thread(docker_client.containers.get, container_name) + await asyncio.to_thread(ctx["container_obj"].restart) + except Exception as restart_exc: + logger.warning( + "Failed to restart container %s after network error: %s", + container_name, + restart_exc, + ) + duration_ms = int((time.monotonic() - start) * 1000) return EvalOutcome( model_output=model_output, diff --git a/services/worker/worker/tasks.py b/services/worker/worker/tasks.py index 794de5d..8f90fa6 100644 --- a/services/worker/worker/tasks.py +++ b/services/worker/worker/tasks.py @@ -386,6 +386,7 @@ async def _validate_defense_container( "Unexpected error during validation of defense %s: %s", defense_submission_id, e, + exc_info=True, ) try: mark_defense_failed(defense_submission_id, f"Unexpected validation error: {e}") From e7d78ea685fdd747734862b89f83399f2af4edbf Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 14:06:21 -0500 Subject: [PATCH 11/38] Fixed admin page auto-close overriding manual open close; close 52 --- services/api/core/submission_control.py | 15 +++++++++++++-- .../admin/competition/CompetitionPage.tsx | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/services/api/core/submission_control.py b/services/api/core/submission_control.py index 5e9e60f..ba63152 100644 --- a/services/api/core/submission_control.py +++ b/services/api/core/submission_control.py @@ -91,7 +91,12 @@ def set_manual_closed( closed: bool, updated_by: str | None, ) -> SubmissionControl: - """Toggle the manual close flag and return the updated control state.""" + """Toggle the manual close flag and return the updated control state. + + When opening submissions (closed=False), also clears a lapsed scheduled + close time so the submission window is actually open afterwards. + """ + now = _utcnow() row = ( db.execute( text( @@ -100,6 +105,11 @@ def set_manual_closed( VALUES (1, :manual_closed, :updated_at, :updated_by) ON CONFLICT (id) DO UPDATE SET manual_closed = EXCLUDED.manual_closed, + close_at = CASE + WHEN NOT :manual_closed AND submission_control.close_at <= :now + THEN NULL + ELSE submission_control.close_at + END, updated_at = EXCLUDED.updated_at, updated_by = EXCLUDED.updated_by RETURNING manual_closed, close_at, updated_at, updated_by @@ -107,8 +117,9 @@ def set_manual_closed( ), { "manual_closed": closed, - "updated_at": _utcnow(), + "updated_at": now, "updated_by": updated_by, + "now": now, }, ) .mappings() diff --git a/services/frontend/src/components/admin/competition/CompetitionPage.tsx b/services/frontend/src/components/admin/competition/CompetitionPage.tsx index 6b86aef..8b1bee3 100644 --- a/services/frontend/src/components/admin/competition/CompetitionPage.tsx +++ b/services/frontend/src/components/admin/competition/CompetitionPage.tsx @@ -194,6 +194,7 @@ function SubmissionSection() {

Schedule Auto-Close

+

Opening submissions after scheduled auto-close will empty the auto-close date.

{status.close_at && (

Scheduled:{' '} From fcd018cc4d36e5e297e37229bda5230036d38c94 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 14:11:06 -0500 Subject: [PATCH 12/38] Fixed semver spilling; close #52 (forgot the # last time), close #56 --- .../src/components/EvaluationMatrix.tsx | 20 +++++++++++-------- .../src/components/SubmissionHistory.tsx | 5 ++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/services/frontend/src/components/EvaluationMatrix.tsx b/services/frontend/src/components/EvaluationMatrix.tsx index 64e2c82..260e081 100644 --- a/services/frontend/src/components/EvaluationMatrix.tsx +++ b/services/frontend/src/components/EvaluationMatrix.tsx @@ -136,13 +136,14 @@ export default function EvaluationMatrix() { {attackers.map(atk => ( -

{atk.username}
+
{atk.username}
{atk.display_name && ( -
{atk.display_name}
+
{atk.display_name}
)} -
v{atk.version}
+
v{atk.version}
))} @@ -150,12 +151,15 @@ export default function EvaluationMatrix() { {defenders.map((def, di) => ( - -
{def.username}
+ +
{def.username}
{def.display_name && ( -
{def.display_name}
+
{def.display_name}
)} -
v{def.version}
+
v{def.version}
{attackers.map(atk => { const key = `${atk.submission_id}/${def.submission_id}`; diff --git a/services/frontend/src/components/SubmissionHistory.tsx b/services/frontend/src/components/SubmissionHistory.tsx index cbe9e92..1bd79e3 100644 --- a/services/frontend/src/components/SubmissionHistory.tsx +++ b/services/frontend/src/components/SubmissionHistory.tsx @@ -247,7 +247,10 @@ export default function SubmissionHistory({ type, title }: Props) { {formatDate(sub.created_at)} - + v{sub.version} From 7ba647eb7070be5bfedaddf9866b2e1d65ea207e Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 14:15:11 -0500 Subject: [PATCH 13/38] Fixed submit alert not having a close button and the page not being scrollable (sorry Ali); close #58 --- .../src/components/Attack_submit.astro | 22 ++++++++++++++++--- .../src/components/Defense_submit.astro | 19 ++++++++++++++-- services/frontend/src/pages/submission.astro | 4 ++-- 3 files changed, 38 insertions(+), 7 deletions(-) diff --git a/services/frontend/src/components/Attack_submit.astro b/services/frontend/src/components/Attack_submit.astro index 732523b..b11f058 100644 --- a/services/frontend/src/components/Attack_submit.astro +++ b/services/frontend/src/components/Attack_submit.astro @@ -90,6 +90,19 @@ } }); + function _attachDismiss(el: HTMLElement) { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.textContent = '×'; + btn.className = 'ml-3 text-current opacity-60 hover:opacity-100 leading-none flex-shrink-0'; + btn.setAttribute('aria-label', 'Dismiss'); + btn.addEventListener('click', () => { + el.className = 'hidden text-sm rounded-lg px-3 py-2'; + el.textContent = ''; + }); + el.appendChild(btn); + } + document.getElementById('attack-form')!.addEventListener('submit', async (e) => { e.preventDefault(); @@ -102,7 +115,8 @@ if (!file) { feedback.textContent = 'Please select a ZIP file.'; - feedback.className = 'text-sm rounded-lg px-3 py-2 bg-red-50 text-red-700 border border-red-200'; + feedback.className = 'flex items-center justify-between text-sm rounded-lg px-3 py-2 bg-red-50 text-red-700 border border-red-200'; + _attachDismiss(feedback); return; } @@ -121,7 +135,8 @@ if (!res.ok) throw new Error(data.detail ?? `Error ${res.status}`); feedback.textContent = `Submitted (ID: ${data.submission_id})`; - feedback.className = 'text-sm rounded-lg px-3 py-2 bg-green-50 text-green-700 border border-green-200'; + feedback.className = 'flex items-center justify-between text-sm rounded-lg px-3 py-2 bg-green-50 text-green-700 border border-green-200'; + _attachDismiss(feedback); (e.target as HTMLFormElement).reset(); attackIdleView.classList.remove('hidden'); attackChosenView.classList.add('hidden'); @@ -130,7 +145,8 @@ } catch (err: unknown) { feedback.textContent = (err instanceof Error ? err.message : null) ?? 'Submission failed.'; - feedback.className = 'text-sm rounded-lg px-3 py-2 bg-red-50 text-red-700 border border-red-200'; + feedback.className = 'flex items-center justify-between text-sm rounded-lg px-3 py-2 bg-red-50 text-red-700 border border-red-200'; + _attachDismiss(feedback); } finally { btn.disabled = false; btn.textContent = 'Submit Attack'; diff --git a/services/frontend/src/components/Defense_submit.astro b/services/frontend/src/components/Defense_submit.astro index 313150e..2391c16 100644 --- a/services/frontend/src/components/Defense_submit.astro +++ b/services/frontend/src/components/Defense_submit.astro @@ -146,6 +146,19 @@ \ No newline at end of file + diff --git a/services/frontend/src/components/Past_submit.astro b/services/frontend/src/components/Past_submit.astro index 4801475..8b52804 100644 --- a/services/frontend/src/components/Past_submit.astro +++ b/services/frontend/src/components/Past_submit.astro @@ -2,7 +2,7 @@ import SubmissionHistory from './SubmissionHistory'; --- -
+

Submission History

diff --git a/services/frontend/src/pages/rules.astro b/services/frontend/src/pages/rules.astro index 47a1802..5c41d95 100644 --- a/services/frontend/src/pages/rules.astro +++ b/services/frontend/src/pages/rules.astro @@ -14,7 +14,7 @@ import { Content } from '../content/rules.md'; MLSEC Rules - +
diff --git a/services/frontend/src/pages/submission.astro b/services/frontend/src/pages/submission.astro index 7788027..9853ea9 100644 --- a/services/frontend/src/pages/submission.astro +++ b/services/frontend/src/pages/submission.astro @@ -21,7 +21,7 @@ const session = await getSession(Astro.request); {session ? ( -
+
From cc767c788f9500d0da40fb9b786366a95c7a92ce Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 19:38:44 -0500 Subject: [PATCH 17/38] Implemented export CSV's to admin competition page; close #54 --- services/api/routers/admin.py | 234 ++++++++++++++++++ .../admin/competition/CompetitionPage.tsx | 196 +++++++++++++++ 2 files changed, 430 insertions(+) diff --git a/services/api/routers/admin.py b/services/api/routers/admin.py index c91cfd4..df5dfbc 100644 --- a/services/api/routers/admin.py +++ b/services/api/routers/admin.py @@ -1,5 +1,6 @@ from __future__ import annotations +import csv import io import json import logging @@ -8,6 +9,7 @@ from uuid import UUID, uuid4 from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile, status +from fastapi.responses import StreamingResponse from sqlalchemy import text from sqlalchemy.orm import Session @@ -1627,3 +1629,235 @@ def activate_submission( message=str(exc), ) raise + + +# --------------------------------------------------------------------------- +# CSV exports +# --------------------------------------------------------------------------- + +def _csv_response(rows: list[list], filename: str) -> StreamingResponse: + """Build a StreamingResponse from a 2-D list of CSV rows.""" + buf = io.StringIO() + writer = csv.writer(buf) + writer.writerows(rows) + buf.seek(0) + return StreamingResponse( + iter([buf.getvalue()]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + +def _submission_label(username: str, display_name: str | None, version: str) -> str: + return f"{username} / {display_name or version}" + + +@router.get("/export/scores/all") +def export_all_evaluation_scores( + _: AuthenticatedUser = Depends(require_admin_user), + db: Session = Depends(get_db), +) -> StreamingResponse: + """Download confusion-matrix CSV (TP, FP, FN, TN) for all active submission pairs.""" + axis_rows = db.execute( + text(""" + SELECT u.username, s.display_name, s.version, s.id::text, a.submission_type + FROM active_submissions a + JOIN submissions s ON s.id = a.submission_id + JOIN users u ON u.id = a.user_id + WHERE u.disabled_at IS NULL AND s.deleted_at IS NULL + ORDER BY a.submission_type, u.username + """) + ).fetchall() + + attackers = [r for r in axis_rows if r[4] == "attack"] + defenders = [r for r in axis_rows if r[4] == "defense"] + + if not attackers or not defenders: + return _csv_response( + [["No active submission pairs available."]], + "evaluation_scores_all.csv", + ) + + attack_ids = [r[3] for r in attackers] + defense_ids = [r[3] for r in defenders] + + file_rows = db.execute( + text(""" + SELECT eps.defense_submission_id::text, + eps.attack_submission_id::text, + af.is_malware, + efr.model_output + FROM evaluation_pair_scores eps + JOIN evaluation_runs er ON er.id = eps.latest_evaluation_run_id + JOIN evaluation_file_results efr ON efr.evaluation_run_id = er.id + JOIN attack_files af ON af.id = efr.attack_file_id + WHERE eps.defense_submission_id::text = ANY(:def_ids) + AND eps.attack_submission_id::text = ANY(:atk_ids) + AND eps.latest_evaluation_run_id IS NOT NULL + AND efr.model_output IS NOT NULL + AND af.is_malware IS NOT NULL + """), + {"def_ids": defense_ids, "atk_ids": attack_ids}, + ).fetchall() + + confusion: dict[tuple[str, str], dict[str, int]] = {} + for fr in file_rows: + key = (fr[0], fr[1]) + if key not in confusion: + confusion[key] = {"tp": 0, "fp": 0, "fn": 0, "tn": 0} + if fr[3] == 1 and fr[2]: confusion[key]["tp"] += 1 + elif fr[3] == 1 and not fr[2]: confusion[key]["fp"] += 1 + elif fr[3] == 0 and fr[2]: confusion[key]["fn"] += 1 + elif fr[3] == 0 and not fr[2]: confusion[key]["tn"] += 1 + + header = ["Defense \\ Attack"] + [_submission_label(r[0], r[1], r[2]) for r in attackers] + data_rows: list[list] = [header] + for d in defenders: + did = d[3] + cells = [] + for a in attackers: + c = confusion.get((did, a[3])) + cells.append(f"({c['tp']},{c['fp']},{c['fn']},{c['tn']})" if c else "") + data_rows.append([_submission_label(d[0], d[1], d[2])] + cells) + + return _csv_response(data_rows, "evaluation_scores_all.csv") + + +@router.get("/export/scores/individual") +def export_individual_evaluation_scores( + defense_submission_id: UUID = Query(...), + attack_submission_id: UUID = Query(...), + _: AuthenticatedUser = Depends(require_admin_user), + db: Session = Depends(get_db), +) -> StreamingResponse: + """Download per-file model output for a single defense/attack pair.""" + rows = db.execute( + text(""" + SELECT af.filename, efr.model_output, af.is_malware + FROM evaluation_pair_scores eps + JOIN evaluation_runs er ON er.id = eps.latest_evaluation_run_id + JOIN evaluation_file_results efr ON efr.evaluation_run_id = er.id + JOIN attack_files af ON af.id = efr.attack_file_id + WHERE eps.defense_submission_id = :def_id + AND eps.attack_submission_id = :atk_id + AND eps.latest_evaluation_run_id IS NOT NULL + ORDER BY af.filename + """), + {"def_id": str(defense_submission_id), "atk_id": str(attack_submission_id)}, + ).fetchall() + + if not rows: + return _csv_response( + [["No evaluation data found for this pair."]], + "evaluation_scores_individual.csv", + ) + + filenames = [r[0] or "unknown" for r in rows] + outputs = [str(r[1]) if r[1] is not None else "" for r in rows] + ground = [str(r[2]) if r[2] is not None else "" for r in rows] + + return _csv_response( + [ + [""] + filenames, + ["Model Output"] + outputs, + ["Is Malware (ground truth)"] + ground, + ], + "evaluation_scores_individual.csv", + ) + + +@router.get("/export/validation-scores") +def export_validation_scores( + _: AuthenticatedUser = Depends(require_admin_user), + db: Session = Depends(get_db), +) -> StreamingResponse: + """Download heuristic-validation result grid: defenses vs validation samples.""" + rows = db.execute( + text(""" + SELECT u.username, s.display_name, s.version, s.id::text, + hs.filename, hfr.model_output + FROM heurval_results hr + JOIN submissions s ON s.id = hr.defense_submission_id + JOIN users u ON u.id = s.user_id + JOIN heurval_file_results hfr ON hfr.heurval_result_id = hr.id + JOIN heurval_samples hs ON hs.id = hfr.sample_id + WHERE s.deleted_at IS NULL + ORDER BY s.created_at, hs.filename + """) + ).fetchall() + + if not rows: + return _csv_response([["No validation scores available."]], "validation_scores.csv") + + sub_order: list[str] = [] + sub_labels: dict[str, str] = {} + cells: dict[str, dict[str, int | None]] = {} + all_files: set[str] = set() + + for r in rows: + sid = r[3] + if sid not in sub_labels: + sub_order.append(sid) + sub_labels[sid] = _submission_label(r[0], r[1], r[2]) + cells[sid] = {} + cells[sid][r[4]] = r[5] + all_files.add(r[4]) + + sorted_files = sorted(all_files) + data_rows: list[list] = [["Defense"] + sorted_files] + for sid in sub_order: + row_cells = [str(cells[sid].get(f, "")) for f in sorted_files] + data_rows.append([sub_labels[sid]] + row_cells) + + return _csv_response(data_rows, "validation_scores.csv") + + +@router.get("/export/behavioral-analysis") +def export_behavioral_analysis( + _: AuthenticatedUser = Depends(require_admin_user), + db: Session = Depends(get_db), +) -> StreamingResponse: + """Download behavioral analysis status grid: attacks vs template files.""" + rows = db.execute( + text(""" + SELECT u.username, s.display_name, s.version, s.id::text, + orig.filename AS template_filename, + af.behavior_status + FROM submissions s + JOIN users u ON u.id = s.user_id + JOIN attack_files af ON af.attack_submission_id = s.id + JOIN attack_files orig ON orig.id = af.original_file_id + WHERE s.submission_type = 'attack' + AND s.deleted_at IS NULL + AND af.original_file_id IS NOT NULL + ORDER BY s.created_at, orig.filename + """) + ).fetchall() + + if not rows: + return _csv_response( + [["No behavioral analysis data available."]], + "behavioral_analysis.csv", + ) + + sub_order: list[str] = [] + sub_labels: dict[str, str] = {} + cells: dict[str, dict[str, str]] = {} + all_files: set[str] = set() + + for r in rows: + sid = r[3] + if sid not in sub_labels: + sub_order.append(sid) + sub_labels[sid] = _submission_label(r[0], r[1], r[2]) + cells[sid] = {} + cells[sid][r[4]] = r[5] or "" + all_files.add(r[4]) + + sorted_files = sorted(all_files) + data_rows: list[list] = [["Attack"] + sorted_files] + for sid in sub_order: + row_cells = [cells[sid].get(f, "") for f in sorted_files] + data_rows.append([sub_labels[sid]] + row_cells) + + return _csv_response(data_rows, "behavioral_analysis.csv") diff --git a/services/frontend/src/components/admin/competition/CompetitionPage.tsx b/services/frontend/src/components/admin/competition/CompetitionPage.tsx index 8b1bee3..1392045 100644 --- a/services/frontend/src/components/admin/competition/CompetitionPage.tsx +++ b/services/frontend/src/components/admin/competition/CompetitionPage.tsx @@ -445,6 +445,201 @@ function ValidationSamplesSection() { ); } +// --------------------------------------------------------------------------- +// Downloads +// --------------------------------------------------------------------------- + +interface UserOption { id: string; username: string; email: string; } +interface SubOption { id: string; display_name: string | null; version: string; status: string; } + +async function triggerCsvDownload(path: string, filename: string): Promise { + const res = await adminFetch(path); + if (!res.ok) throw new Error(`Download failed (${res.status})`); + const blob = await res.blob(); + const url = URL.createObjectURL(blob); + const a = document.createElement('a'); + a.href = url; + a.download = filename; + a.click(); + URL.revokeObjectURL(url); +} + +function DownloadButton({ + label, path, filename, +}: { label: string; path: string; filename: string }) { + const [busy, setBusy] = useState(false); + const [err, setErr] = useState(null); + + async function handle() { + setBusy(true); + setErr(null); + try { + await triggerCsvDownload(path, filename); + } catch (e) { + setErr(e instanceof Error ? e.message : 'Download failed.'); + } finally { + setBusy(false); + } + } + + return ( +
+ + {err && {err}} +
+ ); +} + +function IndividualScoresDownload() { + const [users, setUsers] = useState([]); + const [defUserId, setDefUserId] = useState(''); + const [atkUserId, setAtkUserId] = useState(''); + const [defSubs, setDefSubs] = useState([]); + const [atkSubs, setAtkSubs] = useState([]); + const [defSubId, setDefSubId] = useState(''); + const [atkSubId, setAtkSubId] = useState(''); + const [busy, setBusy] = useState(false); + const [err, setErr] = useState(null); + + useEffect(() => { + adminFetch('/admin/users?limit=200') + .then(r => r.ok ? r.json() : null) + .then(d => { if (d) setUsers(d.items ?? []); }) + .catch(() => {}); + }, []); + + useEffect(() => { + if (!defUserId) { setDefSubs([]); setDefSubId(''); return; } + adminFetch(`/admin/submissions/users/${defUserId}`) + .then(r => r.ok ? r.json() : null) + .then(d => { + setDefSubs((d?.submissions ?? []).filter((s: { submission_type: string }) => s.submission_type === 'defense')); + setDefSubId(''); + }) + .catch(() => {}); + }, [defUserId]); + + useEffect(() => { + if (!atkUserId) { setAtkSubs([]); setAtkSubId(''); return; } + adminFetch(`/admin/submissions/users/${atkUserId}`) + .then(r => r.ok ? r.json() : null) + .then(d => { + setAtkSubs((d?.submissions ?? []).filter((s: { submission_type: string }) => s.submission_type === 'attack')); + setAtkSubId(''); + }) + .catch(() => {}); + }, [atkUserId]); + + async function handle() { + if (!defSubId || !atkSubId) return; + setBusy(true); + setErr(null); + try { + await triggerCsvDownload( + `/admin/export/scores/individual?defense_submission_id=${defSubId}&attack_submission_id=${atkSubId}`, + 'evaluation_scores_individual.csv', + ); + } catch (e) { + setErr(e instanceof Error ? e.message : 'Download failed.'); + } finally { + setBusy(false); + } + } + + const selectCls = "text-xs border border-gray-200 rounded px-2 py-1.5 m-1 text-gray-700 focus:outline-none focus:ring-2 focus:ring-primary/30 disabled:opacity-40 max-w-48"; + + return ( +
+
+

Individual Evaluation Scores

+

Per-file model output for a specific attacker-defender submission pair.

+
+
+
+

Defender

+ + +
+
+

Attacker

+ + +
+
+
+ + {err && {err}} +
+
+ ); +} + +const BULK_EXPORTS: { label: string; description: string; path: string; filename: string }[] = [ + { + label: 'All Evaluation Scores', + description: 'Confusion-matrix results (TP/FP/FN/TN) for every active attacker-defender pair.', + path: '/admin/export/scores/all', + filename: 'evaluation_scores_all.csv', + }, + { + label: 'Defense Validation Scores', + description: 'Per-sample model output for each defense submission across all defense validation samples.', + path: '/admin/export/validation-scores', + filename: 'validation_scores.csv', + }, + { + label: 'Behavioral Analysis', + description: 'Behavior classification status for each attack file relative to its source template.', + path: '/admin/export/behavioral-analysis', + filename: 'behavioral_analysis.csv', + }, +]; + +function DownloadsSection() { + return ( +
+
+ {BULK_EXPORTS.map(({ label, description, path, filename }) => ( +
+
+

{label}

+

{description}

+
+ +
+ ))} +
+ +
+
+
+ ); +} + // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- @@ -456,6 +651,7 @@ export default function CompetitionPage() { +
); } From e08c6ef5c6d6fe55f43bedd870ec347fdb174351 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 19:50:59 -0500 Subject: [PATCH 18/38] Added cooldown, now appears in submission; close #59 --- config.yaml | 4 +- services/api/core/config.py | 8 ++- services/api/core/submission_control.py | 52 +++++++++++++++++++ services/api/routers/submissions.py | 34 +++++++++++- .../src/components/Attack_submit.astro | 49 +++++++++++++++++ .../src/components/Defense_submit.astro | 49 +++++++++++++++++ 6 files changed, 193 insertions(+), 3 deletions(-) diff --git a/config.yaml b/config.yaml index 0a26b30..58c8251 100644 --- a/config.yaml +++ b/config.yaml @@ -20,7 +20,7 @@ worker: heurval_malware_tpr_minimum: 0.30 heurval_goodware_fpr_minimum: 0.0 heurval_goodware_tpr_minimum: 0.30 - reject_heurval_failures: true + reject_heurval_failures: false source: # Resource limits max_zip_size_mb: 512 @@ -56,3 +56,5 @@ worker: application: login_code: 'ABC' + defense_submission_cooldown: 30 # seconds between defense submissions per user; 0 = no cooldown + attack_submission_cooldown: 30 # seconds between attack submissions per user; 0 = no cooldown diff --git a/services/api/core/config.py b/services/api/core/config.py index b4abf24..160a3dd 100644 --- a/services/api/core/config.py +++ b/services/api/core/config.py @@ -24,6 +24,8 @@ class MinIOConfig(BaseModel): class ApplicationConfig(BaseModel): join_code: str | None = None + defense_submission_cooldown: int = 0 + attack_submission_cooldown: int = 0 class AppConfig(BaseModel): @@ -49,7 +51,11 @@ def get_config() -> AppConfig: join_code = app_data.get("login_code") return AppConfig( minio=MinIOConfig(**minio_data), - application=ApplicationConfig(join_code=join_code), + application=ApplicationConfig( + join_code=join_code, + defense_submission_cooldown=int(app_data.get("defense_submission_cooldown", 0)), + attack_submission_cooldown=int(app_data.get("attack_submission_cooldown", 0)), + ), ) except Exception as e: logger.error( diff --git a/services/api/core/submission_control.py b/services/api/core/submission_control.py index ba63152..9cabaa6 100644 --- a/services/api/core/submission_control.py +++ b/services/api/core/submission_control.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math from dataclasses import dataclass from datetime import datetime, timezone @@ -187,3 +188,54 @@ def ensure_submissions_open(db: Session) -> None: status_code=status.HTTP_403_FORBIDDEN, detail="Submissions are closed (deadline passed)", ) + + +def get_cooldown_remaining( + db: Session, + *, + user_id: str, + submission_type: str, + cooldown_seconds: int, +) -> int | None: + """Return remaining cooldown seconds, or None if no cooldown is active.""" + if cooldown_seconds <= 0: + return None + row = db.execute( + text( + """ + SELECT MAX(created_at) + FROM submissions + WHERE user_id = :user_id + AND submission_type = :submission_type + AND deleted_at IS NULL + """ + ), + {"user_id": user_id, "submission_type": submission_type}, + ).fetchone() + if row is None or row[0] is None: + return None + last_submitted = _as_utc(row[0]) + elapsed = (_utcnow() - last_submitted).total_seconds() + remaining = cooldown_seconds - elapsed + return math.ceil(remaining) if remaining > 0 else None + + +def check_cooldown( + db: Session, + *, + user_id: str, + submission_type: str, + cooldown_seconds: int, +) -> None: + """Raise HTTP 429 if the user submitted within the cooldown window.""" + remaining = get_cooldown_remaining( + db, + user_id=user_id, + submission_type=submission_type, + cooldown_seconds=cooldown_seconds, + ) + if remaining: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail=f"Please wait {remaining} seconds before submitting again.", + ) diff --git a/services/api/routers/submissions.py b/services/api/routers/submissions.py index 293189a..15d43b4 100644 --- a/services/api/routers/submissions.py +++ b/services/api/routers/submissions.py @@ -24,7 +24,8 @@ validate_github_url_format, validate_semver_format, ) -from core.submission_control import ensure_submissions_open +from core.config import get_config +from core.submission_control import check_cooldown, ensure_submissions_open, get_cooldown_remaining from routers.queue import _insert_job, _publish_task from schemas.jobs import JobType from schemas.submissions import ( @@ -106,6 +107,7 @@ def create_defense_docker( """ # Enforce admin-controlled submission window before accepting new work. ensure_submissions_open(db) + check_cooldown(db, user_id=str(current_user.user_id), submission_type='defense', cooldown_seconds=get_config().application.defense_submission_cooldown) # 1. Validate format validate_docker_image_format(req.docker_image) validate_semver_format(req.version) @@ -198,6 +200,7 @@ def create_defense_github( """ # Enforce admin-controlled submission window before accepting new work. ensure_submissions_open(db) + check_cooldown(db, user_id=str(current_user.user_id), submission_type='defense', cooldown_seconds=get_config().application.defense_submission_cooldown) # 1. Validate format validate_github_url_format(req.git_repo) validate_semver_format(req.version) @@ -292,6 +295,7 @@ async def create_defense_zip( """ # Enforce admin-controlled submission window before accepting new work. ensure_submissions_open(db) + check_cooldown(db, user_id=str(current_user.user_id), submission_type='defense', cooldown_seconds=get_config().application.defense_submission_cooldown) settings = get_settings() # 1. Validate file @@ -448,6 +452,7 @@ async def create_attack_zip( """ # Enforce admin-controlled submission window before accepting new work. ensure_submissions_open(db) + check_cooldown(db, user_id=str(current_user.user_id), submission_type='attack', cooldown_seconds=get_config().application.attack_submission_cooldown) settings = get_settings() # 1. Validate file @@ -745,3 +750,30 @@ def set_active_submission( submission_id=submission_id, submission_type=sub_type, ) + + +@router.get("/cooldown") +def get_submission_cooldown( + current_user: AuthenticatedUser = Depends(get_authenticated_user), + db: Session = Depends(get_db), +) -> dict: + """Return remaining cooldown seconds for each submission type. + + A null value means no cooldown is currently active. + """ + cfg = get_config() + user_id = str(current_user.user_id) + return { + "defense_remaining_seconds": get_cooldown_remaining( + db, + user_id=user_id, + submission_type="defense", + cooldown_seconds=cfg.application.defense_submission_cooldown, + ), + "attack_remaining_seconds": get_cooldown_remaining( + db, + user_id=user_id, + submission_type="attack", + cooldown_seconds=cfg.application.attack_submission_cooldown, + ), + } diff --git a/services/frontend/src/components/Attack_submit.astro b/services/frontend/src/components/Attack_submit.astro index b11f058..91ec387 100644 --- a/services/frontend/src/components/Attack_submit.astro +++ b/services/frontend/src/components/Attack_submit.astro @@ -4,6 +4,8 @@

Attack Submission

+ +
@@ -103,6 +105,53 @@ el.appendChild(btn); } + (function initAttackCooldown() { + const cooldownEl = document.getElementById('attack-cooldown') as HTMLParagraphElement; + let timer: ReturnType | null = null; + + function startCountdown(seconds: number) { + let remaining = seconds; + if (timer) clearInterval(timer); + + function tick() { + if (remaining <= 0) { + cooldownEl.classList.add('hidden'); + cooldownEl.textContent = ''; + if (timer) clearInterval(timer); + return; + } + const mm = String(Math.floor(remaining / 60)).padStart(2, '0'); + const ss = String(remaining % 60).padStart(2, '0'); + cooldownEl.textContent = `You can submit again in ${mm}:${ss}`; + cooldownEl.classList.remove('hidden'); + remaining--; + } + + tick(); + timer = setInterval(tick, 1000); + } + + async function checkCooldown() { + try { + const res = await fetch('/api/submissions/cooldown'); + if (!res.ok) return; + const data = await res.json(); + const secs: number | null = data.attack_remaining_seconds; + if (secs && secs > 0) { + startCountdown(secs); + } + } catch {} + } + + checkCooldown(); + + document.addEventListener('submission-created', (e: Event) => { + if ((e as CustomEvent).detail?.type === 'attack') { + checkCooldown(); + } + }); + })(); + document.getElementById('attack-form')!.addEventListener('submit', async (e) => { e.preventDefault(); diff --git a/services/frontend/src/components/Defense_submit.astro b/services/frontend/src/components/Defense_submit.astro index 2391c16..4724f8a 100644 --- a/services/frontend/src/components/Defense_submit.astro +++ b/services/frontend/src/components/Defense_submit.astro @@ -4,6 +4,8 @@

Model Submission

+ +
@@ -159,6 +161,53 @@ el.appendChild(btn); } + (function initDefenseCooldown() { + const cooldownEl = document.getElementById('defense-cooldown') as HTMLParagraphElement; + let timer: ReturnType | null = null; + + function startCountdown(seconds: number) { + let remaining = seconds; + if (timer) clearInterval(timer); + + function tick() { + if (remaining <= 0) { + cooldownEl.classList.add('hidden'); + cooldownEl.textContent = ''; + if (timer) clearInterval(timer); + return; + } + const mm = String(Math.floor(remaining / 60)).padStart(2, '0'); + const ss = String(remaining % 60).padStart(2, '0'); + cooldownEl.textContent = `You can submit again in ${mm}:${ss}`; + cooldownEl.classList.remove('hidden'); + remaining--; + } + + tick(); + timer = setInterval(tick, 1000); + } + + async function checkCooldown() { + try { + const res = await fetch('/api/submissions/cooldown'); + if (!res.ok) return; + const data = await res.json(); + const secs: number | null = data.defense_remaining_seconds; + if (secs && secs > 0) { + startCountdown(secs); + } + } catch {} + } + + checkCooldown(); + + document.addEventListener('submission-created', (e: Event) => { + if ((e as CustomEvent).detail?.type === 'defense') { + checkCooldown(); + } + }); + })(); + document.getElementById('defense-form')!.addEventListener('submit', async (e) => { e.preventDefault(); From 5b8d74ec21d048ab9658991c0c4e812f0ea5a55e Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 19:59:28 -0500 Subject: [PATCH 19/38] Added more info to admin job logs. It's not too much- idk specifically what Ali wants, but I think this should be good; close #64 --- services/api/routers/admin.py | 141 ++++++++++++++ services/api/schemas/admin.py | 22 +++ .../src/components/admin/logs/LogsPage.tsx | 179 ++++++++++++++++-- 3 files changed, 323 insertions(+), 19 deletions(-) diff --git a/services/api/routers/admin.py b/services/api/routers/admin.py index df5dfbc..ff13070 100644 --- a/services/api/routers/admin.py +++ b/services/api/routers/admin.py @@ -55,6 +55,9 @@ AdminEvaluationPairRecord, AdminSubmissionEvaluationsResponse, AdminActivateSubmissionResponse, + JobDetailResponse, + JobDetailSubmission, + JobDetailEvalRun, ) router = APIRouter( @@ -463,6 +466,144 @@ def get_recent_jobs( return AdminJobLogsResponse(count=len(items), items=items) +@router.get("/logs/jobs/{job_id}/detail", response_model=JobDetailResponse) +def get_job_detail( + job_id: str, + _: AuthenticatedUser = Depends(require_admin_user), + db: Session = Depends(get_db), +) -> JobDetailResponse: + """Return extended detail for a single job record.""" + row = ( + db.execute( + text( + """ + SELECT id, job_type, status, requested_by_user_id, payload, created_at, updated_at + FROM jobs + WHERE id = :id + """ + ), + {"id": job_id}, + ) + .mappings() + .fetchone() + ) + if row is None: + raise HTTPException(status_code=404, detail="Job not found") + + job = AdminJobLogRecord(**row) + payload = row["payload"] or {} + submission: JobDetailSubmission | None = None + eval_runs: list[JobDetailEvalRun] = [] + + if job.job_type == "D": + sub_id = payload.get("defense_submission_id") + if sub_id: + sub_row = ( + db.execute( + text( + """ + SELECT s.id, s.version, s.display_name, s.status, + d.source_type + FROM submissions s + LEFT JOIN defense_submission_details d ON d.submission_id = s.id + WHERE s.id = :id + """ + ), + {"id": sub_id}, + ) + .mappings() + .fetchone() + ) + if sub_row: + submission = JobDetailSubmission( + submission_id=str(sub_row["id"]), + version=sub_row["version"], + display_name=sub_row["display_name"], + status=sub_row["status"], + source_type=sub_row["source_type"], + ) + run_rows = ( + db.execute( + text( + """ + SELECT id, attack_submission_id, status, duration_ms + FROM evaluation_runs + WHERE defense_submission_id = :id + ORDER BY created_at DESC + LIMIT 10 + """ + ), + {"id": sub_id}, + ) + .mappings() + .fetchall() + ) + eval_runs = [ + JobDetailEvalRun( + id=str(r["id"]), + counterpart_id=str(r["attack_submission_id"]), + status=r["status"], + duration_ms=r["duration_ms"], + ) + for r in run_rows + ] + + elif job.job_type == "A": + sub_id = payload.get("attack_submission_id") + if sub_id: + sub_row = ( + db.execute( + text( + """ + SELECT s.id, s.version, s.display_name, s.status, + a.file_count + FROM submissions s + LEFT JOIN attack_submission_details a ON a.submission_id = s.id + WHERE s.id = :id + """ + ), + {"id": sub_id}, + ) + .mappings() + .fetchone() + ) + if sub_row: + submission = JobDetailSubmission( + submission_id=str(sub_row["id"]), + version=sub_row["version"], + display_name=sub_row["display_name"], + status=sub_row["status"], + file_count=sub_row["file_count"], + ) + run_rows = ( + db.execute( + text( + """ + SELECT id, defense_submission_id, status, duration_ms + FROM evaluation_runs + WHERE attack_submission_id = :id + ORDER BY created_at DESC + LIMIT 10 + """ + ), + {"id": sub_id}, + ) + .mappings() + .fetchall() + ) + eval_runs = [ + JobDetailEvalRun( + id=str(r["id"]), + counterpart_id=str(r["defense_submission_id"]), + status=r["status"], + duration_ms=r["duration_ms"], + ) + for r in run_rows + ] + + return JobDetailResponse(job=job, submission=submission, evaluation_runs=eval_runs) + + @router.get("/logs/evaluations", response_model=AdminEvaluationLogsResponse) def get_recent_evaluations( _: AuthenticatedUser = Depends(require_admin_user), diff --git a/services/api/schemas/admin.py b/services/api/schemas/admin.py index 07d7a5a..d39f2fd 100644 --- a/services/api/schemas/admin.py +++ b/services/api/schemas/admin.py @@ -38,6 +38,28 @@ class AdminJobLogsResponse(BaseModel): items: list[AdminJobLogRecord] +class JobDetailSubmission(BaseModel): + submission_id: str + version: str + display_name: str | None + status: str + source_type: str | None = None + file_count: int | None = None + + +class JobDetailEvalRun(BaseModel): + id: str + counterpart_id: str + status: str | None + duration_ms: int | None + + +class JobDetailResponse(BaseModel): + job: AdminJobLogRecord + submission: JobDetailSubmission | None + evaluation_runs: list[JobDetailEvalRun] + + class AdminEvaluationLogRecord(BaseModel): id: UUID defense_submission_id: UUID diff --git a/services/frontend/src/components/admin/logs/LogsPage.tsx b/services/frontend/src/components/admin/logs/LogsPage.tsx index cb51809..28abc2b 100644 --- a/services/frontend/src/components/admin/logs/LogsPage.tsx +++ b/services/frontend/src/components/admin/logs/LogsPage.tsx @@ -1,4 +1,4 @@ -import { useState, useEffect, useRef } from 'react'; +import { useState, useEffect, useRef, Fragment } from 'react'; import { adminFetch } from '../../../lib/adminApi'; type Tab = 'audit' | 'jobs' | 'evaluations' | 'sessions'; @@ -218,13 +218,103 @@ interface JobRecord { updated_at: string; } +interface JobDetailSub { + submission_id: string; + version: string; + display_name: string | null; + status: string; + source_type?: string | null; + file_count?: number | null; +} + +interface JobDetailRun { + id: string; + counterpart_id: string; + status: string | null; + duration_ms: number | null; +} + +interface JobDetail { + submission: JobDetailSub | null; + evaluation_runs: JobDetailRun[]; + fetching?: boolean; +} + const JOB_STATUSES = ['queued', 'running', 'done', 'failed']; +function JobDetailPanel({ detail, jobType }: { detail: JobDetail; jobType: string }) { + if (detail.fetching) { + return

Loading details...

; + } + const { submission, evaluation_runs } = detail; + const counterpartLabel = jobType === 'D' ? 'Attack' : 'Defense'; + return ( +
+
+

Submission

+ {submission ? ( +
+ {submission.display_name && ( + <> +
Name
+
{submission.display_name}
+ + )} +
Version
+
{submission.version}
+
Status
+
+ {jobType === 'D' && submission.source_type && ( + <> +
Source
+
{submission.source_type}
+ + )} + {jobType === 'A' && submission.file_count != null && ( + <> +
Files
+
{submission.file_count}
+ + )} +
+ ) : ( +

No submission data.

+ )} +
+ {evaluation_runs.length > 0 && ( +
+

Evaluation Runs

+ + + + + + + + + + {evaluation_runs.map(r => ( + + + + + + ))} + +
{counterpartLabel}StatusDuration
{shortId(r.counterpart_id)}{r.duration_ms != null ? `${r.duration_ms} ms` : '-'}
+
+ )} +
+ ); +} + function JobsTab() { - const [records, setRecords] = useState([]); - const [loading, setLoading] = useState(true); - const [statusFilter, setStatus] = useState(''); - const [limit, setLimit] = useState(50); + const [records, setRecords] = useState([]); + const [loading, setLoading] = useState(true); + const [statusFilter, setStatus] = useState(''); + const [limit, setLimit] = useState(50); + const [expanded, setExpanded] = useState>(new Set()); + const [details, setDetails] = useState>({}); const first = useRef(true); async function load(s: string, lim: number) { @@ -244,6 +334,29 @@ function JobsTab() { return () => clearTimeout(t); }, [statusFilter, limit]); + async function toggleExpand(id: string) { + setExpanded(prev => { + const next = new Set(prev); + next.has(id) ? next.delete(id) : next.add(id); + return next; + }); + if (details[id]) return; + setDetails(prev => ({ ...prev, [id]: { submission: null, evaluation_runs: [], fetching: true } })); + try { + const res = await adminFetch(`/admin/logs/jobs/${id}/detail`); + const d = res.ok ? await res.json() : null; + setDetails(prev => ({ + ...prev, + [id]: { + submission: d?.submission ?? null, + evaluation_runs: d?.evaluation_runs ?? [], + }, + })); + } catch { + setDetails(prev => ({ ...prev, [id]: { submission: null, evaluation_runs: [] } })); + } + } + return ( <> load(statusFilter, limit)}> @@ -258,23 +371,51 @@ function JobsTab() { - {records.map(r => ( - - {formatDateTime(r.created_at)} - - - {r.job_type === 'D' ? 'Defense' : r.job_type === 'A' ? 'Attack' : r.job_type === 'S' ? 'Seeding' : r.job_type} - - - - {shortId(r.requested_by_user_id)} - {formatDateTime(r.updated_at)} - - ))} + {records.map(r => { + const isOpen = expanded.has(r.id); + const detail = details[r.id]; + return ( + + + + + + {formatDateTime(r.created_at)} + + + {r.job_type === 'D' ? 'Defense' : r.job_type === 'A' ? 'Attack' : r.job_type === 'S' ? 'Seeding' : r.job_type} + + + + {shortId(r.requested_by_user_id)} + {formatDateTime(r.updated_at)} + + {isOpen && ( + + + {detail ? ( + + ) : ( +

Loading details...

+ )} + + + )} + + ); + })} ); From 15eb7c299ab0670811ddee85d893f10165780376 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 20:10:54 -0500 Subject: [PATCH 20/38] Added more information to submission dropdowns (hash, time, source, etc.) --- services/api/routers/submissions.py | 44 ++++++ services/api/schemas/submissions.py | 11 ++ .../src/components/SubmissionHistory.tsx | 125 ++++++++++++------ 3 files changed, 141 insertions(+), 39 deletions(-) diff --git a/services/api/routers/submissions.py b/services/api/routers/submissions.py index 15d43b4..b1d0378 100644 --- a/services/api/routers/submissions.py +++ b/services/api/routers/submissions.py @@ -32,6 +32,7 @@ CreateDefenseDockerRequest, CreateDefenseGitHubRequest, SetActiveResponse, + SubmissionDetailResponse, SubmissionListItem, SubmissionHistoryResponse, SubmissionResponse, @@ -752,6 +753,49 @@ def set_active_submission( ) +@router.get("/{submission_id}/detail", response_model=SubmissionDetailResponse) +def get_submission_detail( + submission_id: str, + current_user: AuthenticatedUser = Depends(get_authenticated_user), + db: Session = Depends(get_db), +) -> SubmissionDetailResponse: + """Return source metadata for a submission belonging to the authenticated user.""" + row = db.execute( + text( + """ + SELECT + s.id, + s.submission_type, + s.created_at, + d.source_type, + d.sha256, + d.docker_image, + d.git_repo, + a.zip_sha256 + FROM submissions s + LEFT JOIN defense_submission_details d ON d.submission_id = s.id + LEFT JOIN attack_submission_details a ON a.submission_id = s.id + WHERE s.id = :id + AND s.user_id = :user_id + AND s.deleted_at IS NULL + """ + ), + {"id": submission_id, "user_id": str(current_user.user_id)}, + ).mappings().fetchone() + + if row is None: + raise HTTPException(status_code=404, detail="Submission not found") + + return SubmissionDetailResponse( + submission_id=str(row["id"]), + created_at=row["created_at"].isoformat(), + source_type=row["source_type"], + sha256=row["sha256"] or row["zip_sha256"], + docker_image=row["docker_image"], + git_repo=row["git_repo"], + ) + + @router.get("/cooldown") def get_submission_cooldown( current_user: AuthenticatedUser = Depends(get_authenticated_user), diff --git a/services/api/schemas/submissions.py b/services/api/schemas/submissions.py index ea736a2..f0632ec 100644 --- a/services/api/schemas/submissions.py +++ b/services/api/schemas/submissions.py @@ -113,3 +113,14 @@ class SubmissionHistoryResponse(BaseModel): total: int limit: int offset: int + + +class SubmissionDetailResponse(BaseModel): + """Source metadata for a single submission, fetched on demand.""" + + submission_id: str + created_at: str + source_type: str | None + sha256: str | None + docker_image: str | None + git_repo: str | None diff --git a/services/frontend/src/components/SubmissionHistory.tsx b/services/frontend/src/components/SubmissionHistory.tsx index 1bd79e3..df42e6b 100644 --- a/services/frontend/src/components/SubmissionHistory.tsx +++ b/services/frontend/src/components/SubmissionHistory.tsx @@ -12,6 +12,11 @@ interface Submission { is_active: boolean; heurval_tpr: number | null; heurval_fpr: number | null; + detail_loaded?: boolean; + source_type?: string | null; + sha256?: string | null; + docker_image?: string | null; + git_repo?: string | null; } interface Props { @@ -45,6 +50,16 @@ function formatDate(iso: string): string { }); } +function formatDateTime(iso: string): string { + return new Date(iso).toLocaleString(undefined, { + month: 'short', + day: 'numeric', + year: 'numeric', + hour: '2-digit', + minute: '2-digit', + }); +} + function StarIcon({ filled }: { filled: boolean }) { if (filled) { return ( @@ -115,44 +130,64 @@ function ExpandedDetail({ sub }: { sub: Submission }) { const { status, functional_error, submission_type, heurval_tpr, heurval_fpr } = sub; const showHeurval = submission_type === 'defense' && (heurval_tpr !== null || heurval_fpr !== null); - if (status === 'submitted') { - return

Queued for processing.

; - } - if (status === 'validating') { - return

Validation in progress.

; - } - if (status === 'validated') { - return ( - <> -

Validation passed. Waiting for evaluation.

- {showHeurval && } - - ); - } - if (status === 'evaluating') { - return ( - <> -

Currently being evaluated.

- {showHeurval && } - - ); - } - if (status === 'evaluated') { - return ( - <> -

Evaluation complete.

- {showHeurval && } - - ); - } - if (status === 'error') { - return ( -

- {functional_error ?? 'An error occurred during processing.'} -

- ); - } - return null; + return ( + <> + {status === 'submitted' &&

Queued for processing.

} + {status === 'validating' &&

Validation in progress.

} + {(status === 'validated') && ( + <> +

Validation passed. Waiting for evaluation.

+ {showHeurval && } + + )} + {status === 'evaluating' && ( + <> +

Currently being evaluated.

+ {showHeurval && } + + )} + {status === 'evaluated' && ( + <> +

Evaluation complete.

+ {showHeurval && } + + )} + {status === 'error' && ( +

+ {functional_error ?? 'An error occurred during processing.'} +

+ )} + + {sub.detail_loaded ? ( +
+
Submitted
+
{formatDateTime(sub.created_at)}
+ {sub.sha256 && ( + <> +
File Hash
+
+ {sub.sha256.slice(0, 16)}... +
+ + )} + {sub.docker_image && ( + <> +
DockerHub
+
{sub.docker_image}
+ + )} + {sub.git_repo && ( + <> +
GitHub
+
{sub.git_repo}
+ + )} +
+ ) : ( +

Loading details...

+ )} + + ); } export default function SubmissionHistory({ type, title }: Props) { @@ -205,12 +240,24 @@ export default function SubmissionHistory({ type, title }: Props) { ); }; - const toggleExpanded = (id: string) => { + const toggleExpanded = async (id: string) => { setExpanded(prev => { const next = new Set(prev); next.has(id) ? next.delete(id) : next.add(id); return next; }); + const sub = submissions.find(s => s.submission_id === id); + if (!sub || sub.detail_loaded) return; + try { + const res = await fetch(`/api/submissions/${id}/detail`); + if (!res.ok) return; + const d = await res.json(); + setSubmissions(prev => prev.map(s => + s.submission_id === id + ? { ...s, detail_loaded: true, source_type: d.source_type, sha256: d.sha256, docker_image: d.docker_image, git_repo: d.git_repo } + : s + )); + } catch {} }; return ( From 0d2150289a9c333f5fd19543545214671861532e Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 20:20:56 -0500 Subject: [PATCH 21/38] Changed 'Submit Model' to 'Submit Defense', changed color gradient to colorblind-safe blue/orange. --- .../src/components/Defense_submit.astro | 6 +++--- .../src/components/EvaluationMatrix.tsx | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/services/frontend/src/components/Defense_submit.astro b/services/frontend/src/components/Defense_submit.astro index 4724f8a..b8ee86a 100644 --- a/services/frontend/src/components/Defense_submit.astro +++ b/services/frontend/src/components/Defense_submit.astro @@ -2,7 +2,7 @@ ---
-

Model Submission

+

Defense Submission

@@ -97,7 +97,7 @@ type="submit" class="w-full bg-primary hover:bg-secondary text-white font-semibold py-2.5 rounded-lg transition duration-200" > - Submit Model + Submit Defense @@ -267,7 +267,7 @@ _attachDismiss(feedback); } finally { btn.disabled = false; - btn.textContent = 'Submit Model'; + btn.textContent = 'Submit Defense'; } }); diff --git a/services/frontend/src/components/EvaluationMatrix.tsx b/services/frontend/src/components/EvaluationMatrix.tsx index 7f4f484..52064ce 100644 --- a/services/frontend/src/components/EvaluationMatrix.tsx +++ b/services/frontend/src/components/EvaluationMatrix.tsx @@ -24,28 +24,28 @@ interface LeaderboardData { /** * Maps a score (0.0 to 1.0) to an RGB color. - * 0% = red rgb(220, 50, 50) - * 50% = white rgb(255, 255, 255) - * 100% = green rgb(50, 175, 80) + * 0% = orange rgb(254, 179, 56) + * 50% = white rgb(230, 230, 230) + * 100% = blue rgb(2, 81, 150) */ function scoreToColor(score: number): string { const s = Math.max(0, Math.min(1, score)); if (s <= 0.5) { const t = s / 0.5; - const r = Math.round(220 + (255 - 220) * t); - const g = Math.round(50 + (255 - 50) * t); - const b = Math.round(50 + (255 - 50) * t); + const r = Math.round(254 + (230 - 254) * t); + const g = Math.round(179 + (230 - 179) * t); + const b = Math.round(56 + (230 - 56) * t); return `rgb(${r},${g},${b})`; } const t = (s - 0.5) / 0.5; - const r = Math.round(255 + (50 - 255) * t); - const g = Math.round(255 + (175 - 255) * t); - const b = Math.round(255 + (80 - 255) * t); + const r = Math.round(230 + (2 - 230) * t); + const g = Math.round(230 + (81 - 230) * t); + const b = Math.round(230 + (150 - 230) * t); return `rgb(${r},${g},${b})`; } function textColorForScore(score: number): string { - return score < 0.35 || score > 0.65 ? '#ffffff' : '#374151'; + return score > 0.70 ? '#ffffff' : '#374151'; } export default function EvaluationMatrix() { From f952a5b40f8487ec4686cb0c12e5292be17a381e Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 6 Apr 2026 20:37:14 -0500 Subject: [PATCH 22/38] Fixed github [object Object] issue, added more error response handling, put format validation inside of dockerhub link; close #53 --- services/api/core/submissions.py | 25 ++++++------------- services/api/schemas/submissions.py | 2 +- .../src/components/Attack_submit.astro | 10 +++++++- .../src/components/Defense_submit.astro | 10 +++++++- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/services/api/core/submissions.py b/services/api/core/submissions.py index 6769b26..d2141d7 100644 --- a/services/api/core/submissions.py +++ b/services/api/core/submissions.py @@ -45,32 +45,23 @@ def require_submission_of_type( def validate_docker_image_format(image: str) -> None: """ - Validate Docker image URL/name format. + Validate Docker image reference format. Accepts formats like: + - nginx - nginx:latest - - user/repo:tag - - hub.docker.com/r/user/image - - registry.io/project/image:tag + - user/image:tag + - registry.io/user/image:tag Raises: HTTPException(400): If format is invalid """ image = image.strip() - if not image: + pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._\-/]*(:[a-zA-Z0-9._\-]+)?$" + if not re.match(pattern, image): raise HTTPException( - status_code=400, detail="Docker image cannot be empty") - - # Basic validation - detailed validation happens in worker - # Just check for obviously bad patterns - if " " in image: - raise HTTPException( - status_code=400, detail="Docker image name cannot contain spaces" - ) - - if image.startswith("-") or image.endswith("-"): - raise HTTPException( - status_code=400, detail="Docker image name cannot start or end with dash" + status_code=400, + detail="Invalid Docker image format. Expected: image, image:tag, user/image:tag, or registry/path:tag", ) diff --git a/services/api/schemas/submissions.py b/services/api/schemas/submissions.py index f0632ec..5bdb311 100644 --- a/services/api/schemas/submissions.py +++ b/services/api/schemas/submissions.py @@ -11,7 +11,7 @@ class CreateDefenseDockerRequest(BaseModel): """Request schema for Docker Hub defense submission.""" - docker_image: str = Field(..., min_length=1, max_length=500) + docker_image: str = Field(..., pattern=r"^[a-zA-Z0-9][a-zA-Z0-9._\-/]*(:[a-zA-Z0-9._\-]+)?$", max_length=500) version: str = Field(..., pattern=r"^\d+\.\d+\.\d+$") display_name: str | None = Field(None, max_length=200) diff --git a/services/frontend/src/components/Attack_submit.astro b/services/frontend/src/components/Attack_submit.astro index 91ec387..0c8cd99 100644 --- a/services/frontend/src/components/Attack_submit.astro +++ b/services/frontend/src/components/Attack_submit.astro @@ -92,6 +92,14 @@ } }); + function _extractDetail(detail: unknown): string { + if (typeof detail === 'string') return detail; + if (Array.isArray(detail) && detail.length > 0) { + return (detail as { msg?: string }[]).map(e => e.msg ?? String(e)).join('; '); + } + return 'An unexpected error occurred.'; + } + function _attachDismiss(el: HTMLElement) { const btn = document.createElement('button'); btn.type = 'button'; @@ -181,7 +189,7 @@ const res = await fetch('/api/submissions/attack/zip', { method: 'POST', body: fd }); const data = await res.json(); - if (!res.ok) throw new Error(data.detail ?? `Error ${res.status}`); + if (!res.ok) throw new Error(_extractDetail(data.detail) || `Error ${res.status}`); feedback.textContent = `Submitted (ID: ${data.submission_id})`; feedback.className = 'flex items-center justify-between text-sm rounded-lg px-3 py-2 bg-green-50 text-green-700 border border-green-200'; diff --git a/services/frontend/src/components/Defense_submit.astro b/services/frontend/src/components/Defense_submit.astro index b8ee86a..c959470 100644 --- a/services/frontend/src/components/Defense_submit.astro +++ b/services/frontend/src/components/Defense_submit.astro @@ -148,6 +148,14 @@ diff --git a/services/frontend/src/components/admin/AdminLayout.astro b/services/frontend/src/components/admin/AdminLayout.astro index 0675bc4..19491ef 100644 --- a/services/frontend/src/components/admin/AdminLayout.astro +++ b/services/frontend/src/components/admin/AdminLayout.astro @@ -24,7 +24,7 @@ const navLinks = [ - {title} | MLSEC Admin + MLSEC 2.0 Admin - {title} diff --git a/services/frontend/src/pages/index.astro b/services/frontend/src/pages/index.astro index f754c21..17be251 100644 --- a/services/frontend/src/pages/index.astro +++ b/services/frontend/src/pages/index.astro @@ -12,7 +12,7 @@ import { Content } from '../content/home.md'; - MLSEC + MLSEC 2.0 diff --git a/services/frontend/src/pages/leaderboard.astro b/services/frontend/src/pages/leaderboard.astro index 4efffa8..54970a9 100644 --- a/services/frontend/src/pages/leaderboard.astro +++ b/services/frontend/src/pages/leaderboard.astro @@ -11,7 +11,7 @@ import EvaluationMatrix from "../components/EvaluationMatrix"; - MLSEC Leaderboard + MLSEC 2.0 - Leaderboard diff --git a/services/frontend/src/pages/rules.astro b/services/frontend/src/pages/rules.astro index 5c41d95..2550b92 100644 --- a/services/frontend/src/pages/rules.astro +++ b/services/frontend/src/pages/rules.astro @@ -12,7 +12,7 @@ import { Content } from '../content/rules.md'; - MLSEC Rules + MLSEC 2.0 - Rules diff --git a/services/frontend/src/pages/submission.astro b/services/frontend/src/pages/submission.astro index 9853ea9..a25d594 100644 --- a/services/frontend/src/pages/submission.astro +++ b/services/frontend/src/pages/submission.astro @@ -16,7 +16,7 @@ const session = await getSession(Astro.request); - MLSEC Submission + MLSEC 2.0 - Submission From f53b5cab4ecb6a7fe0dbdf81dde46e0549c7e7b4 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Sun, 12 Apr 2026 13:14:32 -0500 Subject: [PATCH 34/38] Adjusted eval page to be more mobile friendly, added labels to axes --- .../src/components/EvaluationMatrix.tsx | 46 +++++++++++++++---- services/frontend/src/pages/leaderboard.astro | 6 +-- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/services/frontend/src/components/EvaluationMatrix.tsx b/services/frontend/src/components/EvaluationMatrix.tsx index 52064ce..5edec8d 100644 --- a/services/frontend/src/components/EvaluationMatrix.tsx +++ b/services/frontend/src/components/EvaluationMatrix.tsx @@ -129,22 +129,35 @@ export default function EvaluationMatrix() {
+ {/* Axis label row */} - + + )} + + {/* Column headers row */} + + ))} @@ -153,16 +166,29 @@ export default function EvaluationMatrix() { {defenders.map((def, di) => ( + {di === 0 && ( + + )} {attackers.map(atk => { @@ -174,7 +200,7 @@ export default function EvaluationMatrix() { return (
- Defense \ Attack - + + {attackers.length > 0 && ( + + Attack +
+ {attackers.map(atk => (
{atk.username}
{atk.display_name && ( -
{atk.display_name}
+
{atk.display_name}
)} -
v{atk.version}
+
v{atk.version}
+ + Defense + +
{def.username}
{def.display_name && ( -
{def.display_name}
+
{def.display_name}
)} -
v{def.version}
+
v{def.version}
 -
-
-

Evaluation Matrix

+
+
+

Evaluation Matrix

From b3cefd1e9ca1ad14d40df7906e06ef12cd7b3803 Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 13 Apr 2026 13:11:13 -0500 Subject: [PATCH 35/38] Password overhaul, default passwords are listed in .env. Fixed some css stuff in leaderboard --- .env-example | 22 ++++--- .github/workflows/api_ci.yaml | 4 +- .github/workflows/worker_ci.yaml | 8 +-- config.yaml | 2 - docker-compose.yaml | 32 ++++----- services/api/core/config.py | 4 +- services/api/core/database.py | 2 +- services/api/core/settings.py | 4 +- services/api/core/storage.py | 4 +- services/api/tests/conftest.py | 2 +- .../src/components/EvaluationMatrix.tsx | 65 +++++++++---------- services/nginx/nginx.conf | 4 +- services/worker/tests/conftest.py | 16 ++--- .../worker/tests/manual_api_queue_test.py | 2 +- .../tests/manual_test_defense_pipeline.py | 8 +-- services/worker/tests/test_db_results.py | 2 +- services/worker/worker/config.py | 4 +- 17 files changed, 93 insertions(+), 92 deletions(-) diff --git a/.env-example b/.env-example index 1dfe3a3..890e663 100644 --- a/.env-example +++ b/.env-example @@ -1,16 +1,20 @@ # PostgreSQL -# (Currently hardcoded in docker-compose.yaml) +POSTGRES_USER=mlsec2 +POSTGRES_PASSWORD=mlsec2_pw -# MinIO -MINIO_ROOT_USER=minioadmin -MINIO_ROOT_PASSWORD=minioadmin +# MinIO Object Storage +MINIO_ROOT_USER=mlsec2 +MINIO_ROOT_PASSWORD=mlsec2_pw -# Docker Configuration -DOCKER_GID=999 +# RabbitMQ +RABBITMQ_USER=mlsec2 +RABBITMQ_PASSWORD=mlsec2_pw # Gateway Secret -# (Currently hardcoded in docker-compose.yaml, could be moved here if needed) -GATEWAY_SECRET=welovemarcus +GATEWAY_SECRET=mlsec2_pw + +# Docker Configuration +DOCKER_GID=999 -# VirusTotal API key (for attack similarity evaluation) +# VirusTotal API key (for attack behavioral evaluation) VIRUSTOTAL_API_KEY= diff --git a/.github/workflows/api_ci.yaml b/.github/workflows/api_ci.yaml index 38caca6..1653289 100644 --- a/.github/workflows/api_ci.yaml +++ b/.github/workflows/api_ci.yaml @@ -20,7 +20,7 @@ jobs: --health-retries 5 env: - DATABASE_URL: postgresql://postgres:password123@localhost:5433/mlsec_test + DATABASE_URL: postgresql://mlsec2:mlsec2_pw@localhost:5433/mlsec_test REDIS_URL: redis://localhost:6379/0 steps: @@ -33,7 +33,7 @@ jobs: - name: Wait for test database readiness run: | for i in {1..30}; do - if docker exec test-postgres-db pg_isready -U postgres -d mlsec_test; then + if docker exec test-postgres-db pg_isready -U mlsec2 -d mlsec_test; then exit 0 fi sleep 2 diff --git a/.github/workflows/worker_ci.yaml b/.github/workflows/worker_ci.yaml index 8227200..a33ca2e 100644 --- a/.github/workflows/worker_ci.yaml +++ b/.github/workflows/worker_ci.yaml @@ -9,11 +9,11 @@ jobs: runs-on: ubuntu-latest env: - DATABASE_URL: postgresql://postgres:password123@localhost:5433/mlsec_test + DATABASE_URL: postgresql://mlsec2:mlsec2_pw@localhost:5433/mlsec_test REDIS_URL: redis://localhost:6379/0 MINIO_ENDPOINT: localhost:9000 - MINIO_ACCESS_KEY: minioadmin - MINIO_SECRET_KEY: minioadmin + MINIO_ACCESS_KEY: mlsec2 + MINIO_SECRET_KEY: mlsec2_pw CELERY_BROKER_URL: redis://localhost:6379/1 CELERY_RESULT_BACKEND: redis://localhost:6379/2 @@ -39,7 +39,7 @@ jobs: - name: Wait for test database readiness run: | for i in {1..30}; do - if docker exec test-postgres-db pg_isready -U postgres -d mlsec_test; then + if docker exec test-postgres-db pg_isready -U mlsec2 -d mlsec_test; then exit 0 fi sleep 2 diff --git a/config.yaml b/config.yaml index 944d302..eac3f30 100644 --- a/config.yaml +++ b/config.yaml @@ -41,8 +41,6 @@ worker: cleanup_pulled_images: true # Remove pulled images after evaluation (Docker Hub sources only) minio: bucket_name: "mlsec-submissions" - access_key: "mlsec_minio_admin" - secret_key: "mlsec_minio_password_change_in_production" attack: skip_seeding: true # true = skip template seeding and all behavioral checks check_similarity: false # false = skip evaluation, accept all validated attacks diff --git a/docker-compose.yaml b/docker-compose.yaml index ea9aebf..479a15b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,8 +4,8 @@ services: image: postgres:18 container_name: postgres-db environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password123 + POSTGRES_USER: ${POSTGRES_USER:-mlsec2} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mlsec2_pw} POSTGRES_DB: mlsec ports: - "5432:5432" @@ -20,8 +20,8 @@ services: image: postgres:18 container_name: test-postgres-db environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password123 + POSTGRES_USER: ${POSTGRES_USER:-mlsec2} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mlsec2_pw} POSTGRES_DB: mlsec_test ports: - "5433:5432" @@ -35,8 +35,8 @@ services: image: minio/minio:latest container_name: mlsec-minio environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-mlsec2} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-mlsec2_pw} command: server /data --console-address ":9001" ports: - "9000:9000" # API @@ -55,8 +55,8 @@ services: image: rabbitmq:3-management container_name: rabbitmq environment: - RABBITMQ_DEFAULT_USER: mlsec - RABBITMQ_DEFAULT_PASS: mlsec + RABBITMQ_DEFAULT_USER: ${RABBITMQ_USER:-mlsec2} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD:-mlsec2_pw} ports: - "5672:5672" # AMQP - "15672:15672" # Management UI @@ -88,12 +88,12 @@ services: dockerfile: Dockerfile container_name: mlsec-api environment: - DATABASE_URL: postgresql://postgres:password123@postgres:5432/mlsec - CELERY_BROKER_URL: amqp://mlsec:mlsec@rabbitmq:5672// + DATABASE_URL: postgresql://${POSTGRES_USER:-mlsec2}:${POSTGRES_PASSWORD:-mlsec2_pw}@postgres:5432/mlsec + CELERY_BROKER_URL: amqp://${RABBITMQ_USER:-mlsec2}:${RABBITMQ_PASSWORD:-mlsec2_pw}@rabbitmq:5672// REDIS_URL: redis://redis:6379/0 MINIO_ENDPOINT: minio:9000 - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} + MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec2} + MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec2_pw} MINIO_SECURE: "false" ADMIN_ALLOWED_HOSTS: '["172.22.0.1"]' # Docker bridge gateway for localhost SSH tunnel access ADMIN_TRUSTED_PROXY_HOSTS: '["127.0.0.1", "::1", "172.16.0.0/12"]' # Trust nginx on any Docker bridge subnet @@ -127,13 +127,13 @@ services: context: ./services/worker dockerfile: Dockerfile environment: - DATABASE_URL: postgresql://postgres:password123@postgres:5432/mlsec - CELERY_BROKER_URL: amqp://mlsec:mlsec@rabbitmq:5672// + DATABASE_URL: postgresql://${POSTGRES_USER:-mlsec2}:${POSTGRES_PASSWORD:-mlsec2_pw}@postgres:5432/mlsec + CELERY_BROKER_URL: amqp://${RABBITMQ_USER:-mlsec2}:${RABBITMQ_PASSWORD:-mlsec2_pw}@rabbitmq:5672// CELERY_DEFAULT_QUEUE: mlsec REDIS_URL: redis://redis:6379/0 MINIO_ENDPOINT: minio:9000 - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} + MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec2} + MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec2_pw} MINIO_SECURE: "false" VIRUSTOTAL_API_KEY: ${VIRUSTOTAL_API_KEY:-} CELERY_CONCURRENCY: "${WORKER_CONCURRENCY:-4}" diff --git a/services/api/core/config.py b/services/api/core/config.py index 160a3dd..83bb36d 100644 --- a/services/api/core/config.py +++ b/services/api/core/config.py @@ -16,8 +16,8 @@ class MinIOConfig(BaseModel): endpoint: str = "minio:9000" - access_key: str = "minioadmin" - secret_key: str = "minioadmin" + access_key: str = "mlsec2" + secret_key: str = "mlsec2_pw" bucket_name: str = "mlsec-submissions" secure: bool = False diff --git a/services/api/core/database.py b/services/api/core/database.py index 48c1cb0..579a731 100644 --- a/services/api/core/database.py +++ b/services/api/core/database.py @@ -15,7 +15,7 @@ from core.settings import get_settings # TODO: Change this to .env for resolving password, port, etc. -DEFAULT_DATABASE_URL = "postgresql://postgres:password123@postgres:5432/mlsec" +DEFAULT_DATABASE_URL = "postgresql://mlsec2:mlsec2_pw@postgres:5432/mlsec" logger = logging.getLogger(__name__) diff --git a/services/api/core/settings.py b/services/api/core/settings.py index 8bae321..7cc29d7 100644 --- a/services/api/core/settings.py +++ b/services/api/core/settings.py @@ -26,8 +26,8 @@ class Settings(BaseSettings): # MinIO object storage minio_endpoint: str = "minio:9000" - minio_access_key: str = "minioadmin" - minio_secret_key: str = "minioadmin" + minio_access_key: str = "mlsec2" + minio_secret_key: str = "mlsec2_pw" minio_secure: bool = False minio_bucket_name: str = "mlsec-submissions" diff --git a/services/api/core/storage.py b/services/api/core/storage.py index 570709e..3cba640 100644 --- a/services/api/core/storage.py +++ b/services/api/core/storage.py @@ -21,8 +21,8 @@ def get_minio_client() -> Minio: """Singleton MinIO client factory.""" cfg = get_config().minio http_client = urllib3.PoolManager( - timeout=urllib3.Timeout(connect=5, read=30), - retries=urllib3.Retry(total=0), + timeout=urllib3.Timeout(connect=5, read=600), + retries=urllib3.Retry(total=2), ) return Minio( endpoint=cfg.endpoint, diff --git a/services/api/tests/conftest.py b/services/api/tests/conftest.py index c857ef6..ee8494d 100644 --- a/services/api/tests/conftest.py +++ b/services/api/tests/conftest.py @@ -15,7 +15,7 @@ from unittest.mock import MagicMock -TEST_DB_URL = "postgresql://postgres:password123@localhost:5433/mlsec_test" +TEST_DB_URL = os.getenv("DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5433/mlsec_test") engine = create_engine(TEST_DB_URL) TestingSessionLocal = sessionmaker(bind=engine) diff --git a/services/frontend/src/components/EvaluationMatrix.tsx b/services/frontend/src/components/EvaluationMatrix.tsx index 5edec8d..eceea82 100644 --- a/services/frontend/src/components/EvaluationMatrix.tsx +++ b/services/frontend/src/components/EvaluationMatrix.tsx @@ -82,10 +82,10 @@ export default function EvaluationMatrix() { const { attackers, defenders, scores } = data; - if (attackers.length === 0 && defenders.length === 0) { + if (attackers.length === 0 || defenders.length === 0) { return (

- No active submissions yet. Once participants activate a submission, the matrix will appear here. + The matrix will appear once there is at least one active attack submission and one active defense submission.

); } @@ -126,30 +126,41 @@ export default function EvaluationMatrix() {
)} -
- - - {/* Axis label row */} +
+
+
+ + {/* Row 1: Attack axis label. Spans name col + all score cols (no separate corner cell). */} - )} - {/* Column headers row */} + {/* Row 2: Column headers. Defense label starts here and spans down through all defender rows. */} - + ))} - - + {/* Defender rows. Defense label column is covered by the rowSpan above. */} {defenders.map((def, di) => ( - - {di === 0 && ( - - )} +
- + {attackers.length > 0 && ( Attack
- - {attackers.map(atk => ( + + + Defense + + + {attackers.map((atk, ai) => (
@@ -162,25 +173,11 @@ export default function EvaluationMatrix() {
- - Defense - -
@@ -196,11 +193,10 @@ export default function EvaluationMatrix() { const entry = scores[key]; const bgColor = entry && showGradient ? scoreToColor(entry.score) : undefined; const fgColor = entry && showGradient ? textColorForScore(entry.score) : '#374151'; - return (
+
); diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index 438d754..2695586 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -9,7 +9,7 @@ upstream frontend { server { listen 80; listen [::]:80; - client_max_body_size 512M; + client_max_body_size 2048M; location = /health { proxy_pass http://api; @@ -122,6 +122,8 @@ server { proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 600s; + proxy_send_timeout 600s; } location / { diff --git a/services/worker/tests/conftest.py b/services/worker/tests/conftest.py index 0b2e290..eb3e53c 100644 --- a/services/worker/tests/conftest.py +++ b/services/worker/tests/conftest.py @@ -14,10 +14,10 @@ os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0") os.environ.setdefault("CELERY_RESULT_BACKEND", "redis://localhost:6379/0") os.environ.setdefault( - "DATABASE_URL", "postgresql://postgres:password123@localhost:5433/mlsec_test") + "DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5433/mlsec_test") os.environ.setdefault("MINIO_ENDPOINT", "minio:9000") -os.environ.setdefault("MINIO_ACCESS_KEY", "minioadmin") -os.environ.setdefault("MINIO_SECRET_KEY", "minioadmin") +os.environ.setdefault("MINIO_ACCESS_KEY", "mlsec2") +os.environ.setdefault("MINIO_SECRET_KEY", "mlsec2_pw") os.environ.setdefault("CELERY_DEFAULT_QUEUE", "mlsec") @@ -26,8 +26,8 @@ def set_env_vars(): """Environment variables are already set at module level.""" pass - os.environ.setdefault("MINIO_ACCESS_KEY", "minioadmin") - os.environ.setdefault("MINIO_SECRET_KEY", "minioadmin") + os.environ.setdefault("MINIO_ACCESS_KEY", "mlsec2") + os.environ.setdefault("MINIO_SECRET_KEY", "mlsec2_pw") os.environ.setdefault("MINIO_BUCKET", "mlsec-submissions") os.environ.setdefault("GATEWAY_URL", "http://mlsec-gateway:8080/") os.environ.setdefault("GATEWAY_SECRET", "test_secret") @@ -35,7 +35,7 @@ def set_env_vars(): # Test database configuration -TEST_DB_URL = "postgresql://postgres:password123@localhost:5433/mlsec_test" +TEST_DB_URL = os.environ.get("DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5433/mlsec_test") engine = create_engine(TEST_DB_URL) TestingSessionLocal = sessionmaker(bind=engine) @@ -157,8 +157,8 @@ def config_dict(): }, "minio": { "endpoint": "minio:9000", - "access_key": "minioadmin", - "secret_key": "minioadmin", + "access_key": "mlsec2", + "secret_key": "mlsec2_pw", "bucket": "mlsec-submissions", "attack_files_bucket": "attack-files", "secure": False, diff --git a/services/worker/tests/manual_api_queue_test.py b/services/worker/tests/manual_api_queue_test.py index a35412b..93e3331 100644 --- a/services/worker/tests/manual_api_queue_test.py +++ b/services/worker/tests/manual_api_queue_test.py @@ -7,7 +7,7 @@ # Configuration - Use test database API_URL = "http://localhost:8000" -DATABASE_URL = "postgresql://postgres:password123@localhost:5432/mlsec" +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5432/mlsec") SESSION_COOKIE_NAME = os.getenv("AUTH_SESSION_COOKIE_NAME", "mlsec_session") @pytest.fixture(scope="function") diff --git a/services/worker/tests/manual_test_defense_pipeline.py b/services/worker/tests/manual_test_defense_pipeline.py index df4cdb3..7b9d864 100644 --- a/services/worker/tests/manual_test_defense_pipeline.py +++ b/services/worker/tests/manual_test_defense_pipeline.py @@ -9,8 +9,8 @@ from minio import Minio # Configuration -DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:password123@localhost:5432/mlsec") -CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "amqp://mlsec:mlsec@localhost:5672//") +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5432/mlsec") +CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "amqp://mlsec2:mlsec2_pw@localhost:5672//") REDIS_URL = os.getenv("REDIS_URL", "redis://localhost:6379/0") DOCKER_IMAGE = "https://hub.docker.com/r/thompaar003/notconv" # https://hub.docker.com/r/thompaar003/notconv @@ -18,8 +18,8 @@ # https://hub.docker.com/r/thompaar003/evil-defense MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT", "localhost:9000") -MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "mlsec_minio_admin") -MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "mlsec_minio_password_change_in_production") +MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY", "mlsec2") +MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY", "mlsec2_pw") MINIO_BUCKET = os.getenv("MINIO_BUCKET", "mlsec-submissions") def setup_test_data(): diff --git a/services/worker/tests/test_db_results.py b/services/worker/tests/test_db_results.py index 7d80778..2b9cb69 100644 --- a/services/worker/tests/test_db_results.py +++ b/services/worker/tests/test_db_results.py @@ -1,7 +1,7 @@ import os from sqlalchemy import create_engine, text -DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://postgres:password123@localhost:5432/mlsec") +DATABASE_URL = os.getenv("DATABASE_URL", "postgresql://mlsec2:mlsec2_pw@localhost:5432/mlsec") def dump_results(): engine = create_engine(DATABASE_URL) diff --git a/services/worker/worker/config.py b/services/worker/worker/config.py index 10e3550..b24de51 100644 --- a/services/worker/worker/config.py +++ b/services/worker/worker/config.py @@ -102,9 +102,9 @@ class MinIOConfig(BaseModel): endpoint: str = Field(default_factory=lambda: os.getenv( "MINIO_ENDPOINT", "minio:9000")) access_key: str = Field(default_factory=lambda: os.getenv( - "MINIO_ACCESS_KEY", "minioadmin")) + "MINIO_ACCESS_KEY", "mlsec2")) secret_key: str = Field(default_factory=lambda: os.getenv( - "MINIO_SECRET_KEY", "minioadmin")) + "MINIO_SECRET_KEY", "mlsec2_pw")) bucket_name: str = "mlsec-submissions" secure: bool = Field(default_factory=lambda: os.getenv( "MINIO_SECURE", "false").lower() == "true") From a60bb42107bc627fc49c6c1941d6ef40ebbb9e0e Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 13 Apr 2026 13:23:05 -0500 Subject: [PATCH 36/38] Prepped NGINX for HTTPS deployment --- docker-compose.yaml | 13 +++++++++++++ services/nginx/nginx.conf | 24 ++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 479a15b..1a6e39a 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -174,14 +174,25 @@ services: container_name: mlsec-nginx ports: - "80:80" + - "443:443" volumes: - ./services/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro + - certbot-conf:/etc/letsencrypt:ro + - certbot-www:/var/www/certbot:ro networks: - backend_net depends_on: - api - frontend + certbot: + image: certbot/certbot + container_name: mlsec-certbot + volumes: + - certbot-conf:/etc/letsencrypt + - certbot-www:/var/www/certbot + entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew --quiet; sleep 12h & wait $${!}; done'" + # Adminer (Can remove in production if we dont need it) adminer: image: adminer:latest @@ -209,5 +220,7 @@ volumes: redisdata: miniodata: worker_cache: + certbot-conf: + certbot-www: diff --git a/services/nginx/nginx.conf b/services/nginx/nginx.conf index 2695586..6824c67 100644 --- a/services/nginx/nginx.conf +++ b/services/nginx/nginx.conf @@ -9,8 +9,32 @@ upstream frontend { server { listen 80; listen [::]:80; + + # Let's Encrypt ACME challenge + location /.well-known/acme-challenge/ { + root /var/www/certbot; + } + + # Redirect all other HTTP traffic to HTTPS + location / { + return 301 https://$host$request_uri; + } +} + +server { + listen 443 ssl; + listen [::]:443 ssl; client_max_body_size 2048M; + ssl_certificate /etc/letsencrypt/live/mlsec2.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/mlsec2.com/privkey.pem; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers HIGH:!aNULL:!MD5; + ssl_prefer_server_ciphers on; + ssl_session_cache shared:SSL:10m; + ssl_session_timeout 10m; + location = /health { proxy_pass http://api; proxy_http_version 1.1; From 0aae7e9ce84f1f415c9ab38748adb7522031d26b Mon Sep 17 00:00:00 2001 From: gmgrahamgm Date: Mon, 13 Apr 2026 13:45:52 -0500 Subject: [PATCH 37/38] Added more coverage for API --- services/api/tests/test_audit.py | 119 ++++++ services/api/tests/test_config.py | 159 +++++++ services/api/tests/test_core_admin.py | 390 ++++++++++++++++++ services/api/tests/test_storage.py | 189 +++++++++ services/api/tests/test_submission_control.py | 343 +++++++++++++++ 5 files changed, 1200 insertions(+) create mode 100644 services/api/tests/test_audit.py create mode 100644 services/api/tests/test_config.py create mode 100644 services/api/tests/test_core_admin.py create mode 100644 services/api/tests/test_storage.py create mode 100644 services/api/tests/test_submission_control.py diff --git a/services/api/tests/test_audit.py b/services/api/tests/test_audit.py new file mode 100644 index 0000000..7482626 --- /dev/null +++ b/services/api/tests/test_audit.py @@ -0,0 +1,119 @@ +"""Tests for core/audit.py.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, call +from uuid import UUID, uuid4 + +import pytest + +import core.audit as audit_module +from core.audit import log_audit_event + + +def _make_engine_mock(): + mock_conn = MagicMock() + mock_ctx = MagicMock() + mock_ctx.__enter__ = MagicMock(return_value=mock_conn) + mock_ctx.__exit__ = MagicMock(return_value=False) + mock_engine = MagicMock() + mock_engine.begin.return_value = mock_ctx + return mock_engine, mock_conn + + +def test_log_audit_event_minimal(monkeypatch): + mock_engine, mock_conn = _make_engine_mock() + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + log_audit_event(event_type="test.event") + + mock_conn.execute.assert_called_once() + + +def test_log_audit_event_all_fields(monkeypatch): + mock_engine, mock_conn = _make_engine_mock() + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + user_id = uuid4() + log_audit_event( + event_type="auth.login", + user_id=user_id, + email="user@example.com", + ip_address="127.0.0.1", + user_agent="TestAgent/1.0", + success=True, + message="Login successful", + metadata={"key": "value", "count": 42}, + ) + + mock_conn.execute.assert_called_once() + _, params = mock_conn.execute.call_args[0] + assert params["event_type"] == "auth.login" + assert params["user_id"] == str(user_id) + assert params["email"] == "user@example.com" + assert params["ip_address"] == "127.0.0.1" + assert params["user_agent"] == "TestAgent/1.0" + assert params["success"] is True + assert params["message"] == "Login successful" + assert '"key": "value"' in params["metadata"] + assert '"count": 42' in params["metadata"] + + +def test_log_audit_event_none_user_id(monkeypatch): + mock_engine, mock_conn = _make_engine_mock() + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + log_audit_event(event_type="anon.event", user_id=None) + + _, params = mock_conn.execute.call_args[0] + assert params["user_id"] is None + + +def test_log_audit_event_none_metadata_serializes_as_none(monkeypatch): + mock_engine, mock_conn = _make_engine_mock() + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + log_audit_event(event_type="test.event", metadata=None) + + _, params = mock_conn.execute.call_args[0] + assert params["metadata"] is None + + +def test_log_audit_event_swallows_engine_exception(monkeypatch): + def _bad_engine(): + raise RuntimeError("DB is down") + + monkeypatch.setattr(audit_module, "get_engine", _bad_engine) + + result = log_audit_event(event_type="test.event", message="should not crash") + + assert result is None + + +def test_log_audit_event_swallows_execute_exception(monkeypatch): + mock_conn = MagicMock() + mock_conn.execute.side_effect = Exception("execute failed") + mock_ctx = MagicMock() + mock_ctx.__enter__ = MagicMock(return_value=mock_conn) + mock_ctx.__exit__ = MagicMock(return_value=False) + mock_engine = MagicMock() + mock_engine.begin.return_value = mock_ctx + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + result = log_audit_event(event_type="test.event") + + assert result is None + + +def test_log_audit_event_metadata_dict_is_json_serialized(monkeypatch): + mock_engine, mock_conn = _make_engine_mock() + monkeypatch.setattr(audit_module, "get_engine", lambda: mock_engine) + + metadata = {"action": "disable", "target_user": "abc-123"} + log_audit_event(event_type="admin.user_disabled", metadata=metadata) + + _, params = mock_conn.execute.call_args[0] + import json + parsed = json.loads(params["metadata"]) + assert parsed["action"] == "disable" + assert parsed["target_user"] == "abc-123" diff --git a/services/api/tests/test_config.py b/services/api/tests/test_config.py new file mode 100644 index 0000000..237e3fb --- /dev/null +++ b/services/api/tests/test_config.py @@ -0,0 +1,159 @@ +"""Tests for core/config.py.""" + +from __future__ import annotations + +import pytest + +import core.config as config_module +from core.config import AppConfig, get_config + + +@pytest.fixture(autouse=True) +def clear_cache(): + config_module.get_config.cache_clear() + yield + config_module.get_config.cache_clear() + + +def test_get_config_no_file_returns_defaults(monkeypatch, tmp_path): + missing = tmp_path / "nonexistent.yaml" + monkeypatch.setattr(config_module, "Path", lambda p: missing) + + config = get_config() + + assert config.minio.endpoint == "minio:9000" + assert config.minio.access_key == "mlsec2" + assert config.minio.secret_key == "mlsec2_pw" + assert config.minio.bucket_name == "mlsec-submissions" + assert config.minio.secure is False + assert config.application.join_code is None + assert config.application.defense_submission_cooldown == 0 + assert config.application.attack_submission_cooldown == 0 + + +def test_get_config_valid_yaml(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text( + "worker:\n" + " minio:\n" + " endpoint: custom-minio:9001\n" + " access_key: mykey\n" + " secret_key: mysecret\n" + " bucket_name: my-bucket\n" + " secure: true\n" + "application:\n" + " join_code: secret123\n" + " defense_submission_cooldown: 300\n" + " attack_submission_cooldown: 600\n" + ) + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert config.minio.endpoint == "custom-minio:9001" + assert config.minio.access_key == "mykey" + assert config.minio.secret_key == "mysecret" + assert config.minio.bucket_name == "my-bucket" + assert config.minio.secure is True + assert config.application.join_code == "secret123" + assert config.application.defense_submission_cooldown == 300 + assert config.application.attack_submission_cooldown == 600 + + +def test_get_config_login_code_fallback(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text( + "worker:\n" + " minio: {}\n" + "application:\n" + " login_code: fallback_code\n" + ) + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert config.application.join_code == "fallback_code" + + +def test_get_config_join_code_takes_precedence_over_login_code(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text( + "worker:\n" + " minio: {}\n" + "application:\n" + " join_code: primary\n" + " login_code: fallback\n" + ) + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert config.application.join_code == "primary" + + +def test_get_config_malformed_yaml_returns_defaults(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text("[invalid yaml {{{") + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert isinstance(config, AppConfig) + assert config.minio.endpoint == "minio:9000" + assert config.application.defense_submission_cooldown == 0 + + +def test_get_config_empty_yaml_returns_defaults(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text("") + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert isinstance(config, AppConfig) + assert config.minio.endpoint == "minio:9000" + assert config.application.join_code is None + + +def test_get_config_partial_application_section(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text( + "worker:\n" + " minio:\n" + " endpoint: minio:9000\n" + "application:\n" + " defense_submission_cooldown: 120\n" + ) + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert config.application.defense_submission_cooldown == 120 + assert config.application.attack_submission_cooldown == 0 + assert config.application.join_code is None + + +def test_get_config_no_application_section(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text( + "worker:\n" + " minio:\n" + " endpoint: custom:9000\n" + ) + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + config = get_config() + + assert config.application.join_code is None + assert config.application.defense_submission_cooldown == 0 + + +def test_get_config_is_cached(monkeypatch, tmp_path): + config_file = tmp_path / "config.yaml" + config_file.write_text("") + monkeypatch.setattr(config_module, "Path", lambda p: config_file) + + first = get_config() + second = get_config() + + assert first is second diff --git a/services/api/tests/test_core_admin.py b/services/api/tests/test_core_admin.py new file mode 100644 index 0000000..ecec69a --- /dev/null +++ b/services/api/tests/test_core_admin.py @@ -0,0 +1,390 @@ +"""Tests for core/admin.py.""" + +from __future__ import annotations + +import hashlib +import secrets +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from fastapi import HTTPException +from sqlalchemy import text + +from core.admin import ( + _hosts_match, + _is_from_trusted_proxy, + _is_in_allowed_hosts, + _is_in_allowed_networks, + _is_loopback_host, + consume_admin_action_token, + issue_admin_action_token, + require_admin_origin, + require_localhost_request, + require_admin_action_token, +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +class _FakeSettings: + admin_localhost_only = True + admin_trusted_proxy_hosts = [] + admin_forwarded_for_header = "x-forwarded-for" + admin_allowed_hosts = [] + admin_allowed_networks = [] + admin_action_token_ttl_minutes = 5 + + +def _make_request(host="127.0.0.1", headers=None, client=True): + if client: + client_obj = SimpleNamespace(host=host) + else: + client_obj = None + return SimpleNamespace(client=client_obj, headers=headers or {}) + + +def _create_user_and_session(db_session): + uid = uuid4().hex[:8] + user_row = db_session.execute( + text( + "INSERT INTO users (username, email, is_admin) " + "VALUES (:username, :email, true) RETURNING id" + ), + {"username": f"admin_{uid}", "email": f"admin_{uid}@test.com"}, + ).fetchone() + user_id = str(user_row[0]) + + token = f"test-session-{uuid4()}" + token_hash = hashlib.sha256(token.encode()).hexdigest() + now = datetime.now(timezone.utc) + session_row = db_session.execute( + text( + "INSERT INTO user_sessions (user_id, token_hash, expires_at, last_seen_at) " + "VALUES (:user_id, :token_hash, :expires_at, :last_seen_at) RETURNING id" + ), + { + "user_id": user_id, + "token_hash": token_hash, + "expires_at": now + timedelta(hours=2), + "last_seen_at": now, + }, + ).fetchone() + session_id = str(session_row[0]) + db_session.flush() + return user_id, session_id + + +# --------------------------------------------------------------------------- +# _is_loopback_host +# --------------------------------------------------------------------------- + + +class TestIsLoopbackHost: + def test_localhost_string(self): + assert _is_loopback_host("localhost") is True + + def test_ipv4_loopback(self): + assert _is_loopback_host("127.0.0.1") is True + + def test_ipv4_loopback_alt(self): + assert _is_loopback_host("127.0.0.2") is True + + def test_ipv6_loopback(self): + assert _is_loopback_host("::1") is True + + def test_private_ip_is_not_loopback(self): + assert _is_loopback_host("192.168.1.1") is False + + def test_public_ip_is_not_loopback(self): + assert _is_loopback_host("8.8.8.8") is False + + def test_none_returns_false(self): + assert _is_loopback_host(None) is False + + def test_invalid_string_returns_false(self): + assert _is_loopback_host("not-an-ip") is False + + def test_ipv4_mapped_loopback(self): + assert _is_loopback_host("::ffff:127.0.0.1") is True + + +# --------------------------------------------------------------------------- +# _hosts_match +# --------------------------------------------------------------------------- + + +class TestHostsMatch: + def test_identical_strings(self): + assert _hosts_match("localhost", "localhost") is True + + def test_case_insensitive(self): + assert _hosts_match("LOCALHOST", "localhost") is True + + def test_same_ip_different_format(self): + assert _hosts_match("127.0.0.1", "127.0.0.1") is True + + def test_different_hosts(self): + assert _hosts_match("192.168.1.1", "192.168.1.2") is False + + def test_hostname_vs_ip(self): + assert _hosts_match("example.com", "127.0.0.1") is False + + +# --------------------------------------------------------------------------- +# _is_from_trusted_proxy +# --------------------------------------------------------------------------- + + +class TestIsFromTrustedProxy: + def test_exact_match(self): + assert _is_from_trusted_proxy("10.0.0.1", ["10.0.0.1"]) is True + + def test_cidr_match(self): + assert _is_from_trusted_proxy("10.0.0.5", ["10.0.0.0/24"]) is True + + def test_no_match(self): + assert _is_from_trusted_proxy("172.16.0.1", ["10.0.0.0/8"]) is False + + def test_none_host_returns_false(self): + assert _is_from_trusted_proxy(None, ["10.0.0.1"]) is False + + def test_empty_list_returns_false(self): + assert _is_from_trusted_proxy("127.0.0.1", []) is False + + +# --------------------------------------------------------------------------- +# _is_in_allowed_hosts +# --------------------------------------------------------------------------- + + +class TestIsInAllowedHosts: + def test_matching_host(self): + assert _is_in_allowed_hosts("10.1.2.3", ["10.1.2.3"]) is True + + def test_non_matching_host(self): + assert _is_in_allowed_hosts("10.1.2.4", ["10.1.2.3"]) is False + + def test_none_host_returns_false(self): + assert _is_in_allowed_hosts(None, ["10.1.2.3"]) is False + + def test_empty_list_returns_false(self): + assert _is_in_allowed_hosts("10.1.2.3", []) is False + + +# --------------------------------------------------------------------------- +# _is_in_allowed_networks +# --------------------------------------------------------------------------- + + +class TestIsInAllowedNetworks: + def test_ip_in_cidr(self): + assert _is_in_allowed_networks("10.0.0.50", ["10.0.0.0/24"]) is True + + def test_ip_outside_cidr(self): + assert _is_in_allowed_networks("10.0.1.1", ["10.0.0.0/24"]) is False + + def test_none_host_returns_false(self): + assert _is_in_allowed_networks(None, ["10.0.0.0/24"]) is False + + def test_invalid_network_skipped(self): + assert _is_in_allowed_networks("10.0.0.1", ["not-a-network"]) is False + + +# --------------------------------------------------------------------------- +# require_localhost_request +# --------------------------------------------------------------------------- + + +class TestRequireLocalhostRequest: + def test_loopback_host_allowed(self, monkeypatch): + monkeypatch.setattr("core.admin.get_settings", _FakeSettings) + request = _make_request(host="127.0.0.1") + require_localhost_request(request) + + def test_non_loopback_raises_403(self, monkeypatch): + monkeypatch.setattr("core.admin.get_settings", _FakeSettings) + request = _make_request(host="192.168.1.100") + with pytest.raises(HTTPException) as exc_info: + require_localhost_request(request) + assert exc_info.value.status_code == 403 + + def test_non_loopback_in_allowed_hosts_passes(self, monkeypatch): + class _Settings(_FakeSettings): + admin_allowed_hosts = ["10.0.0.5"] + + monkeypatch.setattr("core.admin.get_settings", _Settings) + request = _make_request(host="10.0.0.5") + require_localhost_request(request) + + def test_non_loopback_in_allowed_networks_passes(self, monkeypatch): + class _Settings(_FakeSettings): + admin_allowed_networks = ["10.0.0.0/24"] + + monkeypatch.setattr("core.admin.get_settings", _Settings) + request = _make_request(host="10.0.0.42") + require_localhost_request(request) + + def test_localhost_only_false_skips_check(self, monkeypatch): + class _Settings(_FakeSettings): + admin_localhost_only = False + + monkeypatch.setattr("core.admin.get_settings", _Settings) + request = _make_request(host="8.8.8.8") + require_localhost_request(request) + + def test_ipv6_loopback_allowed(self, monkeypatch): + monkeypatch.setattr("core.admin.get_settings", _FakeSettings) + request = _make_request(host="::1") + require_localhost_request(request) + + +# --------------------------------------------------------------------------- +# require_admin_origin +# --------------------------------------------------------------------------- + + +class TestRequireAdminOrigin: + def test_no_origin_or_referer_with_require_present_raises(self): + request = _make_request(headers={}) + with pytest.raises(HTTPException) as exc_info: + require_admin_origin(request, require_present=True) + assert exc_info.value.status_code == 403 + + def test_no_origin_or_referer_with_require_present_false_passes(self): + request = _make_request(headers={}) + require_admin_origin(request, require_present=False) + + def test_localhost_origin_passes(self): + request = _make_request(headers={"origin": "http://localhost:4321"}) + require_admin_origin(request) + + def test_non_localhost_origin_raises(self): + request = _make_request(headers={"origin": "https://evil.com"}) + with pytest.raises(HTTPException) as exc_info: + require_admin_origin(request) + assert exc_info.value.status_code == 403 + + def test_localhost_referer_passes(self): + request = _make_request(headers={"referer": "http://localhost/admin"}) + require_admin_origin(request) + + def test_non_localhost_referer_raises(self): + request = _make_request(headers={"referer": "https://evil.com/path"}) + with pytest.raises(HTTPException) as exc_info: + require_admin_origin(request) + assert exc_info.value.status_code == 403 + + def test_127_origin_passes(self): + request = _make_request(headers={"origin": "http://127.0.0.1:3000"}) + require_admin_origin(request) + + +# --------------------------------------------------------------------------- +# issue_admin_action_token / require_admin_action_token / consume_admin_action_token +# --------------------------------------------------------------------------- + + +class TestAdminActionToken: + def test_issue_returns_token_and_expiry(self, db_session): + _, session_id = _create_user_and_session(db_session) + + token, expires_at = issue_admin_action_token(db_session, session_id=session_id) + + assert isinstance(token, str) + assert len(token) > 0 + assert isinstance(expires_at, datetime) + assert expires_at > datetime.now(timezone.utc) + + def test_issued_token_is_hashed_in_db(self, db_session): + _, session_id = _create_user_and_session(db_session) + + token, _ = issue_admin_action_token(db_session, session_id=session_id) + + expected_hash = hashlib.sha256(token.encode()).hexdigest() + row = db_session.execute( + text( + "SELECT token_hash FROM admin_action_tokens WHERE session_id = :session_id" + ), + {"session_id": session_id}, + ).fetchone() + assert row is not None + assert row[0] == expected_hash + + def test_require_valid_token_returns_token(self, db_session): + _, session_id = _create_user_and_session(db_session) + token, _ = issue_admin_action_token(db_session, session_id=session_id) + request = _make_request(headers={"x-admin-action": token}) + + result = require_admin_action_token(request, db=db_session, session_id=session_id) + + assert result == token + + def test_require_missing_header_raises_403(self, db_session): + _, session_id = _create_user_and_session(db_session) + request = _make_request(headers={}) + + with pytest.raises(HTTPException) as exc_info: + require_admin_action_token(request, db=db_session, session_id=session_id) + assert exc_info.value.status_code == 403 + + def test_require_wrong_token_raises_403(self, db_session): + _, session_id = _create_user_and_session(db_session) + issue_admin_action_token(db_session, session_id=session_id) + request = _make_request(headers={"x-admin-action": "wrong-token"}) + + with pytest.raises(HTTPException) as exc_info: + require_admin_action_token(request, db=db_session, session_id=session_id) + assert exc_info.value.status_code == 403 + + def test_require_expired_token_raises_403(self, db_session): + _, session_id = _create_user_and_session(db_session) + raw_token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(raw_token.encode()).hexdigest() + past = datetime.now(timezone.utc) - timedelta(minutes=10) + db_session.execute( + text( + "INSERT INTO admin_action_tokens (session_id, token_hash, expires_at) " + "VALUES (:session_id, :token_hash, :expires_at)" + ), + {"session_id": session_id, "token_hash": token_hash, "expires_at": past}, + ) + db_session.flush() + request = _make_request(headers={"x-admin-action": raw_token}) + + with pytest.raises(HTTPException) as exc_info: + require_admin_action_token(request, db=db_session, session_id=session_id) + assert exc_info.value.status_code == 403 + + def test_consume_removes_token(self, db_session): + _, session_id = _create_user_and_session(db_session) + token, _ = issue_admin_action_token(db_session, session_id=session_id) + + consume_admin_action_token(db_session, session_id=session_id, token=token) + db_session.flush() + + row = db_session.execute( + text( + "SELECT token_hash FROM admin_action_tokens WHERE session_id = :session_id" + ), + {"session_id": session_id}, + ).fetchone() + assert row is None + + def test_issue_replaces_existing_token(self, db_session): + _, session_id = _create_user_and_session(db_session) + token1, _ = issue_admin_action_token(db_session, session_id=session_id) + token2, _ = issue_admin_action_token(db_session, session_id=session_id) + + count_row = db_session.execute( + text( + "SELECT COUNT(*) FROM admin_action_tokens WHERE session_id = :session_id" + ), + {"session_id": session_id}, + ).fetchone() + assert count_row[0] == 1 + assert token1 != token2 diff --git a/services/api/tests/test_storage.py b/services/api/tests/test_storage.py new file mode 100644 index 0000000..660c39e --- /dev/null +++ b/services/api/tests/test_storage.py @@ -0,0 +1,189 @@ +"""Tests for core/storage.py.""" + +from __future__ import annotations + +import hashlib +import io +from unittest.mock import MagicMock, patch + +import pytest +from minio.error import S3Error + +import core.storage as storage_module +from core.storage import ( + delete_object, + ensure_bucket_exists, + upload_attack_template, + upload_attack_zip, + upload_defense_zip, + upload_heurval_sample, + upload_heurval_set_zip, +) + + +def _make_s3_error(): + fake_response = MagicMock() + return S3Error(fake_response, "TestError", "something went wrong", "/", "req-1", "host-1") + + +@pytest.fixture(autouse=True) +def mock_storage(monkeypatch): + mock_client = MagicMock() + mock_config = MagicMock() + mock_config.minio.bucket_name = "test-bucket" + monkeypatch.setattr(storage_module, "get_minio_client", lambda: mock_client) + monkeypatch.setattr(storage_module, "get_config", lambda: mock_config) + return mock_client + + +class TestUploadDefenseZip: + def test_returns_correct_keys(self, mock_storage): + content = b"fake zip content" + user_id = "user-123" + submission_id = "sub-456" + + result = upload_defense_zip(io.BytesIO(content), user_id, submission_id) + + assert result["object_key"] == f"defense/{user_id}/{submission_id}.zip" + assert result["sha256"] == hashlib.sha256(content).hexdigest() + assert result["size_bytes"] == len(content) + + def test_calls_put_object_with_correct_args(self, mock_storage): + content = b"zip data" + upload_defense_zip(io.BytesIO(content), "uid", "sid") + + mock_storage.put_object.assert_called_once() + call_kwargs = mock_storage.put_object.call_args.kwargs + assert call_kwargs["bucket_name"] == "test-bucket" + assert call_kwargs["object_name"] == "defense/uid/sid.zip" + assert call_kwargs["length"] == len(content) + assert call_kwargs["content_type"] == "application/zip" + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.put_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + upload_defense_zip(io.BytesIO(b"data"), "uid", "sid") + + +class TestUploadAttackZip: + def test_returns_correct_keys(self, mock_storage): + content = b"attack zip" + user_id = "user-abc" + submission_id = "sub-xyz" + + result = upload_attack_zip(io.BytesIO(content), user_id, submission_id) + + assert result["object_key"] == f"attack/{user_id}/{submission_id}.zip" + assert result["sha256"] == hashlib.sha256(content).hexdigest() + assert result["size_bytes"] == len(content) + + def test_calls_put_object_with_correct_args(self, mock_storage): + content = b"atk data" + upload_attack_zip(io.BytesIO(content), "uid", "sid") + + call_kwargs = mock_storage.put_object.call_args.kwargs + assert call_kwargs["bucket_name"] == "test-bucket" + assert call_kwargs["object_name"] == "attack/uid/sid.zip" + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.put_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + upload_attack_zip(io.BytesIO(b"data"), "uid", "sid") + + +class TestUploadAttackTemplate: + def test_returns_correct_keys(self, mock_storage): + content = b"template zip content" + template_id = "tmpl-001" + + result = upload_attack_template(content, template_id) + + assert result["object_key"] == f"template/{template_id}.zip" + assert result["sha256"] == hashlib.sha256(content).hexdigest() + assert result["size_bytes"] == len(content) + + def test_sha256_matches_actual_hash(self, mock_storage): + content = b"hello world" + result = upload_attack_template(content, "t1") + expected = hashlib.sha256(content).hexdigest() + assert result["sha256"] == expected + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.put_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + upload_attack_template(b"data", "tmpl-id") + + +class TestUploadHeurvalSample: + def test_returns_correct_keys(self, mock_storage): + content = b"sample content" + set_id = "set-001" + label = "malware" + filename = "evil.exe" + + result = upload_heurval_sample(content, set_id, label, filename) + + assert result["object_key"] == f"heurval/{set_id}/{label}/{filename}" + assert result["sha256"] == hashlib.sha256(content).hexdigest() + assert result["size_bytes"] == len(content) + + def test_uses_basename_for_safety(self, mock_storage): + content = b"data" + result = upload_heurval_sample( + content, "s1", "goodware", "/some/nested/path/file.exe" + ) + assert result["object_key"] == "heurval/s1/goodware/file.exe" + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.put_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + upload_heurval_sample(b"data", "set", "malware", "file.exe") + + +class TestUploadHeurvalSetZip: + def test_returns_correct_keys(self, mock_storage): + content = b"heurval zip" + set_id = "hvset-001" + + result = upload_heurval_set_zip(content, set_id) + + assert result["object_key"] == f"heurval/{set_id}/samples.zip" + assert result["sha256"] == hashlib.sha256(content).hexdigest() + assert result["size_bytes"] == len(content) + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.put_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + upload_heurval_set_zip(b"data", "set-id") + + +class TestDeleteObject: + def test_calls_remove_object_with_correct_args(self, mock_storage): + object_key = "defense/user-1/sub-1.zip" + delete_object(object_key) + mock_storage.remove_object.assert_called_once_with( + bucket_name="test-bucket", + object_name=object_key, + ) + + def test_raises_s3_error_on_failure(self, mock_storage): + mock_storage.remove_object.side_effect = _make_s3_error() + with pytest.raises(S3Error): + delete_object("some/key.zip") + + +class TestEnsureBucketExists: + def test_creates_bucket_when_not_exists(self, mock_storage): + mock_storage.bucket_exists.return_value = False + ensure_bucket_exists() + mock_storage.make_bucket.assert_called_once_with("test-bucket") + + def test_skips_make_bucket_when_already_exists(self, mock_storage): + mock_storage.bucket_exists.return_value = True + ensure_bucket_exists() + mock_storage.make_bucket.assert_not_called() + + def test_raises_s3_error_when_bucket_check_fails(self, mock_storage): + mock_storage.bucket_exists.side_effect = _make_s3_error() + with pytest.raises(S3Error): + ensure_bucket_exists() diff --git a/services/api/tests/test_submission_control.py b/services/api/tests/test_submission_control.py new file mode 100644 index 0000000..e1d470e --- /dev/null +++ b/services/api/tests/test_submission_control.py @@ -0,0 +1,343 @@ +"""Tests for core/submission_control.py.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from uuid import uuid4 + +import pytest +from fastapi import HTTPException +from sqlalchemy import text + +from core.submission_control import ( + SubmissionControl, + _as_utc, + check_cooldown, + ensure_submissions_open, + get_cooldown_remaining, + get_submission_control, + set_close_at, + set_manual_closed, +) + + +# --------------------------------------------------------------------------- +# Pure-Python unit tests (no DB) +# --------------------------------------------------------------------------- + + +class TestSubmissionControlIsClosed: + def test_open_by_default(self): + sc = SubmissionControl( + manual_closed=False, + close_at=None, + updated_at=None, + updated_by=None, + ) + assert sc.is_closed() is False + + def test_manual_closed_true(self): + sc = SubmissionControl( + manual_closed=True, + close_at=None, + updated_at=None, + updated_by=None, + ) + assert sc.is_closed() is True + + def test_close_at_in_past(self): + past = datetime.now(timezone.utc) - timedelta(minutes=5) + sc = SubmissionControl( + manual_closed=False, + close_at=past, + updated_at=None, + updated_by=None, + ) + assert sc.is_closed() is True + + def test_close_at_in_future(self): + future = datetime.now(timezone.utc) + timedelta(hours=1) + sc = SubmissionControl( + manual_closed=False, + close_at=future, + updated_at=None, + updated_by=None, + ) + assert sc.is_closed() is False + + def test_both_manual_and_past_close_at(self): + past = datetime.now(timezone.utc) - timedelta(seconds=1) + sc = SubmissionControl( + manual_closed=True, + close_at=past, + updated_at=None, + updated_by=None, + ) + assert sc.is_closed() is True + + def test_is_closed_accepts_explicit_now(self): + future = datetime.now(timezone.utc) + timedelta(hours=1) + sc = SubmissionControl( + manual_closed=False, + close_at=future, + updated_at=None, + updated_by=None, + ) + now_way_ahead = future + timedelta(hours=2) + assert sc.is_closed(now=now_way_ahead) is True + + +class TestAsUtc: + def test_none_returns_none(self): + assert _as_utc(None) is None + + def test_naive_datetime_gets_utc_tzinfo(self): + naive = datetime(2024, 6, 1, 12, 0, 0) + result = _as_utc(naive) + assert result.tzinfo is timezone.utc + assert result.replace(tzinfo=None) == naive + + def test_utc_datetime_unchanged(self): + utc_dt = datetime(2024, 6, 1, 12, 0, 0, tzinfo=timezone.utc) + result = _as_utc(utc_dt) + assert result == utc_dt + + def test_non_utc_aware_datetime_converted(self): + from datetime import timezone as tz + eastern = timezone(timedelta(hours=-5)) + dt = datetime(2024, 6, 1, 7, 0, 0, tzinfo=eastern) + result = _as_utc(dt) + assert result.tzinfo == timezone.utc + assert result.hour == 12 + + +# --------------------------------------------------------------------------- +# DB-backed integration tests +# --------------------------------------------------------------------------- + + +def _create_user(db_session, *, username: str = None, email: str = None) -> str: + username = username or f"sc_user_{uuid4().hex[:8]}" + email = email or f"{uuid4().hex[:8]}@test.com" + row = db_session.execute( + text( + "INSERT INTO users (username, email, is_admin) " + "VALUES (:username, :email, false) RETURNING id" + ), + {"username": username, "email": email}, + ).fetchone() + return str(row[0]) + + +class TestGetSubmissionControl: + def test_returns_defaults_when_no_row(self, db_session): + sc = get_submission_control(db_session) + assert sc.manual_closed is False + assert sc.close_at is None + + def test_returns_correct_state_when_row_exists(self, db_session): + future = datetime.now(timezone.utc) + timedelta(hours=2) + user_id = _create_user(db_session) + db_session.execute( + text( + "INSERT INTO submission_control (id, manual_closed, close_at, updated_by) " + "VALUES (1, true, :close_at, :updated_by) " + "ON CONFLICT (id) DO UPDATE " + "SET manual_closed = EXCLUDED.manual_closed, " + " close_at = EXCLUDED.close_at, " + " updated_by = EXCLUDED.updated_by" + ), + {"close_at": future, "updated_by": user_id}, + ) + db_session.flush() + + sc = get_submission_control(db_session) + + assert sc.manual_closed is True + assert sc.close_at is not None + assert sc.close_at > datetime.now(timezone.utc) + assert sc.updated_by == user_id + + +class TestSetManualClosed: + def test_set_manual_closed_true(self, db_session): + user_id = _create_user(db_session) + sc = set_manual_closed(db_session, closed=True, updated_by=user_id) + assert sc.manual_closed is True + assert sc.updated_by == user_id + + def test_set_manual_closed_false(self, db_session): + user_id = _create_user(db_session) + set_manual_closed(db_session, closed=True, updated_by=user_id) + sc = set_manual_closed(db_session, closed=False, updated_by=user_id) + assert sc.manual_closed is False + + def test_open_clears_lapsed_close_at(self, db_session): + user_id = _create_user(db_session) + past = datetime.now(timezone.utc) - timedelta(hours=1) + set_close_at(db_session, close_at=past, updated_by=user_id) + sc = set_manual_closed(db_session, closed=False, updated_by=user_id) + assert sc.close_at is None + + def test_open_preserves_future_close_at(self, db_session): + user_id = _create_user(db_session) + future = datetime.now(timezone.utc) + timedelta(hours=2) + set_close_at(db_session, close_at=future, updated_by=user_id) + sc = set_manual_closed(db_session, closed=False, updated_by=user_id) + assert sc.close_at is not None + + +class TestSetCloseAt: + def test_set_future_close_at(self, db_session): + user_id = _create_user(db_session) + future = datetime.now(timezone.utc) + timedelta(hours=3) + sc = set_close_at(db_session, close_at=future, updated_by=user_id) + assert sc.close_at is not None + assert sc.close_at > datetime.now(timezone.utc) + + def test_clear_close_at(self, db_session): + user_id = _create_user(db_session) + future = datetime.now(timezone.utc) + timedelta(hours=3) + set_close_at(db_session, close_at=future, updated_by=user_id) + sc = set_close_at(db_session, close_at=None, updated_by=user_id) + assert sc.close_at is None + + +class TestEnsureSubmissionsOpen: + def test_does_not_raise_when_open(self, db_session): + ensure_submissions_open(db_session) + + def test_raises_403_when_manual_closed(self, db_session): + user_id = _create_user(db_session) + set_manual_closed(db_session, closed=True, updated_by=user_id) + with pytest.raises(HTTPException) as exc_info: + ensure_submissions_open(db_session) + assert exc_info.value.status_code == 403 + assert "administrator" in exc_info.value.detail + + def test_raises_403_when_deadline_passed(self, db_session): + user_id = _create_user(db_session) + past = datetime.now(timezone.utc) - timedelta(minutes=1) + set_close_at(db_session, close_at=past, updated_by=user_id) + with pytest.raises(HTTPException) as exc_info: + ensure_submissions_open(db_session) + assert exc_info.value.status_code == 403 + assert "deadline" in exc_info.value.detail + + +class TestGetCooldownRemaining: + def test_returns_none_when_cooldown_zero(self, db_session): + user_id = _create_user(db_session) + result = get_cooldown_remaining( + db_session, + user_id=user_id, + submission_type="defense", + cooldown_seconds=0, + ) + assert result is None + + def test_returns_none_when_no_prior_submissions(self, db_session): + user_id = _create_user(db_session) + result = get_cooldown_remaining( + db_session, + user_id=user_id, + submission_type="defense", + cooldown_seconds=3600, + ) + assert result is None + + def test_returns_remaining_when_within_cooldown(self, db_session): + user_id = _create_user(db_session) + just_now = datetime.now(timezone.utc) - timedelta(seconds=10) + db_session.execute( + text( + "INSERT INTO submissions (user_id, submission_type, version, status, created_at) " + "VALUES (:user_id, 'defense', '1.0.0', 'submitted', :created_at)" + ), + {"user_id": user_id, "created_at": just_now}, + ) + db_session.flush() + + result = get_cooldown_remaining( + db_session, + user_id=user_id, + submission_type="defense", + cooldown_seconds=3600, + ) + assert result is not None + assert result > 0 + assert result <= 3600 + + def test_returns_none_when_cooldown_expired(self, db_session): + user_id = _create_user(db_session) + long_ago = datetime.now(timezone.utc) - timedelta(hours=2) + db_session.execute( + text( + "INSERT INTO submissions (user_id, submission_type, version, status, created_at) " + "VALUES (:user_id, 'defense', '1.0.0', 'submitted', :created_at)" + ), + {"user_id": user_id, "created_at": long_ago}, + ) + db_session.flush() + + result = get_cooldown_remaining( + db_session, + user_id=user_id, + submission_type="defense", + cooldown_seconds=60, + ) + assert result is None + + def test_ignores_deleted_submissions(self, db_session): + user_id = _create_user(db_session) + just_now = datetime.now(timezone.utc) - timedelta(seconds=10) + db_session.execute( + text( + "INSERT INTO submissions " + "(user_id, submission_type, version, status, created_at, deleted_at) " + "VALUES (:user_id, 'defense', '1.0.0', 'submitted', :created_at, :deleted_at)" + ), + {"user_id": user_id, "created_at": just_now, "deleted_at": just_now}, + ) + db_session.flush() + + result = get_cooldown_remaining( + db_session, + user_id=user_id, + submission_type="defense", + cooldown_seconds=3600, + ) + assert result is None + + +class TestCheckCooldown: + def test_does_not_raise_when_no_cooldown(self, db_session): + user_id = _create_user(db_session) + check_cooldown( + db_session, + user_id=user_id, + submission_type="attack", + cooldown_seconds=0, + ) + + def test_raises_429_when_within_cooldown(self, db_session): + user_id = _create_user(db_session) + just_now = datetime.now(timezone.utc) - timedelta(seconds=5) + db_session.execute( + text( + "INSERT INTO submissions (user_id, submission_type, version, status, created_at) " + "VALUES (:user_id, 'attack', '1.0.0', 'submitted', :created_at)" + ), + {"user_id": user_id, "created_at": just_now}, + ) + db_session.flush() + + with pytest.raises(HTTPException) as exc_info: + check_cooldown( + db_session, + user_id=user_id, + submission_type="attack", + cooldown_seconds=3600, + ) + assert exc_info.value.status_code == 429 + assert "wait" in exc_info.value.detail.lower() From 9a12c980bda73fc0b61b7e731d9914383a6c4a29 Mon Sep 17 00:00:00 2001 From: thompaar003 Date: Mon, 13 Apr 2026 15:31:39 -0500 Subject: [PATCH 38/38] Revert "Merge branch 'aaron-prod-uat' into main" This reverts commit e3a3b2224821e818263ac0b736f8d3f6e34fe29d, reversing changes made to ead9cb1bb478fdf036a76e1de8fb15e393f4d8dd. --- docker-compose.prod.yaml | 169 -------------------------- services/api/core/admin.py | 13 +- services/api/main.py | 3 +- services/frontend/Dockerfile.prod | 25 ---- services/frontend/astro.config.mjs | 4 - services/frontend/package.json | 1 - services/frontend/src/lib/adminApi.ts | 3 +- 7 files changed, 6 insertions(+), 212 deletions(-) delete mode 100644 docker-compose.prod.yaml delete mode 100644 services/frontend/Dockerfile.prod diff --git a/docker-compose.prod.yaml b/docker-compose.prod.yaml deleted file mode 100644 index 958699a..0000000 --- a/docker-compose.prod.yaml +++ /dev/null @@ -1,169 +0,0 @@ -services: - postgres: - image: postgres:18 - container_name: postgres-db - restart: unless-stopped - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: password123 - POSTGRES_DB: mlsec - volumes: - - pgdata:/var/lib/postgresql - - ./services/postgres/init:/docker-entrypoint-initdb.d - networks: - - backend_net - - minio: - image: minio/minio:latest - container_name: mlsec-minio - restart: unless-stopped - environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} - command: server /data --console-address ":9001" - volumes: - - miniodata:/data - networks: - - backend_net - - rabbitmq: - image: rabbitmq:3-management - container_name: rabbitmq - restart: unless-stopped - environment: - RABBITMQ_DEFAULT_USER: mlsec - RABBITMQ_DEFAULT_PASS: mlsec - volumes: - - rabbitmqdata:/var/lib/rabbitmq - networks: - - backend_net - - redis: - image: redis:7-alpine - container_name: mlsec-redis - restart: unless-stopped - command: redis-server --appendonly yes - volumes: - - redisdata:/data - networks: - - backend_net - - api: - build: - context: ./services/api - dockerfile: Dockerfile - container_name: mlsec-api - restart: unless-stopped - environment: - DATABASE_URL: postgresql://postgres:password123@postgres:5432/mlsec - CELERY_BROKER_URL: amqp://mlsec:mlsec@rabbitmq:5672// - REDIS_URL: redis://redis:6379/0 - MINIO_ENDPOINT: minio:9000 - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} - MINIO_SECURE: "false" - # Production-specific settings - ADMIN_LOCALHOST_ONLY: "false" - ADMIN_ALLOWED_HOSTS: '["mlsec2.com", "api:8000"]' - ADMIN_TRUSTED_PROXY_HOSTS: '["127.0.0.1", "::1", "172.16.0.0/12", "172.17.0.0/12", "172.18.0.0/12", "172.19.0.0/12", "172.20.0.0/12", "172.21.0.0/12", "172.22.0.0/12"]' - ADMIN_ALLOWED_NETWORKS: '["172.16.0.0/12"]' - CORS_ALLOW_ORIGINS: '["http://mlsec2.com", "http://localhost:4321"]' - CORS_ALLOW_ORIGIN_REGEX: '^https?://(mlsec2\.com|localhost)(:\d+)?$' - volumes: - - ./config.yaml:/app/config.yaml:ro - networks: - - backend_net - depends_on: - - postgres - - rabbitmq - - redis - - minio - - worker: - build: - context: ./services/worker - dockerfile: Dockerfile - container_name: mlsec-worker - restart: unless-stopped - environment: - DATABASE_URL: postgresql://postgres:password123@postgres:5432/mlsec - CELERY_BROKER_URL: amqp://mlsec:mlsec@rabbitmq:5672// - CELERY_DEFAULT_QUEUE: mlsec - REDIS_URL: redis://redis:6379/0 - MINIO_ENDPOINT: minio:9000 - MINIO_ACCESS_KEY: ${MINIO_ROOT_USER:-mlsec_minio_admin} - MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD:-mlsec_minio_password_change_in_production} - MINIO_SECURE: "false" - VIRUSTOTAL_API_KEY: ${VIRUSTOTAL_API_KEY:-} - CELERY_CONCURRENCY: "${WORKER_CONCURRENCY:-4}" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./config.yaml:/app/config.yaml:ro - - worker_cache:/app/cache - group_add: - - "${DOCKER_GID:-999}" - networks: - - backend_net - - defense_net - depends_on: - - postgres - - rabbitmq - - redis - - minio - - frontend: - build: - context: ./services/frontend - dockerfile: Dockerfile.prod - container_name: mlsec-frontend - restart: unless-stopped - environment: - PUBLIC_API_URL: "http://mlsec2.com" - API_INTERNAL_URL: "http://api:8000" - networks: - - backend_net - depends_on: - - api - - nginx: - image: nginx:alpine - container_name: mlsec-nginx - restart: unless-stopped - ports: - - "80:80" - volumes: - - ./services/nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro - networks: - - backend_net - depends_on: - - api - - frontend - - # Gateway to control traffic for student containers - mlsec-gateway: - build: - context: ./services/gateway - dockerfile: Dockerfile - container_name: mlsec-gateway - restart: unless-stopped - cap_add: - - NET_ADMIN - sysctls: - - net.ipv4.ip_forward=1 - networks: - - defense_net - -networks: - backend_net: - name: backend_net - driver: bridge - defense_net: - name: defense_net - driver: bridge - -volumes: - pgdata: - rabbitmqdata: - redisdata: - miniodata: - worker_cache: diff --git a/services/api/core/admin.py b/services/api/core/admin.py index 2053fdc..62ad2b8 100644 --- a/services/api/core/admin.py +++ b/services/api/core/admin.py @@ -149,7 +149,6 @@ def _hash_token(token: str) -> str: def require_admin_origin(request: Request, *, require_present: bool = True) -> None: """Ensure Origin/Referer points to localhost (and is present if required).""" - settings = get_settings() origin = request.headers.get("origin") referer = request.headers.get("referer") @@ -168,22 +167,18 @@ def _origin_host(value: str) -> str | None: if origin: origin_host = _origin_host(origin) - if not _is_loopback_host(origin_host) and not _is_in_allowed_hosts( - origin_host, settings.admin_allowed_hosts - ): + if not _is_loopback_host(origin_host): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Admin actions require localhost or allowed origin", + detail="Admin actions require localhost origin", ) if referer: referer_host = _origin_host(referer) - if not _is_loopback_host(referer_host) and not _is_in_allowed_hosts( - referer_host, settings.admin_allowed_hosts - ): + if not _is_loopback_host(referer_host): raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, - detail="Admin actions require localhost or allowed origin", + detail="Admin actions require localhost origin", ) diff --git a/services/api/main.py b/services/api/main.py index 93ce861..b5a0583 100644 --- a/services/api/main.py +++ b/services/api/main.py @@ -68,8 +68,7 @@ def create_app() -> FastAPI: app.include_router(queue_router) app.include_router(submissions_router, prefix="/api") app.include_router(leaderboard_router) - app.include_router(admin_router) # this needs to be fixed but I think this will fix our problems for now - app.include_router(admin_router, prefix="/api") + app.include_router(admin_router) return app diff --git a/services/frontend/Dockerfile.prod b/services/frontend/Dockerfile.prod deleted file mode 100644 index 4e4f50e..0000000 --- a/services/frontend/Dockerfile.prod +++ /dev/null @@ -1,25 +0,0 @@ -FROM node:20-alpine AS build - -WORKDIR /app - -COPY package.json package-lock.json* ./ -RUN npm install - -COPY . . -RUN npm run build - -FROM node:20-alpine AS runtime - -WORKDIR /app - -COPY --from=build /app/dist ./dist -COPY --from=build /app/package.json ./package.json -COPY --from=build /app/node_modules ./node_modules - -ENV HOST=0.0.0.0 -ENV PORT=4321 -ENV NODE_ENV=production - -EXPOSE 4321 - -CMD ["node", "./dist/server/entry.mjs"] diff --git a/services/frontend/astro.config.mjs b/services/frontend/astro.config.mjs index dbc42a9..8e11199 100644 --- a/services/frontend/astro.config.mjs +++ b/services/frontend/astro.config.mjs @@ -3,16 +3,12 @@ import { defineConfig } from 'astro/config'; import react from '@astrojs/react'; import tailwind from '@astrojs/tailwind'; -import node from '@astrojs/node'; const apiTarget = process.env.API_INTERNAL_URL || 'http://127.0.0.1:8000'; // https://astro.build/config export default defineConfig({ output: 'server', - adapter: node({ - mode: 'standalone', - }), integrations: [react(), tailwind()], vite: { server: { diff --git a/services/frontend/package.json b/services/frontend/package.json index 70f81d8..f34cd5b 100644 --- a/services/frontend/package.json +++ b/services/frontend/package.json @@ -9,7 +9,6 @@ "astro": "astro" }, "dependencies": { - "@astrojs/node": "^9.0.0", "@astrojs/react": "^4.4.2", "@tailwindcss/vite": "^4.1.18", "@types/react": "^19.2.10", diff --git a/services/frontend/src/lib/adminApi.ts b/services/frontend/src/lib/adminApi.ts index 40906b9..cc02002 100644 --- a/services/frontend/src/lib/adminApi.ts +++ b/services/frontend/src/lib/adminApi.ts @@ -8,8 +8,7 @@ */ export async function adminFetch(path: string, init: RequestInit = {}): Promise { - const apiPath = path.startsWith('/admin') ? `/api${path}` : path; - return fetch(apiPath, { + return fetch(path, { ...init, credentials: 'include', });