feat: Add Pydantic YAML validation for card files

.clusterfuzzlite/build.sh

@@ -3,18 +3,19 @@
 # build project
 python3 -m pip install -r requirements.txt
 python3 -m pip install -r install_cornucopia_deps.txt
+python3 -m pip install pydantic==2.12.5
 
 # Build fuzzers into $OUT. These could be detected in other ways.
 for fuzzer in $(find "$SRC/cornucopia/tests/scripts" -name '*_fuzzer.py'); do
     fuzzer_basename=$(basename -s .py "$fuzzer")
     fuzzer_package=${fuzzer_basename}.pkg
 
-    python3 -m PyInstaller --distpath "$OUT" --onefile --exclude IPython --paths "$SRC"/cornucopia:"$SRC"/cornucopia/scripts:"$SRC"/cornucopia/tests/test-files --hidden-import scripts --collect-submodules scripts --name "$fuzzer_package" "$fuzzer"
+    python3 -m PyInstaller --distpath "$OUT" --onefile --exclude IPython --paths "$SRC"/cornucopia:"$SRC"/cornucopia/scripts:"$SRC"/cornucopia/tests/test-files --hidden-import scripts --collect-submodules scripts --hidden-import=pydantic --collect-submodules pydantic --name "$fuzzer_package" "$fuzzer"
 
     echo "#!/bin/sh
 # LLVMFuzzerTestOneInput for fuzzer detection.
 echo "fuzzing now, this is what is here"
-this_dir=\$(dirname \"\$0\")
+this_dir=\$(dirname "\$0")
 ASAN_OPTIONS=\$ASAN_OPTIONS:symbolize=1:external_symbolizer_path=\$this_dir/llvm-symbolizer:detect_leaks=0 \
 \$this_dir/$fuzzer_package \$@" > "$OUT"/"$fuzzer_basename"
     chmod +x "$OUT/$fuzzer_basename"

Pipfile

@@ -21,7 +21,7 @@ pypng = "==0.20220715.0"
 qrcode = "==8.2"
 requests = "==2.32.5"
 types-requests = "==2.32.4.20260107"
-typing_extensions = "==4.8.0"
+typing_extensions = ">=4.14.1"
 urllib3 = "==2.6.3"
 charset-normalizer = "==3.4.4"
 python-docx = "==1.1.0"
@@ -35,6 +35,8 @@ pathvalidate = "==3.3.1"
 security = "==1.3.1"
 colorama = "*"
 mypy = "*"
+pydantic = {version = "==2.12.5", hash = "sha256:e561593fccf61e8a20fc46dfc2dfe075b8be7d0188df33f221ad1f0139180f9d"}
+pydantic-core = {version = "==2.41.5", hash = "sha256:8e7c86f27c585ef37c35e56a96363ab8de4e549a95512445b85c96d3e2f7c1bf"}
 
 [requires]
-python_version = "3.12"
+python_version = "3.14"

scripts/card_models.py

@@ -0,0 +1,101 @@
+"""Pydantic models for validating YAML card and mapping structures.
+
+This module provides validation models for OWASP Cornucopia YAML files,
+ensuring structural integrity while maintaining backward compatibility.
+"""
+
+from pydantic import BaseModel, Field, ValidationError, ConfigDict
+from typing import Dict, List, Optional, Any
+
+
+class Card(BaseModel):
+    """Minimal validation for card YAML structure.
+
+    Validates core required fields while allowing additional fields
+    like capec, stride, owasp_asvs, etc. through extra="allow".
+    """
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(..., min_length=1, description="Required card ID (e.g. VE2, AT3)")
+    value: str = Field(..., min_length=1, description="Required card value (e.g. 2, K, A)")
+    desc: str = Field(..., min_length=10, description="Required card description")
+    url: Optional[str] = None
+    misc: Optional[str] = None
+
+
+class MappingCard(BaseModel):
+    """Minimal validation for mapping card structure.
+
+    Mapping files don't have descriptions, only id, value, and mapping data.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(..., min_length=1, description="Required card ID (e.g. VE2, AT3)")
+    value: str = Field(..., min_length=1, description="Required card value (e.g. 2, K, A)")
+    url: Optional[str] = None
+
+
+class Suit(BaseModel):
+    """Validation for suit structure.
+
+    A suit contains an ID, name, and a collection of cards.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(..., min_length=1)
+    name: str = Field(..., min_length=1)
+    cards: List[Card] = Field(..., min_length=1)
+
+
+class MappingSuit(BaseModel):
+    """Validation for mapping suit structure.
+
+    A mapping suit contains an ID, name, and a collection of mapping cards.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    id: str = Field(..., min_length=1)
+    name: str = Field(..., min_length=1)
+    cards: List[MappingCard] = Field(..., min_length=1)
+
+
+class Meta(BaseModel):
+    """Validation for metadata section.
+
+    Contains edition, component, language, and version information.
+    Additional fields like layouts, templates, and languages are allowed.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    edition: str
+    component: str
+    language: str
+    version: str
+
+
+class CardYAML(BaseModel):
+    """Top-level card YAML structure.
+
+    Represents the complete structure of a card YAML file,
+    including metadata, suits, and optional paragraphs.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    meta: Meta
+    suits: List[Suit] = Field(..., min_length=1)
+
+
+class MappingYAML(BaseModel):
+    """Top-level mapping YAML structure.
+
+    Represents the complete structure of a mapping YAML file,
+    which includes additional mapping information like CAPEC, ASVS, etc.
+    """
+    model_config = ConfigDict(extra="allow")
+
+    meta: Meta
+    suits: List[MappingSuit] = Field(..., min_length=1)
+
+
+# Export ValidationError for convenience
+__all__ = ['Card', 'MappingCard', 'Suit', 'MappingSuit', 'Meta', 'CardYAML', 'MappingYAML', 'ValidationError']

scripts/convert.py

@@ -18,6 +18,26 @@
 from pathvalidate.argparse import validate_filepath_arg
 from pathvalidate import sanitize_filepath
 
+# Add parent directory to path for card_models import
+script_dir = os.path.dirname(os.path.abspath(__file__))
+parent_dir = os.path.dirname(script_dir)
+if parent_dir not in sys.path:
+    sys.path.insert(0, parent_dir)
+
+# Try to import pydantic models, fall back gracefully if not available
+try:
+    from scripts.card_models import CardYAML, MappingYAML, ValidationError as PydanticValidationError
+    PYDANTIC_AVAILABLE = True
+except ImportError:
+    # Define fallback classes to prevent crashes when pydantic is not available
+    class MockValidationError(Exception):
+        pass
+
+    PydanticValidationError = MockValidationError
+    CardYAML = None
+    MappingYAML = None
+    PYDANTIC_AVAILABLE = False
+
 
 class ConvertVars:
     BASE_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
@@ -636,6 +656,14 @@ def get_mapping_data_for_edition(
     with open(mappingfile, "r", encoding="utf-8") as f:
         try:
             data = yaml.safe_load(f)
+            # Validate structure with Pydantic if available
+            if PYDANTIC_AVAILABLE:
+                try:
+                    validated = MappingYAML(**data)
+                    data = validated.model_dump(exclude_none=True)  # Convert back to dict, exclude None values
+                except PydanticValidationError as ve:
+                    logging.warning(f"Mapping file validation warning for {mappingfile}: {ve}")
+                    # Continue with unvalidated data for backward compatibility
         except yaml.YAMLError as e:
             logging.info(f"Error loading yaml file: {mappingfile}. Error = {e}")
             data = {}
@@ -737,6 +765,15 @@ def get_language_data(
     with open(language_file, "r", encoding="utf-8") as f:
         try:
             data = yaml.safe_load(f)
+            # Validate structure with Pydantic if available
+            if PYDANTIC_AVAILABLE:
+                try:
+                    validated = CardYAML(**data)
+                    data = validated.model_dump(exclude_none=True)  # Convert back to dict, exclude None values
+                except PydanticValidationError as ve:
+                    logging.error(f"Card file validation failed for {language_file}: {ve}")
+                    # For card files, validation failure is more critical
+                    data = {}
         except yaml.YAMLError as e:
             logging.error(f"Error loading yaml file: {language_file}. Error = {e}")
             data = {}