feat: Add Pydantic YAML validation for card files - Add pydantic as production dependency - Create validation models - Integrate validation in convert.py - Add 21 comprehensive tests - All 27 tests passing

Author: khushal-winner

.clusterfuzzlite/build.sh

@@ -3,18 +3,19 @@
 # build project
 python3 -m pip install -r requirements.txt
 python3 -m pip install -r install_cornucopia_deps.txt
+python3 -m pip install pydantic==2.12.5
 
 # Build fuzzers into $OUT. These could be detected in other ways.
 for fuzzer in $(find "$SRC/cornucopia/tests/scripts" -name '*_fuzzer.py'); do
     fuzzer_basename=$(basename -s .py "$fuzzer")
     fuzzer_package=${fuzzer_basename}.pkg
 
-    python3 -m PyInstaller --distpath "$OUT" --onefile --exclude IPython --paths "$SRC"/cornucopia:"$SRC"/cornucopia/scripts:"$SRC"/cornucopia/tests/test-files --hidden-import scripts --collect-submodules scripts --name "$fuzzer_package" "$fuzzer"
+    python3 -m PyInstaller --distpath "$OUT" --onefile --exclude IPython --paths "$SRC"/cornucopia:"$SRC"/cornucopia/scripts:"$SRC"/cornucopia/tests/test-files --hidden-import scripts --collect-submodules scripts --hidden-import=pydantic --collect-submodules pydantic --name "$fuzzer_package" "$fuzzer"
 
     echo "#!/bin/sh
 # LLVMFuzzerTestOneInput for fuzzer detection.
 echo "fuzzing now, this is what is here"
-this_dir=\$(dirname \"\$0\")
+this_dir=\$(dirname "\$0")
 ASAN_OPTIONS=\$ASAN_OPTIONS:symbolize=1:external_symbolizer_path=\$this_dir/llvm-symbolizer:detect_leaks=0 \
 \$this_dir/$fuzzer_package \$@" > "$OUT"/"$fuzzer_basename"
     chmod +x "$OUT/$fuzzer_basename"

Pipfile

@@ -35,6 +35,7 @@ pathvalidate = "==3.3.1"
 security = "==1.3.1"
 colorama = "*"
 mypy = "*"
+pydantic = "==2.12.5"
 
 [requires]
 python_version = "3.12"

scripts/card_models.py

@@ -0,0 +1,101 @@
+"""Pydantic models for validating YAML card and mapping structures.
+
+This module provides validation models for OWASP Cornucopia YAML files,
+ensuring structural integrity while maintaining backward compatibility.
+"""
+
+from pydantic import BaseModel, Field, ValidationError, ConfigDict
+from typing import Dict, List, Optional, Any
+
+
+class Card(BaseModel):
+    """Minimal validation for card YAML structure.
+    
+    Validates core required fields while allowing additional fields
+    like capec, stride, owasp_asvs, etc. through extra="allow".
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    id: str = Field(..., min_length=1, description="Required card ID (e.g. VE2, AT3)")
+    value: str = Field(..., min_length=1, description="Required card value (e.g. 2, K, A)")
+    desc: str = Field(..., min_length=10, description="Required card description")
+    url: Optional[str] = None
+    misc: Optional[str] = None
+
+
+class MappingCard(BaseModel):
+    """Minimal validation for mapping card structure.
+    
+    Mapping files don't have descriptions, only id, value, and mapping data.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    id: str = Field(..., min_length=1, description="Required card ID (e.g. VE2, AT3)")
+    value: str = Field(..., min_length=1, description="Required card value (e.g. 2, K, A)")
+    url: Optional[str] = None
+
+
+class Suit(BaseModel):
+    """Validation for suit structure.
+    
+    A suit contains an ID, name, and a collection of cards.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    id: str = Field(..., min_length=1)
+    name: str = Field(..., min_length=1)
+    cards: List[Card] = Field(..., min_length=1)
+
+
+class MappingSuit(BaseModel):
+    """Validation for mapping suit structure.
+    
+    A mapping suit contains an ID, name, and a collection of mapping cards.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    id: str = Field(..., min_length=1)
+    name: str = Field(..., min_length=1)
+    cards: List[MappingCard] = Field(..., min_length=1)
+
+
+class Meta(BaseModel):
+    """Validation for metadata section.
+    
+    Contains edition, component, language, and version information.
+    Additional fields like layouts, templates, and languages are allowed.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    edition: str
+    component: str
+    language: str
+    version: str
+
+
+class CardYAML(BaseModel):
+    """Top-level card YAML structure.
+    
+    Represents the complete structure of a card YAML file,
+    including metadata, suits, and optional paragraphs.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    meta: Meta
+    suits: List[Suit] = Field(..., min_length=1)
+
+
+class MappingYAML(BaseModel):
+    """Top-level mapping YAML structure.
+    
+    Represents the complete structure of a mapping YAML file,
+    which includes additional mapping information like CAPEC, ASVS, etc.
+    """
+    model_config = ConfigDict(extra="allow")
+    
+    meta: Meta
+    suits: List[MappingSuit] = Field(..., min_length=1)
+
+
+# Export ValidationError for convenience
+__all__ = ['Card', 'MappingCard', 'Suit', 'MappingSuit', 'Meta', 'CardYAML', 'MappingYAML', 'ValidationError']

scripts/convert.py

@@ -18,6 +18,14 @@
 from pathvalidate.argparse import validate_filepath_arg
 from pathvalidate import sanitize_filepath
 
+# Add parent directory to path for card_models import
+script_dir = os.path.dirname(os.path.abspath(__file__))
+parent_dir = os.path.dirname(script_dir)
+if parent_dir not in sys.path:
+    sys.path.insert(0, parent_dir)
+
+from scripts.card_models import CardYAML, MappingYAML, ValidationError as PydanticValidationError
+
 
 class ConvertVars:
     BASE_PATH = os.path.split(os.path.dirname(os.path.realpath(__file__)))[0]
@@ -636,6 +644,13 @@ def get_mapping_data_for_edition(
     with open(mappingfile, "r", encoding="utf-8") as f:
         try:
             data = yaml.safe_load(f)
+            # Validate structure with Pydantic
+            try:
+                validated = MappingYAML(**data)
+                data = validated.model_dump(exclude_none=True)  # Convert back to dict, exclude None values
+            except PydanticValidationError as ve:
+                logging.warning(f"Mapping file validation warning for {mappingfile}: {ve}")
+                # Continue with unvalidated data for backward compatibility
         except yaml.YAMLError as e:
             logging.info(f"Error loading yaml file: {mappingfile}. Error = {e}")
             data = {}
@@ -737,6 +752,14 @@ def get_language_data(
     with open(language_file, "r", encoding="utf-8") as f:
         try:
             data = yaml.safe_load(f)
+            # Validate structure with Pydantic
+            try:
+                validated = CardYAML(**data)
+                data = validated.model_dump(exclude_none=True)  # Convert back to dict, exclude None values
+            except PydanticValidationError as ve:
+                logging.error(f"Card file validation failed for {language_file}: {ve}")
+                # For card files, validation failure is more critical
+                data = {}
         except yaml.YAMLError as e:
             logging.error(f"Error loading yaml file: {language_file}. Error = {e}")
             data = {}