""" Pattern Validation Test Suite Tests to ensure all error patterns are syntactically correct and complete. Validates JSON schema, regex syntax, i18n completeness, and metadata. Run: pytest tests/test_pattern_validation.py -v ═══════════════════════════════════════════════════════════════════════════ ⚠️ CRITICAL: Field Name Consistency Warning ═══════════════════════════════════════════════════════════════════════════ This test suite validates JSON pattern files against the schema. IMPORTANT: JSON Schema Field Names (DO NOT CHANGE): - patterns/schema.json uses "regex" (NOT "pattern") - patterns/schema.json uses "error_key" (NOT "suggestion_key") Common Mistake: ❌ required_fields = ["id", "pattern", "suggestion_key", ...] ✅ required_fields = ["id", "regex", "error_key", ...] If tests fail with "missing required field", check that test expectations match the actual schema definition in patterns/schema.json. Last Modified: 2026-01-03 (Fixed field name mismatch) ═══════════════════════════════════════════════════════════════════════════ """ import json import re from pathlib import Path from typing import Dict, List, Set import pytest import i18n # Project paths PROJECT_ROOT = Path(__file__).parent.parent PATTERNS_DIR = PROJECT_ROOT / "patterns" SCHEMA_PATH = PATTERNS_DIR / "schema.json" # Expected languages SUPPORTED_LANGUAGES = ["en", "zh_TW", "zh_CN", "ja", "de", "fr", "it", "es", "ko"] # Valid categories VALID_CATEGORIES = { "data_type", "framework", "memory", "model_loading", "custom_nodes", "workflow", "generic" } def load_all_pattern_files() -> List[Path]: """Load all JSON pattern files from patterns/ directory.""" pattern_files = [] for pattern_type in ["builtin", "community"]: pattern_dir = PATTERNS_DIR / pattern_type if pattern_dir.exists(): pattern_files.extend(pattern_dir.glob("*.json")) return pattern_files def load_pattern_file(file_path: Path) -> Dict: """Load and parse a single pattern JSON file.""" with open(file_path, "r", encoding="utf-8") as f: return json.load(f) def extract_error_keys_from_i18n() -> Dict[str, str]: """ Extract ERROR_KEYS mapping from the i18n loader. Returns: Dict mapping uppercase keys to lowercase suggestion keys. Example: {"TYPE_MISMATCH": "type_mismatch", "CUDNN_ERROR": "cudnn_error", ...} """ return dict(i18n.ERROR_KEYS) def extract_suggestions_from_i18n() -> Dict[str, Set[str]]: """Extract all SUGGESTIONS keys for each language from loaded i18n data.""" return {lang: set(i18n.SUGGESTIONS.get(lang, {})) for lang in SUPPORTED_LANGUAGES} class TestPatternValidation: """Pattern validation test suite""" @pytest.fixture(scope="class") def all_patterns(self): """Load all patterns from all JSON files""" all_patterns = [] for file_path in load_all_pattern_files(): data = load_pattern_file(file_path) for pattern in data.get("patterns", []): pattern["_source_file"] = str(file_path.relative_to(PROJECT_ROOT)) all_patterns.append(pattern) return all_patterns @pytest.fixture(scope="class") def error_keys(self): """Extract ERROR_KEYS from i18n.py""" return extract_error_keys_from_i18n() @pytest.fixture(scope="class") def suggestions(self): """Extract SUGGESTIONS from i18n.py""" return extract_suggestions_from_i18n() def test_all_patterns_valid_json(self): """Test 1: All pattern files are valid JSON""" for file_path in load_all_pattern_files(): try: load_pattern_file(file_path) except json.JSONDecodeError as e: pytest.fail(f"Invalid JSON in {file_path}: {e}") def test_all_patterns_have_required_fields(self, all_patterns): """Test 2: All patterns have required fields""" # ⚠️ CRITICAL: These field names MUST match patterns/schema.json # DO NOT change "regex" to "pattern" or "error_key" to "suggestion_key" # See file docstring for detailed explanation required_fields = ["id", "regex", "error_key", "priority", "category"] for pattern in all_patterns: for field in required_fields: assert field in pattern, ( f"Pattern {pattern.get('id', 'UNKNOWN')} in {pattern['_source_file']} " f"missing required field: {field}" ) def test_all_regex_compile(self, all_patterns): """Test 3: All regex patterns compile successfully""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") # ⚠️ CRITICAL: Use "regex" (schema field name), NOT "pattern" regex_str = pattern.get("regex", "") try: re.compile(regex_str) except re.error as e: pytest.fail( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has invalid regex: {e}" ) def test_all_suggestions_exist(self, all_patterns, error_keys, suggestions): """Test 4: All error_key values exist in i18n.py ERROR_KEYS and SUGGESTIONS""" # Check ERROR_KEYS mapping for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") # ⚠️ CRITICAL: Use "error_key" (schema field name), NOT "suggestion_key" error_key = pattern.get("error_key", "") assert error_key in error_keys, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has error_key '{error_key}' not found in ERROR_KEYS" ) # Check all languages have translations for lang in SUPPORTED_LANGUAGES: assert lang in suggestions, f"Language '{lang}' not found in SUGGESTIONS" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") error_key = pattern.get("error_key", "") # Get the suggestion key from ERROR_KEYS mapping # Example: error_key="CUDNN_ERROR" -> suggestion_key="cudnn_error" suggestion_key = error_keys.get(error_key, "") assert suggestion_key in suggestions[lang], ( f"Pattern {pattern_id} (error_key: {error_key}) missing {lang} translation. " f"Expected suggestion key '{suggestion_key}' in SUGGESTIONS['{lang}']" ) def test_priority_ranges(self, all_patterns): """Test 5: All priorities are within valid range 0-100""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") priority = pattern.get("priority", 0) assert isinstance(priority, int), ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has non-integer priority: {priority}" ) assert 0 <= priority <= 100, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has priority {priority} outside valid range 0-100" ) def test_no_duplicate_ids(self, all_patterns): """Test 6: No duplicate pattern IDs""" seen_ids = {} for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") source_file = pattern["_source_file"] if pattern_id in seen_ids: pytest.fail( f"Duplicate pattern ID '{pattern_id}' found in:\n" f" - {seen_ids[pattern_id]}\n" f" - {source_file}" ) seen_ids[pattern_id] = source_file def test_category_validity(self, all_patterns): """Test 7: All categories are valid""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") category = pattern.get("category", "") assert category in VALID_CATEGORIES, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has invalid category '{category}'. " f"Valid categories: {', '.join(sorted(VALID_CATEGORIES))}" ) def test_pattern_statistics(self, all_patterns): """Test 8: Print pattern statistics (always passes)""" total = len(all_patterns) by_category = {} by_source = {} for pattern in all_patterns: category = pattern.get("category", "unknown") source_file = pattern["_source_file"] by_category[category] = by_category.get(category, 0) + 1 by_source[source_file] = by_source.get(source_file, 0) + 1 print(f"\n{'='*60}") print(f"Pattern Statistics") print(f"{'='*60}") print(f"Total patterns: {total}") print(f"\nBy category:") for cat, count in sorted(by_category.items()): print(f" {cat:20s}: {count}") print(f"\nBy source file:") for src, count in sorted(by_source.items()): print(f" {src:40s}: {count}") print(f"{'='*60}\n") if __name__ == "__main__": pytest.main([__file__, "-v", "-s"])