""" Pattern Validation Test Suite Tests to ensure all error patterns are syntactically correct and complete. Validates JSON schema, regex syntax, i18n completeness, and metadata. Run: pytest tests/test_pattern_validation.py -v """ import json import re from pathlib import Path from typing import Dict, List, Set import pytest # Project paths PROJECT_ROOT = Path(__file__).parent.parent PATTERNS_DIR = PROJECT_ROOT / "patterns" SCHEMA_PATH = PATTERNS_DIR / "schema.json" I18N_PATH = PROJECT_ROOT / "i18n.py" # Expected languages SUPPORTED_LANGUAGES = ["en", "zh_TW", "zh_CN", "ja", "de", "fr", "it", "es", "ko"] # Valid categories VALID_CATEGORIES = { "data_type", "framework", "memory", "model_loading", "custom_nodes", "workflow", "generic" } def load_all_pattern_files() -> List[Path]: """Load all JSON pattern files from patterns/ directory.""" pattern_files = [] for pattern_type in ["builtin", "community"]: pattern_dir = PATTERNS_DIR / pattern_type if pattern_dir.exists(): pattern_files.extend(pattern_dir.glob("*.json")) return pattern_files def load_pattern_file(file_path: Path) -> Dict: """Load and parse a single pattern JSON file.""" with open(file_path, "r", encoding="utf-8") as f: return json.load(f) def extract_error_keys_from_i18n() -> Set[str]: """Extract all ERROR_KEYS from i18n.py""" with open(I18N_PATH, "r", encoding="utf-8") as f: content = f.read() # Find ERROR_KEYS dictionary start = content.find("ERROR_KEYS") if start == -1: return set() # Extract keys (simplified parsing) keys = set() lines = content[start:].split("\n") for line in lines: if '"' in line and ":" in line: # Extract key from line like: "oom": "oom", match = re.search(r'"([^"]+)":\s*"', line) if match: keys.add(match.group(1)) if line.strip() == "}": break return keys def extract_suggestions_from_i18n() -> Dict[str, Set[str]]: """Extract all SUGGESTIONS keys for each language from i18n.py""" with open(I18N_PATH, "r", encoding="utf-8") as f: content = f.read() suggestions_by_lang = {} # Find SUGGESTIONS dictionary start = content.find("SUGGESTIONS") if start == -1: return suggestions_by_lang # Extract suggestions for each language for lang in SUPPORTED_LANGUAGES: lang_start = content.find(f'"{lang}":', start) if lang_start == -1: continue keys = set() lines = content[lang_start:].split("\n") brace_count = 0 started = False for line in lines: if "{" in line: brace_count += line.count("{") started = True if "}" in line: brace_count -= line.count("}") if started and '"' in line and ":" in line: match = re.search(r'"([^"]+)":\s*"', line) if match: keys.add(match.group(1)) if started and brace_count == 0: break suggestions_by_lang[lang] = keys return suggestions_by_lang class TestPatternValidation: """Pattern validation test suite""" @pytest.fixture(scope="class") def all_patterns(self): """Load all patterns from all JSON files""" all_patterns = [] for file_path in load_all_pattern_files(): data = load_pattern_file(file_path) for pattern in data.get("patterns", []): pattern["_source_file"] = str(file_path.relative_to(PROJECT_ROOT)) all_patterns.append(pattern) return all_patterns @pytest.fixture(scope="class") def error_keys(self): """Extract ERROR_KEYS from i18n.py""" return extract_error_keys_from_i18n() @pytest.fixture(scope="class") def suggestions(self): """Extract SUGGESTIONS from i18n.py""" return extract_suggestions_from_i18n() def test_all_patterns_valid_json(self): """Test 1: All pattern files are valid JSON""" for file_path in load_all_pattern_files(): try: load_pattern_file(file_path) except json.JSONDecodeError as e: pytest.fail(f"Invalid JSON in {file_path}: {e}") def test_all_patterns_have_required_fields(self, all_patterns): """Test 2: All patterns have required fields""" required_fields = ["id", "pattern", "priority", "category", "suggestion_key"] for pattern in all_patterns: for field in required_fields: assert field in pattern, ( f"Pattern {pattern.get('id', 'UNKNOWN')} in {pattern['_source_file']} " f"missing required field: {field}" ) def test_all_regex_compile(self, all_patterns): """Test 3: All regex patterns compile successfully""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") regex_str = pattern.get("pattern", "") try: re.compile(regex_str) except re.error as e: pytest.fail( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has invalid regex: {e}" ) def test_all_suggestions_exist(self, all_patterns, error_keys, suggestions): """Test 4: All suggestion_key values exist in i18n.py""" # Check ERROR_KEYS mapping for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") suggestion_key = pattern.get("suggestion_key", "") assert suggestion_key in error_keys, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has suggestion_key '{suggestion_key}' not found in ERROR_KEYS" ) # Check all languages have translations for lang in SUPPORTED_LANGUAGES: assert lang in suggestions, f"Language '{lang}' not found in SUGGESTIONS" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") suggestion_key = pattern.get("suggestion_key", "") # Map through ERROR_KEYS to get the actual key if suggestion_key in error_keys: # For simplicity, assume suggestion_key IS the final key # (in reality, ERROR_KEYS maps pattern keys to suggestion keys) assert suggestion_key in suggestions[lang], ( f"Pattern {pattern_id} missing {lang} translation for '{suggestion_key}'" ) def test_priority_ranges(self, all_patterns): """Test 5: All priorities are within valid range 50-95""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") priority = pattern.get("priority", 0) assert isinstance(priority, int), ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has non-integer priority: {priority}" ) assert 50 <= priority <= 95, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has priority {priority} outside valid range 50-95" ) def test_no_duplicate_ids(self, all_patterns): """Test 6: No duplicate pattern IDs""" seen_ids = {} for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") source_file = pattern["_source_file"] if pattern_id in seen_ids: pytest.fail( f"Duplicate pattern ID '{pattern_id}' found in:\n" f" - {seen_ids[pattern_id]}\n" f" - {source_file}" ) seen_ids[pattern_id] = source_file def test_category_validity(self, all_patterns): """Test 7: All categories are valid""" for pattern in all_patterns: pattern_id = pattern.get("id", "UNKNOWN") category = pattern.get("category", "") assert category in VALID_CATEGORIES, ( f"Pattern {pattern_id} in {pattern['_source_file']} " f"has invalid category '{category}'. " f"Valid categories: {', '.join(sorted(VALID_CATEGORIES))}" ) def test_pattern_statistics(self, all_patterns): """Test 8: Print pattern statistics (always passes)""" total = len(all_patterns) by_category = {} by_source = {} for pattern in all_patterns: category = pattern.get("category", "unknown") source_file = pattern["_source_file"] by_category[category] = by_category.get(category, 0) + 1 by_source[source_file] = by_source.get(source_file, 0) + 1 print(f"\n{'='*60}") print(f"Pattern Statistics") print(f"{'='*60}") print(f"Total patterns: {total}") print(f"\nBy category:") for cat, count in sorted(by_category.items()): print(f" {cat:20s}: {count}") print(f"\nBy source file:") for src, count in sorted(by_source.items()): print(f" {src:40s}: {count}") print(f"{'='*60}\n") if __name__ == "__main__": pytest.main([__file__, "-v", "-s"])