New data structures for tokenization pipeline: tokenizer/base.py: - ImageToken: unified token format (text, confidence, source, type) - TokenType enum: TAG, PHRASE, KEY_VALUE, SENTENCE - TokenSource enum: WD14, JOYTAG, PIXAI, FLORENCE_*, etc. - TokenBatch: collection with filtering methods - SOURCE_WEIGHTS: reliability weighting per source classifier/categories.py: - CanonicalCategory enum: 9 categories + UNCATEGORIZED, FILTERED_META - SubjectDetailType enum: hair, eyes, face, body, clothing, skin, accessories - CategoryKeywords: exact/prefix/suffix/contains pattern matching - Keyword dictionaries for all categories - META_FILTER_PATTERNS: patterns to filter out - FLORENCE_KEY_TO_CATEGORY: key routing for florence_analyze classifier/base.py: - TokenClassification: classification result with primary/secondary categories - ClassifiedImage: organized tokens by category for assembly Tests: - test_phase1_infrastructure.py: 21 unit tests (all passing) - test_real_image.py: real metadata token extraction demo - sample_metadata*.json: real image metadata samples All 21 tests pass. Ready for Phase 2: Tokenization. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
365 lines
13 KiB
Python
365 lines
13 KiB
Python
"""Tests for Phase 1: Core Infrastructure.
|
|
|
|
Run standalone:
|
|
cd ComfyUI-AI-Photography-Toolkit
|
|
python tests/compose/test_phase1_infrastructure.py
|
|
|
|
Run with pytest (requires torch installed):
|
|
python -m pytest tests/compose/test_phase1_infrastructure.py -v
|
|
"""
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
# Add project root to path for standalone execution
|
|
PROJECT_ROOT = Path(__file__).parent.parent.parent
|
|
sys.path.insert(0, str(PROJECT_ROOT))
|
|
|
|
# Use importlib for standalone testing (bypasses torch dependency)
|
|
import importlib.util
|
|
|
|
|
|
def load_module(name: str, path: str):
|
|
"""Load a module directly from file path."""
|
|
spec = importlib.util.spec_from_file_location(name, PROJECT_ROOT / path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
# Load modules
|
|
tokenizer_base = load_module("tokenizer_base", "core/compose/tokenizer/base.py")
|
|
categories = load_module("categories", "core/compose/classifier/categories.py")
|
|
|
|
# Import classes
|
|
ImageToken = tokenizer_base.ImageToken
|
|
TokenType = tokenizer_base.TokenType
|
|
TokenSource = tokenizer_base.TokenSource
|
|
TokenBatch = tokenizer_base.TokenBatch
|
|
SOURCE_WEIGHTS = tokenizer_base.SOURCE_WEIGHTS
|
|
|
|
CanonicalCategory = categories.CanonicalCategory
|
|
SubjectDetailType = categories.SubjectDetailType
|
|
CategoryKeywords = categories.CategoryKeywords
|
|
CATEGORY_ORDER = categories.CATEGORY_ORDER
|
|
CATEGORY_KEYWORDS = categories.CATEGORY_KEYWORDS
|
|
SUBJECT_DETAIL_KEYWORDS = categories.SUBJECT_DETAIL_KEYWORDS
|
|
is_meta_pattern = categories.is_meta_pattern
|
|
get_florence_key_category = categories.get_florence_key_category
|
|
|
|
|
|
# =============================================================================
|
|
# ImageToken Tests
|
|
# =============================================================================
|
|
|
|
def test_image_token_creation():
|
|
"""Test basic token creation."""
|
|
token = ImageToken(
|
|
text="brown hair",
|
|
confidence=0.915,
|
|
source=TokenSource.WD14,
|
|
token_type=TokenType.TAG
|
|
)
|
|
assert token.text == "brown hair"
|
|
assert token.confidence == 0.915
|
|
assert token.source == TokenSource.WD14
|
|
assert token.token_type == TokenType.TAG
|
|
print("[PASS] test_image_token_creation")
|
|
|
|
|
|
def test_image_token_normalization():
|
|
"""Test text normalization (lowercase, whitespace)."""
|
|
token = ImageToken(
|
|
text=" Brown Hair ",
|
|
confidence=0.9,
|
|
source=TokenSource.WD14,
|
|
token_type=TokenType.TAG
|
|
)
|
|
assert token.text == "brown hair", f"Expected 'brown hair', got '{token.text}'"
|
|
assert token.metadata["original"] == "Brown Hair"
|
|
print("[PASS] test_image_token_normalization")
|
|
|
|
|
|
def test_image_token_confidence_clamping():
|
|
"""Test confidence is clamped to [0, 1]."""
|
|
token1 = ImageToken("test", 1.5, TokenSource.WD14, TokenType.TAG)
|
|
token2 = ImageToken("test", -0.5, TokenSource.WD14, TokenType.TAG)
|
|
assert token1.confidence == 1.0
|
|
assert token2.confidence == 0.0
|
|
print("[PASS] test_image_token_confidence_clamping")
|
|
|
|
|
|
def test_image_token_weighted_confidence():
|
|
"""Test weighted confidence calculation."""
|
|
token = ImageToken("test", 0.9, TokenSource.WD14, TokenType.TAG)
|
|
# WD14 has weight 1.0, so weighted = 0.9 * 1.0 = 0.9
|
|
assert token.weighted_confidence == 0.9
|
|
|
|
token2 = ImageToken("test", 0.9, TokenSource.NUDENET, TokenType.TAG)
|
|
# NUDENET has weight 0.8, so weighted = 0.9 * 0.8 = 0.72
|
|
assert abs(token2.weighted_confidence - 0.72) < 0.001
|
|
print("[PASS] test_image_token_weighted_confidence")
|
|
|
|
|
|
def test_image_token_equality():
|
|
"""Test token equality based on normalized text."""
|
|
token1 = ImageToken("brown hair", 0.9, TokenSource.WD14, TokenType.TAG)
|
|
token2 = ImageToken("Brown Hair", 0.8, TokenSource.JOYTAG, TokenType.TAG)
|
|
token3 = ImageToken("black hair", 0.9, TokenSource.WD14, TokenType.TAG)
|
|
|
|
assert token1 == token2, "Same text should be equal"
|
|
assert token1 != token3, "Different text should not be equal"
|
|
print("[PASS] test_image_token_equality")
|
|
|
|
|
|
def test_image_token_key_value():
|
|
"""Test KEY_VALUE token type."""
|
|
token = ImageToken(
|
|
text="front view",
|
|
confidence=0.95,
|
|
source=TokenSource.FLORENCE_ANALYZE,
|
|
token_type=TokenType.KEY_VALUE,
|
|
metadata={"key": "camera_angle"}
|
|
)
|
|
assert token.key == "camera_angle"
|
|
print("[PASS] test_image_token_key_value")
|
|
|
|
|
|
# =============================================================================
|
|
# TokenBatch Tests
|
|
# =============================================================================
|
|
|
|
def test_token_batch_basic():
|
|
"""Test basic batch operations."""
|
|
batch = TokenBatch()
|
|
batch.add(ImageToken("brown hair", 0.9, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("long hair", 0.8, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("a woman", 0.95, TokenSource.FLORENCE_CAPTION, TokenType.PHRASE))
|
|
|
|
assert len(batch) == 3
|
|
print("[PASS] test_token_batch_basic")
|
|
|
|
|
|
def test_token_batch_filter_by_source():
|
|
"""Test filtering tokens by source."""
|
|
batch = TokenBatch()
|
|
batch.add(ImageToken("brown hair", 0.9, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("long hair", 0.8, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("a woman", 0.95, TokenSource.FLORENCE_CAPTION, TokenType.PHRASE))
|
|
|
|
wd14_tokens = batch.filter_by_source(TokenSource.WD14)
|
|
assert len(wd14_tokens) == 2
|
|
print("[PASS] test_token_batch_filter_by_source")
|
|
|
|
|
|
def test_token_batch_filter_by_confidence():
|
|
"""Test filtering tokens by confidence threshold."""
|
|
batch = TokenBatch()
|
|
batch.add(ImageToken("high", 0.9, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("medium", 0.5, TokenSource.WD14, TokenType.TAG))
|
|
batch.add(ImageToken("low", 0.2, TokenSource.WD14, TokenType.TAG))
|
|
|
|
high_conf = batch.filter_by_confidence(0.8)
|
|
assert len(high_conf) == 1
|
|
assert high_conf[0].text == "high"
|
|
print("[PASS] test_token_batch_filter_by_confidence")
|
|
|
|
|
|
# =============================================================================
|
|
# Category Tests
|
|
# =============================================================================
|
|
|
|
def test_canonical_categories():
|
|
"""Test all 9 canonical categories exist."""
|
|
expected = [
|
|
"quality_boosters", "subject", "subject_details", "action_pose",
|
|
"environment", "lighting", "style_medium", "composition", "technical"
|
|
]
|
|
for exp in expected:
|
|
assert any(c.value == exp for c in CanonicalCategory), f"Missing category: {exp}"
|
|
print("[PASS] test_canonical_categories")
|
|
|
|
|
|
def test_category_order():
|
|
"""Test category output order."""
|
|
assert len(CATEGORY_ORDER) == 9
|
|
assert CATEGORY_ORDER[0] == CanonicalCategory.QUALITY_BOOSTERS
|
|
assert CATEGORY_ORDER[2] == CanonicalCategory.SUBJECT
|
|
print("[PASS] test_category_order")
|
|
|
|
|
|
def test_subject_detail_types():
|
|
"""Test subject detail subcategories."""
|
|
expected = ["hair", "eyes", "face", "body", "clothing", "skin", "accessories"]
|
|
for exp in expected:
|
|
assert any(s.value == exp for s in SubjectDetailType), f"Missing subcategory: {exp}"
|
|
print("[PASS] test_subject_detail_types")
|
|
|
|
|
|
# =============================================================================
|
|
# Keyword Matching Tests
|
|
# =============================================================================
|
|
|
|
def test_quality_keywords():
|
|
"""Test quality booster keyword matching."""
|
|
keywords = CATEGORY_KEYWORDS[CanonicalCategory.QUALITY_BOOSTERS]
|
|
assert keywords.matches("realistic")
|
|
assert keywords.matches("high quality")
|
|
assert keywords.matches("8k")
|
|
assert not keywords.matches("brown hair")
|
|
print("[PASS] test_quality_keywords")
|
|
|
|
|
|
def test_subject_keywords():
|
|
"""Test subject keyword matching."""
|
|
keywords = CATEGORY_KEYWORDS[CanonicalCategory.SUBJECT]
|
|
assert keywords.matches("1girl")
|
|
assert keywords.matches("solo")
|
|
assert keywords.matches("woman")
|
|
assert not keywords.matches("brown hair")
|
|
print("[PASS] test_subject_keywords")
|
|
|
|
|
|
def test_subject_detail_hair_keywords():
|
|
"""Test hair keyword matching."""
|
|
keywords = SUBJECT_DETAIL_KEYWORDS[SubjectDetailType.HAIR]
|
|
assert keywords.matches("brown hair")
|
|
assert keywords.matches("long hair")
|
|
assert keywords.matches("ponytail")
|
|
assert not keywords.matches("brown eyes")
|
|
print("[PASS] test_subject_detail_hair_keywords")
|
|
|
|
|
|
def test_subject_detail_clothing_keywords():
|
|
"""Test clothing keyword matching."""
|
|
keywords = SUBJECT_DETAIL_KEYWORDS[SubjectDetailType.CLOTHING]
|
|
assert keywords.matches("dress")
|
|
assert keywords.matches("white crop top")
|
|
assert keywords.matches("kimono")
|
|
assert keywords.matches("sari")
|
|
print("[PASS] test_subject_detail_clothing_keywords")
|
|
|
|
|
|
def test_environment_keywords():
|
|
"""Test environment keyword matching."""
|
|
keywords = CATEGORY_KEYWORDS[CanonicalCategory.ENVIRONMENT]
|
|
assert keywords.matches("indoor")
|
|
assert keywords.matches("grey background")
|
|
assert keywords.matches("studio")
|
|
print("[PASS] test_environment_keywords")
|
|
|
|
|
|
def test_lighting_keywords():
|
|
"""Test lighting keyword matching."""
|
|
keywords = CATEGORY_KEYWORDS[CanonicalCategory.LIGHTING]
|
|
assert keywords.matches("soft light")
|
|
assert keywords.matches("high key")
|
|
assert keywords.matches("warm tones")
|
|
print("[PASS] test_lighting_keywords")
|
|
|
|
|
|
# =============================================================================
|
|
# META Filter Tests
|
|
# =============================================================================
|
|
|
|
def test_meta_filter_positive():
|
|
"""Test META patterns are correctly identified."""
|
|
assert is_meta_pattern("the image shows a woman")
|
|
assert is_meta_pattern("standing in the middle of the image")
|
|
assert is_meta_pattern("The overall mood is calm")
|
|
assert is_meta_pattern("showcasing her curves")
|
|
assert is_meta_pattern("conveying a sense of elegance")
|
|
print("[PASS] test_meta_filter_positive")
|
|
|
|
|
|
def test_meta_filter_negative():
|
|
"""Test non-META content passes through."""
|
|
assert not is_meta_pattern("a woman standing")
|
|
assert not is_meta_pattern("brown hair")
|
|
assert not is_meta_pattern("wearing a white dress")
|
|
assert not is_meta_pattern("soft lighting")
|
|
print("[PASS] test_meta_filter_negative")
|
|
|
|
|
|
# =============================================================================
|
|
# Florence Key Mapping Tests
|
|
# =============================================================================
|
|
|
|
def test_florence_key_mapping():
|
|
"""Test florence_analyze key to category mapping."""
|
|
assert get_florence_key_category("camera_angle") == CanonicalCategory.COMPOSITION
|
|
assert get_florence_key_category("art_style") == CanonicalCategory.STYLE_MEDIUM
|
|
assert get_florence_key_category("location") == CanonicalCategory.ENVIRONMENT
|
|
assert get_florence_key_category("clothing") == CanonicalCategory.SUBJECT_DETAILS
|
|
assert get_florence_key_category("gender") == CanonicalCategory.SUBJECT
|
|
assert get_florence_key_category("unknown_key") is None
|
|
print("[PASS] test_florence_key_mapping")
|
|
|
|
|
|
# =============================================================================
|
|
# Run All Tests
|
|
# =============================================================================
|
|
|
|
def run_all_tests():
|
|
"""Run all tests and report results."""
|
|
tests = [
|
|
# ImageToken tests
|
|
test_image_token_creation,
|
|
test_image_token_normalization,
|
|
test_image_token_confidence_clamping,
|
|
test_image_token_weighted_confidence,
|
|
test_image_token_equality,
|
|
test_image_token_key_value,
|
|
# TokenBatch tests
|
|
test_token_batch_basic,
|
|
test_token_batch_filter_by_source,
|
|
test_token_batch_filter_by_confidence,
|
|
# Category tests
|
|
test_canonical_categories,
|
|
test_category_order,
|
|
test_subject_detail_types,
|
|
# Keyword matching tests
|
|
test_quality_keywords,
|
|
test_subject_keywords,
|
|
test_subject_detail_hair_keywords,
|
|
test_subject_detail_clothing_keywords,
|
|
test_environment_keywords,
|
|
test_lighting_keywords,
|
|
# META filter tests
|
|
test_meta_filter_positive,
|
|
test_meta_filter_negative,
|
|
# Florence key mapping tests
|
|
test_florence_key_mapping,
|
|
]
|
|
|
|
passed = 0
|
|
failed = 0
|
|
|
|
print("=" * 60)
|
|
print("Phase 1: Core Infrastructure Tests")
|
|
print("=" * 60)
|
|
print()
|
|
|
|
for test in tests:
|
|
try:
|
|
test()
|
|
passed += 1
|
|
except AssertionError as e:
|
|
print(f"[FAIL] {test.__name__}: {e}")
|
|
failed += 1
|
|
except Exception as e:
|
|
print(f"[FAIL] {test.__name__}: {type(e).__name__}: {e}")
|
|
failed += 1
|
|
|
|
print()
|
|
print("=" * 60)
|
|
print(f"Results: {passed} passed, {failed} failed")
|
|
print("=" * 60)
|
|
|
|
return failed == 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
success = run_all_tests()
|
|
sys.exit(0 if success else 1)
|