from __future__ import annotations import copy import importlib.util import json import sys import unittest import uuid from pathlib import Path from types import SimpleNamespace from unittest.mock import patch import torch ROOT = Path(__file__).resolve().parents[1] COMFY_ROOT = ROOT.parents[1] def load_nodes_module(): if str(COMFY_ROOT) not in sys.path: sys.path.insert(0, str(COMFY_ROOT)) package_name = f"diffusiongemma_grounding_integration_{uuid.uuid4().hex}" spec = importlib.util.spec_from_file_location( package_name, ROOT / "__init__.py", submodule_search_locations=[str(ROOT)], ) if spec is None or spec.loader is None: raise RuntimeError("could not load DiffusionGemma Prompt Builder") package = importlib.util.module_from_spec(spec) sys.modules[package_name] = package spec.loader.exec_module(package) return sys.modules[f"{package_name}.nodes"] class GroundingGuardIntegrationTests(unittest.TestCase): @classmethod def setUpClass(cls) -> None: cls.nodes = load_nodes_module() def runtime(self): return self.nodes.RuntimeConfig( model_path="unused-local-checkpoint", backend="transformers_inprocess", dtype="auto", quantization="none", local_files_only=True, unload_policy="unload_after_run", max_memory_gb=20.0, status={"ready": True, "supports_pixels": True}, ) def target(self): return self.nodes.TargetProfileConfig(target_profile="ltx") def image_context(self): return self.nodes.GemmaContext( user_prompt="Describe the reference faithfully.", images=torch.zeros((1, 8, 8, 3), dtype=torch.float32), source="image", media_metadata={ "source": "image", "reference_image_count": 1, "image_role": "primary_image_source", }, ) def video_context(self, frame_count: int = 5): return self.nodes.GemmaContext( user_prompt="Describe the video faithfully.", images=torch.zeros((frame_count, 8, 8, 3), dtype=torch.float32), source="video", media_metadata={ "source": "video", "sampled_indices": [index * 10 for index in range(frame_count)], "sampled_frame_count": frame_count, "video_sampled_frame_count": frame_count, "source_fps": 10.0, "sample_fps": 1.0, "video_role": "primary_video_recreation_source", }, ) @staticmethod def uncertain_ledger(status: str = "uncertain") -> dict: return { "schema": "dg-grounding-ledger/1", "analysis_status": status, "observed_facts": [], "inferred_facts": [], "creative_additions": [], "uncertainties": ["Visual evidence could not be verified."], "grounding_failure_reasons": ["insufficient visual certainty"], } @staticmethod def grounded_ledger(registry: dict) -> dict: facts: list[dict] = [] fact_index = 1 for asset in registry.get("assets", []): samples = asset.get("samples", []) chosen_samples = samples if asset.get("kind") == "video" and len(samples) > 1: chosen_samples = [samples[0], samples[-1]] for sample in chosen_samples: evidence = { "asset_id": asset["asset_id"], "sample_ordinal": sample["sample_ordinal"], "source_frame_index": sample["source_frame_index"], "timecode_seconds": sample["timecode_seconds"], } category = "action" if asset.get("kind") == "video" else "object" facts.append( { "fact_id": f"fact-{fact_index}", "claim": f"A directly visible fact from {asset['asset_id']}.", "confidence": "high", "categories": [category], "evidence": [evidence], "typed_claims": [], } ) fact_index += 1 return { "schema": "dg-grounding-ledger/1", "analysis_status": "grounded", "observed_facts": facts, "inferred_facts": [], "creative_additions": [], "uncertainties": [], "grounding_failure_reasons": [], } @staticmethod def packet(prompt: str = "A verified generator prompt.", ledger: dict | None = None) -> dict: packet = { "ltx_prompt": prompt, "ideogram_prompt": "", "minimax_h3_prompt": "", "negative_prompt": "", "scene_segments": [], "metadata": { "target_profile": "ltx", "ready_for_generation": True, "blocked_reasons": [], }, } if ledger is not None: packet["grounding_ledger"] = copy.deepcopy(ledger) return packet def backend_result( self, ledger: dict, *, transport_confirmed: bool = True, telemetry_refusal: bool = False, ): return self.nodes.BackendRunResult( decoded_text=json.dumps(ledger), transport={ "pixel_transport_confirmed": transport_confirmed, "not_applicable": False, "mismatch_reasons": [], }, telemetry={ "schema_version": "dg-denoising-telemetry/1", "forward_count": 2, "snapshots": [{"canvas": 1, "step": 1}], "stream_ended": True, "final_output": {"text": [], "token_count": [1]}, "errors": [], "refusal": {"detected": telemetry_refusal}, }, effective_sampling={"profile": "checkpoint_defaults"}, timing={"total_seconds": 0.01}, ) def truncated_evidence_result(self, registry: dict, complete_fact_count: int = 5): ledger = self.grounded_ledger(registry) template = ledger["observed_facts"][0] facts: list[dict] = [] for fact_index in range(1, complete_fact_count + 2): fact = copy.deepcopy(template) fact["fact_id"] = f"fact-{fact_index}" fact["claim"] = f"Visible detail {fact_index} deliberately described." facts.append(fact) ledger["observed_facts"] = facts decoded_text = json.dumps(ledger) partial_claim = f"Visible detail {complete_fact_count + 1} deliberately" decoded_text = decoded_text[: decoded_text.index(partial_claim) + len(partial_claim)] result = self.backend_result(ledger) result.decoded_text = decoded_text return result @staticmethod def legacy_result(packet: dict): return packet, "compiler raw", "", "", True, "", packet["metadata"] def test_settings_and_cot_public_contract_preserve_existing_outputs(self) -> None: settings_inputs = self.nodes.DiffusionGemmaGroundingGuardSettings.INPUT_TYPES() required = settings_inputs["required"] optional = settings_inputs["optional"] self.assertEqual(required["mode"][0], ["off", "audit", "strict"]) self.assertEqual(required["mode"][1]["default"], "audit") self.assertTrue(required["retry_on_uncertain"][1]["default"]) self.assertEqual(required["sampling_profile"][1]["default"], "checkpoint_defaults") self.assertEqual(required["seed"][1]["max"], (1 << 64) - 1) self.assertTrue(optional["external_evidence_json"][1]["forceInput"]) self.assertEqual( optional["evidence_token_budget"][0], ["auto", "768", "1024", "1280"], ) self.assertEqual(optional["evidence_token_budget"][1]["default"], "auto") self.assertTrue(optional["evidence_token_budget"][1]["advanced"]) self.assertEqual( self.nodes.DiffusionGemmaGroundingGuardSettings.RETURN_TYPES, ("DG_GROUNDING_GUARD_CONFIG", "STRING"), ) cot_inputs = self.nodes.DiffusionGemmaCoTGenerator.INPUT_TYPES() self.assertIn("grounding_guard_config", cot_inputs["optional"]) self.assertEqual( self.nodes.DiffusionGemmaCoTGenerator.RETURN_NAMES[:4], ("final_json", "reasoning_text", "raw_response", "metadata_json"), ) self.assertEqual( self.nodes.DiffusionGemmaCoTGenerator.RETURN_NAMES[4:], ("grounding_status", "grounding_report_json"), ) config, config_json = self.nodes.DiffusionGemmaGroundingGuardSettings().build() self.assertEqual(config.mode, "audit") self.assertEqual(config.evidence_token_budget, "auto") self.assertEqual(json.loads(config_json)["sampling_settings"]["max_denoising_steps"], 48) self.assertEqual(json.loads(config_json)["evidence_token_budget"], "auto") legacy_config, _ = self.nodes.DiffusionGemmaGroundingGuardSettings().build( "strict", False, "checkpoint_defaults", 7, False, "legacy_trace", '{"schema":"legacy-external"}', ) self.assertEqual(legacy_config.evidence_token_budget, "auto") self.assertEqual( json.loads(legacy_config.external_evidence_json)["schema"], "legacy-external", ) selected_config, selected_json = self.nodes.DiffusionGemmaGroundingGuardSettings().build( evidence_token_budget="1280" ) self.assertEqual(selected_config.evidence_token_budget, "1280") self.assertEqual(json.loads(selected_json)["evidence_token_budget"], "1280") def test_strict_guard_block_survives_splitter_and_generation_gate(self) -> None: guard_reasons = [ "visual_grounding_unverified", "visual_grounding_insufficient_coverage", ] guard_report = { "schema": "dg-grounding-report/1", "analysis_status": "uncertain", "decision": "block", "would_block": True, "grounding_guard_would_block": True, "blocked_reasons": guard_reasons, "retry_reasons": ["role coverage remained incomplete after retry"], "validation": { "errors": ["ledger:role_specific_category_missing:picture:2"] }, } context = self.image_context() splitter = self.nodes.DiffusionGemmaJSONSplitter() gate = self.nodes.DiffusionGemmaGenerationGate() prompt_index = {"ltx": 0, "ideogram4": 1, "minimax_h3": 13} for profile in prompt_index: with self.subTest(profile=profile): target = self.nodes.TargetProfileConfig( target_profile=profile, minimax_h3_mode="t2va", ) blocked_packet = self.nodes._blocked_grounding_packet( target, copy.deepcopy(guard_report), ) split = splitter.split(json.dumps(blocked_packet), context, target) metadata = json.loads(split[4]) self.assertFalse(split[12]) self.assertEqual(split[0], "") self.assertEqual(split[1], "") self.assertEqual(split[13], "") self.assertEqual(metadata["blocked_reasons"], guard_reasons) self.assertFalse(metadata["claim_verification_passed"]) with self.assertRaises(ValueError) as raised: gate.gate(split[prompt_index[profile]], split[12], split[4]) message = str(raised.exception) self.assertIn("Grounding Guard blocked generation", message) self.assertIn("visual_grounding_insufficient_coverage", message) self.assertIn( "ledger:role_specific_category_missing:picture:2", message ) self.assertNotIn("minimax_h3_ref_missing", message) self.assertNotIn("empty_generation_prompt", message) stale_metadata = { "blocked_reasons": ["empty_generation_prompt"], "grounding_guard": guard_report, } with self.assertRaisesRegex( ValueError, "visual_grounding_insufficient_coverage" ): gate.gate( "A stale splitter incorrectly supplied this fallback prompt.", True, json.dumps(stale_metadata), ) for broken_decision in (None, "blok"): with self.subTest(broken_decision=broken_decision): malformed_report = copy.deepcopy(guard_report) malformed_report["config"] = {"mode": "strict"} if broken_decision is None: malformed_report.pop("decision") else: malformed_report["decision"] = broken_decision target = self.nodes.TargetProfileConfig(target_profile="ltx") blocked_packet = self.nodes._blocked_grounding_packet( target, malformed_report, ) split = splitter.split(json.dumps(blocked_packet), context, target) self.assertFalse(split[12]) self.assertEqual(split[0], "") self.assertEqual( json.loads(split[4])["blocked_reasons"], guard_reasons ) with self.assertRaisesRegex( ValueError, "visual_grounding_insufficient_coverage" ): gate.gate(split[0], split[12], split[4]) audit_metadata = { "grounding_guard": { **guard_report, "decision": "warn", "config": {"mode": "audit"}, } } self.assertEqual( gate.gate("Audit mode remains non-blocking.", True, json.dumps(audit_metadata)), ("Audit mode remains non-blocking.",), ) def test_branch_generation_gate_blocks_only_its_prompt_output(self) -> None: gate = self.nodes.DiffusionGemmaBranchGenerationGate() metadata = { "blocked_reasons": [ "json_parse_invalid", "minimax_h3_ref_missing_summary", ] } blocked_prompt, status, ready = gate.gate( "An invalid H3 prompt that must not reach the sampler.", False, json.dumps(metadata), ) self.assertIsInstance(blocked_prompt, self.nodes.ExecutionBlocker) self.assertIsNone(blocked_prompt.message) self.assertFalse(ready) self.assertIn("did not pass validation", status) self.assertIn("json_parse_invalid", status) self.assertIn("minimax_h3_ref_missing_summary", status) self.assertIn("produced no media output", status) self.assertIn("select refresh once", status) valid_prompt = "A validated prompt for an independent comparison branch." self.assertEqual( gate.gate(valid_prompt, True, json.dumps({"blocked_reasons": []})), ( valid_prompt, "DiffusionGemma branch is ready for generation.", True, ), ) def test_declared_h3_asset_without_tensor_blocks_strict_before_generation(self) -> None: runtime = self.runtime() context = self.nodes.GemmaContext( user_prompt="Use the declared character reference.", images=None, source="image", media_metadata={ "source": "image", "minimax_h3_mode": "ref2va", "minimax_h3_reference_image_batch_count": 1, "minimax_h3_reference_manifest": ( ": [dg:identity,appearance] protagonist identity" ), }, ) target = self.nodes.TargetProfileConfig( target_profile="minimax_h3", minimax_h3_mode="ref2va", ) with patch.object( self.nodes, "_run_generation_packet_legacy" ) as legacy, patch.object(self.nodes, "_maybe_unload") as unload: result = self.nodes._run_generation_packet( runtime, context, target, grounding_guard_config={"mode": "strict"}, ) legacy.assert_not_called() unload.assert_called_once_with(runtime) self.assertEqual(result[0]["minimax_h3_prompt"], "") self.assertFalse(result[-1]["ready_for_generation"]) self.assertEqual( result[-1]["blocked_reasons"][:2], ["visual_grounding_unverified", "visual_transport_error"], ) self.assertEqual(result[-1]["grounding_guard"]["model_call_count"], 0) self.assertIn( "picture:1", result[-1]["grounding_guard"]["asset_registry"][ "unattached_declared_visual_asset_ids" ], ) def test_declared_h3_asset_without_tensor_only_warns_in_audit(self) -> None: runtime = self.runtime() context = self.nodes.GemmaContext( user_prompt="Use the declared character reference.", images=None, source="image", media_metadata={ "source": "image", "minimax_h3_mode": "ref2va", "minimax_h3_reference_image_batch_count": 1, "minimax_h3_reference_manifest": ( ": [dg:identity,appearance] protagonist identity" ), }, ) target = self.nodes.TargetProfileConfig( target_profile="minimax_h3", minimax_h3_mode="ref2va", ) packet = self.packet("") packet["ltx_prompt"] = "" packet["minimax_h3_prompt"] = "legacy H3 prompt retained in audit" packet["metadata"]["target_profile"] = "minimax_h3" with patch.object( self.nodes, "_run_generation_packet_legacy", return_value=self.legacy_result(packet), ) as legacy, patch.object(self.nodes, "_maybe_unload"): result = self.nodes._run_generation_packet( runtime, context, target, grounding_guard_config={"mode": "audit"}, ) legacy.assert_called_once() self.assertEqual( result[0]["minimax_h3_prompt"], "legacy H3 prompt retained in audit", ) self.assertTrue(result[-1]["ready_for_generation"]) self.assertEqual(result[-1]["grounding_guard"]["decision"], "warn") self.assertTrue( result[-1]["grounding_guard"]["grounding_guard_would_block"] ) self.assertEqual( result[-1]["grounding_guard"]["blocked_reasons"][:2], ["visual_grounding_unverified", "visual_transport_error"], ) def test_strict_evidence_rejects_prose_prefix_but_accepts_native_thinking(self) -> None: ledger = self.grounded_ledger( self.nodes._grounding_asset_registry(self.image_context()) ) exact = json.dumps(ledger) self.assertEqual( self.nodes._strict_evidence_json_payload(f"check citations\n{exact}"), exact, ) with self.assertRaisesRegex( self.nodes.GroundingLedgerError, "without a conversational prefix", ): self.nodes._strict_evidence_json_payload( f"<|thought|>check citations<|end_thought|><|final_json|>{exact}<|end_final_json|>" ) with self.assertRaisesRegex( self.nodes.GroundingLedgerError, "without a conversational prefix", ): self.nodes._strict_evidence_json_payload(f"Here is the evidence:\n{exact}") with self.assertRaisesRegex( self.nodes.GroundingLedgerError, "root must be the ledger", ): self.nodes._strict_evidence_json_payload( json.dumps({"grounding_ledger": ledger, "ltx_prompt": "must not pass"}) ) def test_strict_evidence_example_uses_the_authoritative_h3_asset_id(self) -> None: registry = self.nodes.build_asset_registry( { "source": "image", "minimax_h3_mode": "ref2va", "minimax_h3_reference_image_batch_count": 1, "minimax_h3_reference_manifest": ( ": [dg:identity,appearance] protagonist identity" ), }, 1, ) prompt = self.nodes._strict_evidence_prompt(registry) self.assertIn('"asset_id": "picture:1"', prompt) self.assertNotIn('"asset_id": "image:1"', prompt) self.assertIn("Use no more than 4 observed facts total", prompt) self.assertIn("Completing and closing the strict JSON object takes priority", prompt) retry_prompt = self.nodes._strict_evidence_prompt(registry, retry=True) self.assertIn("previous ledger did not pass validation", retry_prompt) self.assertIn("Keep every JSON object and array closed", retry_prompt) self.assertEqual(self.nodes._strict_evidence_max_new_tokens(2048), 768) self.assertEqual(self.nodes._strict_evidence_max_new_tokens(2048, retry=True), 1024) for budget, expected in (("768", 768), ("1024", 1024), ("1280", 1280)): with self.subTest(evidence_token_budget=budget): self.assertEqual( self.nodes._strict_evidence_max_new_tokens( 512, evidence_token_budget=budget, ), expected, ) self.assertEqual( self.nodes._strict_evidence_max_new_tokens( 512, retry=True, evidence_token_budget=budget, ), expected, ) def test_h3_grounding_annotations_are_host_only(self) -> None: manifest = ( ": [dg:identity,appearance] protagonist identity\n" "